+
    œ°³jƒ»  ã                   óÞ  € ^ RI Ht ^ RIt^ RIt^ RIHt ^ RIHt ^RIH	t
 ^RIHt ^RIHt ^RIHt ^RIHt ^R	IHt ^R
IHt ^RIHt ^RIHtHtHt ^RIHtHtH t  ^RI!H"t" ^RI#H$t$H%t%H&t& ^RI'H(t(H)t) ^RI*H+t+H,t, ^RI-H.t. ]&P^                  ! ]04      t1 ! R R]Pd                  4      t3 ! R R]Pd                  4      t4 ! R R]4      t5 ! R R]4      t6 ! R R]4      t7 ! R R]Pd                  4      t8 ! R R ]Pd                  4      t9R>R! R" llt: ! R# R$]Pd                  4      t; ! R% R&]Pd                  4      t< ! R' R(]4      t= ! R) R*]Pd                  4      t> ! R+ R,]Pd                  4      t? ! R- R.]4      t@ ! R/ R0]Pd                  4      tA]% ! R1 R2]4      4       tBR?R3 R4 lltC]% ! R5 R6]B4      4       tD]%! R7R87       ! R9 R:]B4      4       tE]%! R;R87       ! R< R=]B4      4       tF. R@OtGR# )Aé    )ÚCallableN)ÚCrossEntropyLoss)Úinitialization)ÚACT2FN)Úfilter_output_hidden_states)Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)Úcreate_bidirectional_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚCausalLMOutputÚSequenceClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModelÚ*get_torch_context_manager_or_global_device)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs©ÚHubertConfigc                   ó8   a a€ ] tR t^0t oV 3R ltR tRtVtV ;t# )ÚHubertPositionalConvEmbeddingc                óz  <€ \         SV `  4        \        P                  ! VP                  VP                  VP
                  VP
                  ^,          VP                  R7      V n        RV n        \        VRR4      '       d(   \        P                  ! VP                  4      V n        EMÇ\        P                  P                  p\        \        P                  P                  R4      '       d%   \        P                  P                  P                  p\        4       '       Ed5   ^ RIpVP"                  P%                  V P                  P&                  ^ R7      ;_uu_ 4        V! V P                  R^R7      V n        RRR4       \        V P                  R	4      '       dV   V P                  P                  P&                  P(                  pV P                  P                  P&                  P*                  pM,V P                  P,                  pV P                  P.                  pVP"                  P1                  W4       VP"                  P1                  W4       MV! V P                  R^R7      V n        \3        VP
                  4      V n        \6        VP8                  ,          V n        R#   + '       g   i     EL7; i)
é   )Úkernel_sizeÚpaddingÚgroupsNÚconv_pos_batch_normFÚweight_norm©Úmodifier_rankÚweight)ÚnameÚdimÚparametrizations)ÚsuperÚ__init__ÚnnÚConv1dÚhidden_sizeÚnum_conv_pos_embeddingsÚnum_conv_pos_embedding_groupsÚconvÚ
batch_normÚgetattrÚBatchNorm1dÚutilsr%   Úhasattrr+   r   Ú	deepspeedÚzeroÚGatheredParametersr(   Ú	original0Ú	original1Úweight_gÚweight_vÚregister_external_parameterÚHubertSamePadLayerr"   r   Úfeat_extract_activationÚ
activation)ÚselfÚconfigr%   r9   r>   r?   Ú	__class__s   &&    €Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/hubert/modeling_hubert.pyr-   Ú&HubertPositionalConvEmbedding.__init__1   sÔ  ø€ Ü‰ÑÔÜ—I’IØ×ÑØ×ÑØ×6Ñ6Ø×2Ñ2°aÕ7Ø×7Ñ7ô
ˆŒ	ð ˆŒÜ�6Ð0°%×8Ò8Ü Ÿnšn¨V×-?Ñ-?Ó@ˆDŽOäŸ(™(×.Ñ.ˆKÜ”r—x‘x×0Ñ0°-×@Ò@Ü Ÿh™h×7Ñ7×CÑC�ä)×+Ó+Û à—^‘^×6Ñ6°t·y±y×7GÑ7GÐWXÐ6×YÕYÙ +¨D¯I©I¸HÈ!Ô L�D”I÷ Zä˜4Ÿ9™9Ð&8×9Ò9Ø#Ÿy™y×9Ñ9×@Ñ@×JÑJ�HØ#Ÿy™y×9Ñ9×@Ñ@×JÑJ‘Hà#Ÿy™y×1Ñ1�HØ#Ÿy™y×1Ñ1�HØ—‘×:Ñ:¸4ÔJØ—‘×:Ñ:¸4ÕJá'¨¯	©	¸ÀaÔH�”	ä)¨&×*HÑ*HÓIˆŒÜ  ×!?Ñ!?Õ@ˆŽ÷ Z×YÐYús   Å J)Ê)J:	c                óò   € VP                  ^^4      pV P                  e   V P                  V4      pV P                  V4      pV P                  V4      pV P	                  V4      pVP                  ^^4      pV# )é   )Ú	transposer4   r3   r"   rC   ©rD   Úhidden_statess   &&rG   ÚforwardÚ%HubertPositionalConvEmbedding.forwardV   sn   € Ø%×/Ñ/°°1Ó5ˆØ�?‰?Ò&Ø ŸO™O¨MÓ:ˆMØŸ	™	 -Ó0ˆØŸ™ ]Ó3ˆØŸ™¨Ó6ˆà%×/Ñ/°°1Ó5ˆØÐó    )rC   r4   r3   r"   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r-   rN   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rF   Ú__classdict__s   @@rG   r   r   0   s   ù‡ € õ#A÷J	ò 	rP   r   c                   ó8   a a€ ] tR t^bt oV 3R ltR tRtVtV ;t# )rA   c                ó`   <€ \         SV `  4        V^,          ^ 8X  d
   ^V n        R# ^ V n        R# )r    N)r,   r-   Únum_pad_remove)rD   r1   rF   s   &&€rG   r-   ÚHubertSamePadLayer.__init__c   s)   ø€ Ü‰ÑÔØ#:¸QÕ#>À!Ô#C˜aˆÖÈˆÖrP   c                óZ   € V P                   ^ 8”  d   VRRRV P                   ) 13,          pV# )r   ºNNNN©r]   rL   s   &&rG   rN   ÚHubertSamePadLayer.forwardg   s6   € Ø×Ñ Ô"Ø)¨!¨QÐ0F°4×3FÑ3FÐ2FÐ0FÐ*FÕGˆMØÐrP   ra   rQ   rY   s   @@rG   rA   rA   b   s   ù‡ € õK÷ò rP   rA   c                   ó<   a a€ ] tR t^mt oRV 3R lltR tRtVtV ;t# )ÚHubertNoLayerNormConvLayerc                óœ  <€ \         SV `  4        V^ 8”  d   VP                  V^,
          ,          M^V n        VP                  V,          V n        \
        P                  ! V P                  V P                  VP                  V,          VP                  V,          VP                  R7      V n
        \        VP                  ,          V n        R# )r   ©r!   ÚstrideÚbiasN)r,   r-   Úconv_dimÚin_conv_dimÚout_conv_dimr.   r/   Úconv_kernelÚconv_strideÚ	conv_biasr3   r   rB   rC   ©rD   rE   Úlayer_idrF   s   &&&€rG   r-   Ú#HubertNoLayerNormConvLayer.__init__n   s—   ø€ Ü‰ÑÔØ<DÀq¼L˜6Ÿ?™?¨8°a­<Ö8ÈaˆÔØ"ŸO™O¨HÕ5ˆÔä—I’IØ×ÑØ×ÑØ×*Ñ*¨8Õ4Ø×%Ñ% hÕ/Ø×!Ñ!ô
ˆŒ	ô ! ×!?Ñ!?Õ@ˆŽrP   c                óJ   € V P                  V4      pV P                  V4      pV# ©N)r3   rC   rL   s   &&rG   rN   Ú"HubertNoLayerNormConvLayer.forward|   s$   € ØŸ	™	 -Ó0ˆØŸ™¨Ó6ˆØÐrP   )rC   r3   rj   rk   ©r   rQ   rY   s   @@rG   rd   rd   m   s   ù‡ € ÷A÷ò rP   rd   c                   ó<   a a€ ] tR t^‚t oRV 3R lltR tRtVtV ;t# )ÚHubertLayerNormConvLayerc                óê  <€ \         SV `  4        V^ 8”  d   VP                  V^,
          ,          M^V n        VP                  V,          V n        \
        P                  ! V P                  V P                  VP                  V,          VP                  V,          VP                  R7      V n
        \
        P                  ! V P                  RR7      V n        \        VP                  ,          V n        R# )r   rf   T)Úelementwise_affineN)r,   r-   ri   rj   rk   r.   r/   rl   rm   rn   r3   Ú	LayerNormÚ
layer_normr   rB   rC   ro   s   &&&€rG   r-   Ú!HubertLayerNormConvLayer.__init__ƒ   s¯   ø€ Ü‰ÑÔØ<DÀq¼L˜6Ÿ?™?¨8°a­<Ö8ÈaˆÔØ"ŸO™O¨HÕ5ˆÔä—I’IØ×ÑØ×ÑØ×*Ñ*¨8Õ4Ø×%Ñ% hÕ/Ø×!Ñ!ô
ˆŒ	ô Ÿ,š, t×'8Ñ'8ÈTÔRˆŒÜ  ×!?Ñ!?Õ@ˆŽrP   c                ó´   € V P                  V4      pVP                  RR4      pV P                  V4      pVP                  RR4      pV P                  V4      pV# )r    éþÿÿÿéÿÿÿÿ)r3   rK   r{   rC   rL   s   &&rG   rN   Ú HubertLayerNormConvLayer.forward’   sV   € ØŸ	™	 -Ó0ˆà%×/Ñ/°°BÓ7ˆØŸ™¨Ó6ˆØ%×/Ñ/°°BÓ7ˆàŸ™¨Ó6ˆØÐrP   ©rC   r3   rj   r{   rk   ru   rQ   rY   s   @@rG   rw   rw   ‚   s   ù‡ € ÷A÷ò rP   rw   c                   ó<   a a€ ] tR t^�t oRV 3R lltR tRtVtV ;t# )ÚHubertGroupNormConvLayerc                ó   <€ \         SV `  4        V^ 8”  d   VP                  V^,
          ,          M^V n        VP                  V,          V n        \
        P                  ! V P                  V P                  VP                  V,          VP                  V,          VP                  R7      V n
        \        VP                  ,          V n        \
        P                  ! V P                  V P                  RR7      V n        R# )r   rf   T)Ú
num_groupsÚnum_channelsÚaffineN)r,   r-   ri   rj   rk   r.   r/   rl   rm   rn   r3   r   rB   rC   Ú	GroupNormr{   ro   s   &&&€rG   r-   Ú!HubertGroupNormConvLayer.__init__ž   s¹   ø€ Ü‰ÑÔØ<DÀq¼L˜6Ÿ?™?¨8°a­<Ö8ÈaˆÔØ"ŸO™O¨HÕ5ˆÔä—I’IØ×ÑØ×ÑØ×*Ñ*¨8Õ4Ø×%Ñ% hÕ/Ø×!Ñ!ô
ˆŒ	ô ! ×!?Ñ!?Õ@ˆŒäŸ,š,°$×2CÑ2CÐRV×RcÑRcÐlpÔqˆŽrP   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# rs   )r3   r{   rC   rL   s   &&rG   rN   Ú HubertGroupNormConvLayer.forward®   s2   € ØŸ	™	 -Ó0ˆØŸ™¨Ó6ˆØŸ™¨Ó6ˆØÐrP   r�   ru   rQ   rY   s   @@rG   rƒ   rƒ   �   s   ù‡ € ÷r÷ ò rP   rƒ   c                   óB   a a€ ] tR t^µt oRtV 3R ltR tR tRtVt	V ;t
# )ÚHubertFeatureEncoderz.Construct the features from raw audio waveformc           	     ó   <€ \         SV `  4        VP                  R 8X  dR   \        V^ R7      .\	        VP
                  ^,
          4       Uu. uF  p\        W^,           R7      NK  	  up,           pMYVP                  R8X  d0   \	        VP
                  4       Uu. uF  p\        WR7      NK  	  ppM\        RVP                   R24      h\        P                  ! V4      V n        RV n        RV n        R# u upi u upi )Úgroup)rp   Úlayerz`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FTN)r,   r-   Úfeat_extract_normrƒ   ÚrangeÚnum_feat_extract_layersrd   rw   Ú
ValueErrorr.   Ú
ModuleListÚconv_layersÚgradient_checkpointingÚ_requires_grad)rD   rE   Úir–   rF   s   &&  €rG   r-   ÚHubertFeatureEncoder.__init__¸   sð   ø€ Ü‰ÑÔà×#Ñ# wÔ.Ü3°FÀQÔGÐHÜLQÐRX×RpÑRpÐstÕRtÔLuóLÙLuÀqÔ*¨6À½E×BÑLuñLõ ‰Kð ×%Ñ%¨Ô0ÜQVÐW]×WuÑWuÔQvÓwÑQvÈAÔ3°F×GÑQvˆKÐwˆKäØ0°×1IÑ1IÐ0JÐJsÐtóð ô Ÿ=š=¨Ó5ˆÔØ&+ˆÔ#Ø"ˆÖùòLùò xs   ÁC6ÂC;c                óP   € V P                  4        F
  pR Vn        K  	  R V n        R# )FN)Ú
parametersÚrequires_gradr˜   ©rD   Úparams   & rG   Ú_freeze_parametersÚ'HubertFeatureEncoder._freeze_parametersÉ   s#   € Ø—_‘_Ö&ˆEØ"'ˆEÖñ 'à#ˆÖrP   c                ó¤   € VR,          pV P                   '       d   V P                  '       d   RVn        V P                   F  pV! V4      pK  	  V# )r`   T©r`   N)r˜   Útrainingr�   r–   )rD   Úinput_valuesrM   Ú
conv_layers   &&  rG   rN   ÚHubertFeatureEncoder.forwardÎ   sL   € Ø$ WÕ-ˆð ××Ð 4§=§= =Ø*.ˆMÔ'à×*Ô*ˆJÙ& }Ó5ŠMñ +ð ÐrP   )r˜   r–   r—   )rR   rS   rT   rU   Ú__doc__r-   r    rN   rV   rW   rX   rY   s   @@rG   r�   r�   µ   s   ù‡ € Ù8õ#ò"$÷

ò 
rP   r�   c                   ó8   a a€ ] tR t^Ût oV 3R ltR tRtVtV ;t# )ÚHubertFeatureProjectionc                ó’  <€ \         SV `  4        VP                  V n        V P                  '       d9   \        P                  ! VP
                  R,          VP                  R7      V n        \        P                  ! VP
                  R,          VP                  4      V n
        \        P                  ! VP                  4      V n        R# )rJ   ©ÚepsNr   )r,   r-   Úfeat_proj_layer_normr.   rz   ri   Úlayer_norm_epsr{   ÚLinearr0   Ú
projectionÚDropoutÚfeat_proj_dropoutÚdropout©rD   rE   rF   s   &&€rG   r-   Ú HubertFeatureProjection.__init__Ü   s€   ø€ Ü‰ÑÔØ$*×$?Ñ$?ˆÔ!Ø×$×$Ð$Ü Ÿlšl¨6¯?©?¸2Õ+>ÀF×DYÑDYÔZˆDŒOÜŸ)š) F§O¡O°BÕ$7¸×9KÑ9KÓLˆŒÜ—z’z &×":Ñ":Ó;ˆŽrP   c                ó�   € V P                   '       d   V P                  V4      pV P                  V4      pV P                  V4      pV# rs   )r®   r{   r±   r´   rL   s   &&rG   rN   ÚHubertFeatureProjection.forwardä   s>   € à×$×$Ð$Ø ŸO™O¨MÓ:ˆMØŸ™¨Ó6ˆØŸ™ ]Ó3ˆØÐrP   )r´   r®   r{   r±   rQ   rY   s   @@rG   rª   rª   Û   s   ù‡ € õ<÷ò rP   rª   c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R,          R\        R	\
        \        ,          /# )
r    ÚmoduleÚqueryÚkeyÚvalueÚattention_maskNÚscalingr´   Úkwargs)r.   ÚModuleÚtorchÚTensorÚfloatr   r   )Úformats   "rG   Ú__annotate__rÆ   í   s{   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô �T�\ð%ô ð%ô Ô'Õ(ñ%rP   c                 ó¶  € Vf   VP                  R4      R,          p\        P                  ! WP                  ^^4      4      V,          pVe	   W„,           p\        P
                  P                  VRR7      p\        P
                  P                  W†V P                  R7      p\        P                  ! Wƒ4      p	V	P                  ^^4      P                  4       p	W˜3# )N©r*   )Úpr¤   r   ç      à¿)
ÚsizerÂ   ÚmatmulrK   r.   Ú
functionalÚsoftmaxr´   r¤   Ú
contiguous)
rº   r»   r¼   r½   r¾   r¿   r´   rÀ   Úattn_weightsÚattn_outputs
   &&&&&&&,  rG   Úeager_attention_forwardrÒ   í   s±   € ð ‚Ø—*‘*˜R“. DÕ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÕE€LàÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$rP   c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
ÚHubertAttentioni	  z=Multi-headed attention from 'Attention Is All You Need' paperc                óR   <€ V ^8„  d   QhRS[ RS[ RS[RS[RS[RS[RS[R,          /# )	r    Ú	embed_dimÚ	num_headsr´   Ú
is_decoderrh   Ú	is_causalrE   N)ÚintrÄ   Úboolr   )rÅ   rZ   s   "€rG   rÆ   ÚHubertAttention.__annotate__  s_   ø€ ÷ Cñ CáðCñ ðCñ ð	Cñ
 ðCñ ðCñ ðCñ ˜tÕ#ñCrP   c                ó  <€ \         SV `  4        Wn        W n        W0n        W,          V n        Wpn        V P
                  V,          V P                  8w  d   \        R V P                   RV R24      hV P
                  R,          V n        W@n	        W`n
        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        \        P                  ! WVR7      V n        R# )z;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).)rh   NrÊ   )r,   r-   rÖ   r×   r´   Úhead_dimrE   r”   r¿   rØ   rÙ   r.   r°   Úk_projÚv_projÚq_projÚout_proj)	rD   rÖ   r×   r´   rØ   rh   rÙ   rE   rF   s	   &&&&&&&&€rG   r-   ÚHubertAttention.__init__  sÎ   ø€ ô 	‰ÑÔØ"ŒØ"ŒØŒØ!Õ.ˆŒØŒà�M‰M˜IÕ%¨$¯.©.Ô8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÕ*ˆŒØ$ŒØ"Œä—i’i 	¸4Ô@ˆŒÜ—i’i 	¸4Ô@ˆŒÜ—i’i 	¸4Ô@ˆŒÜŸ	š	 )¸TÔBˆŽrP   c                ó   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# )r    rM   Úkey_value_statesNr¾   Úoutput_attentionsrÀ   Úreturn)rÂ   rÃ   rÛ   r   r   Útuple)rÅ   rZ   s   "€rG   rÆ   rÜ   +  s~   ø€ ÷ 0)ñ 0)á—|‘|ð0)ñ  Ÿ,™,¨Õ-ð0)ñ Ÿ™ tÕ+ð	0)ñ
   $�;ð0)ñ Ñ-Õ.ð0)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ0)rP   c           	     ó  € VRJpVP                   RR p. VORNV P                  N5pV P                  V4      P                  V4      P	                  ^^4      p	V'       d   TMTp
. V
P                   RR ORNV P                  N5pV P                  V
4      P                  V4      P	                  ^^4      pV P                  V
4      P                  V4      P	                  ^^4      p\        P                  ! V P                  P                  \        4      pT! V V	VVV3RV P                  '       g   RMV P                  RV P                  RV/VB w  ppVP                  ! . VORN5!  P!                  4       pV P#                  V4      pVV3# )z#Input shape: Batch x Time x ChannelNr´   ç        r¿   ræ   r   )ÚshaperÞ   rá   ÚviewrK   rß   rà   r   Úget_interfacerE   Ú_attn_implementationrÒ   r¤   r´   r¿   ÚreshaperÏ   râ   )rD   rM   rå   r¾   ræ   rÀ   Úis_cross_attentionÚinput_shapeÚhidden_shapeÚquery_statesÚcurrent_statesÚkv_shapeÚ
key_statesÚvalue_statesÚattention_interfacerÑ   rÐ   s   &&&&&,           rG   rN   ÚHubertAttention.forward+  s–  € ð .°TÐ9Ðð $×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆð —{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆç-?Ñ)À]ˆØB�^×)Ñ)¨#¨2Ð.ÐB°ÐB°D·M±MÑBˆØ—[‘[ Ó0×5Ñ5°hÓ?×IÑIÈ!ÈQÓOˆ
Ø—{‘{ >Ó2×7Ñ7¸ÓA×KÑKÈAÈqÓQˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ
%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð
%
ð —L‘Lð
%
ð 0ð
%
ð ñ
%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—m‘m KÓ0ˆà˜LÐ(Ð(rP   )rE   r´   rÖ   rÞ   rÙ   rØ   rß   r×   râ   rá   r¿   rà   )rê   FTFN)NNF)
rR   rS   rT   rU   r¨   r-   rN   rV   rW   rX   rY   s   @@rG   rÔ   rÔ   	  s#   ù‡ € ÙG÷Cõ C÷>0)÷ 0)ò 0)rP   rÔ   c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚHubertFeedForwardi^  c                ó  <€ \         SV `  4        \        P                  ! VP                  4      V n        \        P                  ! VP                  VP                  4      V n	        \        VP                  \        4      '       d   \        VP                  ,          V n        MVP                  V n        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                   4      V n        R # rs   )r,   r-   r.   r²   Úactivation_dropoutÚintermediate_dropoutr°   r0   Úintermediate_sizeÚintermediate_denseÚ
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnÚoutput_denseÚhidden_dropoutÚoutput_dropoutrµ   s   &&€rG   r-   ÚHubertFeedForward.__init___  s®   ø€ Ü‰ÑÔÜ$&§J¢J¨v×/HÑ/HÓ$IˆÔ!ä"$§)¢)¨F×,>Ñ,>À×@XÑ@XÓ"YˆÔÜ�f×'Ñ'¬×-Ò-Ü'-¨f×.?Ñ.?Õ'@ˆDÕ$à'-×'8Ñ'8ˆDÔ$äŸIšI f×&>Ñ&>À×@RÑ@RÓSˆÔÜ Ÿjšj¨×)>Ñ)>Ó?ˆÖrP   c                ó°   € V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P	                  V4      pV# rs   )r   r  rþ   r  r  rL   s   &&rG   rN   ÚHubertFeedForward.forwardl  sX   € Ø×/Ñ/°Ó>ˆØ×0Ñ0°Ó?ˆØ×1Ñ1°-Ó@ˆà×)Ñ)¨-Ó8ˆØ×+Ñ+¨MÓ:ˆØÐrP   )r  r   rþ   r  r  rQ   rY   s   @@rG   rû   rû   ^  s   ù‡ € õ@÷ò rP   rû   c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚHubertEncoderLayeriv  c                ó¸  <€ \         SV `  4        \        VP                  VP                  VP
                  R VR7      V n        \        P                  ! VP                  4      V n
        \        P                  ! VP                  VP                  R7      V n        \        V4      V n        \        P                  ! VP                  VP                  R7      V n        R# )F©rÖ   r×   r´   rØ   rE   r¬   N)r,   r-   rÔ   r0   Únum_attention_headsÚattention_dropoutÚ	attentionr.   r²   r  r´   rz   r¯   r{   rû   Úfeed_forwardÚfinal_layer_normrµ   s   &&€rG   r-   ÚHubertEncoderLayer.__init__w  s�   ø€ Ü‰ÑÔÜ(Ø×(Ñ(Ø×0Ñ0Ø×,Ñ,ØØô
ˆŒô —z’z &×"7Ñ"7Ó8ˆŒÜŸ,š, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ-¨fÓ5ˆÔÜ "§¢¨V×-?Ñ-?ÀV×EZÑEZÔ [ˆÖrP   c                ór   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          /# ©r    rM   r¾   NrÀ   ©rÂ   rÃ   r   r   )rÅ   rZ   s   "€rG   rÆ   ÚHubertEncoderLayer.__annotate__†  s<   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ Ñ+Õ,ñ	rP   c                óÜ   € TpV P                   ! V3R V/VB w  rV P                  V4      pWA,           pV P                  V4      pWP                  V4      ,           pV P	                  V4      pV# ©r¾   )r  r´   r{   r  r  ©rD   rM   r¾   rÀ   Úattn_residualÚ_s   &&&,  rG   rN   ÚHubertEncoderLayer.forward†  sr   € ð &ˆØŸ>š>¨-ÑaÈÐaÐZ`ÑaÑˆØŸ™ ]Ó3ˆØ%Õ5ˆàŸ™¨Ó6ˆØ%×(9Ñ(9¸-Ó(HÕHˆØ×-Ñ-¨mÓ<ˆàÐrP   )r  r´   r  r  r{   rs   rQ   rY   s   @@rG   r  r  v  s   ù‡ € õ\÷÷ ò rP   r  c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚHubertEncoderi˜  c                ó¢  <€ \         SV `  4        Wn        \        V4      V n        \
        P                  ! VP                  VP                  R 7      V n	        \
        P                  ! VP                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\!        V4      NK  	  up4      V n        RV n        R# u upi ©r¬   FN)r,   r-   rE   r   Úpos_conv_embedr.   rz   r0   r¯   r{   r²   r  r´   r•   r’   Únum_hidden_layersr  Úlayersr—   ©rD   rE   r  rF   s   && €rG   r-   ÚHubertEncoder.__init__™  s”   ø€ Ü‰ÑÔØŒÜ;¸FÓCˆÔÜŸ,š, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z’z &×"7Ñ"7Ó8ˆŒÜ—m’mÌÈv×OgÑOgÔIhÓ$iÑIhÀAÔ%7¸Ö%?ÑIhÑ$iÓjˆŒØ&+ˆÖ#ùò %jó   Â#Cc                ór   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          /# r  )rÂ   ÚtensorrÃ   r   r   )rÅ   rZ   s   "€rG   rÆ   ÚHubertEncoder.__annotate__¢  sA   ø€ ÷ !@ñ !@á—|‘|ð!@ñ Ÿ™ tÕ+ð!@ñ Ñ+Õ,ñ	!@rP   c                óˆ  € Ve9   VP                  R4      P                  ^^VP                  ^,          4      p^ W( &   \        V P                  VVR7      pV P                  V4      pWP                  VP                  4      ,           pV P                  V4      pV P                  V4      p\        4       ;'       g    \        V 4      pV P                   Fb  p\        P                  ! . 4      pV P                  ;'       d    W€P                  P                   8  p	V	'       d   V'       g   KW  V! V3RV/VB pKd  	  \#        VR7      # ©N)rE   Úinputs_embedsr¾   r¾   )Úlast_hidden_stater   )Ú	unsqueezeÚrepeatrë   r
   rE   r#  ÚtoÚdevicer{   r´   r   r	   r%  rÂ   Úrandr¤   Ú	layerdropr   ©
rD   rM   r¾   rÀ   Úexpand_attention_maskÚposition_embeddingsÚsynced_gpusr�   Údropout_probabilityÚskip_the_layers
   &&&,      rG   rN   ÚHubertEncoder.forward¢  s  € ð Ò%à$2×$<Ñ$<¸RÓ$@×$GÑ$GÈÈ1Èm×NaÑNaÐbcÕNdÓ$eÐ!Ø45ˆMÐ0Ñ1ä2Ø—;‘;Ø'Ø)ô
ˆð #×1Ñ1°-Ó@ÐØ%×(>Ñ(>¸}×?SÑ?SÓ(TÕTˆØŸ™¨Ó6ˆØŸ™ ]Ó3ˆä0Ó2×RÐRÔ6LÈTÓ6Rˆà—[”[ˆEä"'§*¢*¨R£.Ðà!Ÿ]™]×ZÐZÐ/BÇ[Á[×EZÑEZÑ/ZˆNß!§[¡[á % mÑ ]ÀNÐ ]ÐV\Ñ ]’ñ !ô °Ô?Ð?rP   ©rE   r´   r—   r{   r%  r#  rs   rQ   rY   s   @@rG   r   r   ˜  s   ù‡ € õ,÷!@÷ !@ò !@rP   r   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚHubertAttnAdapterLayeriÆ  c                ó¦  <€ \         SV `  4        VP                  V n        VP                  V n        \        P                  ! V P
                  4      V n        \        P                  ! V P
                  V P                  4      V n
        \        P                  ! 4       V n        \        P                  ! V P                  V P
                  4      V n        R# )z‡
Implements adapter modules directly with 3D tensor weight as parameters and without using ModuleList to speed
up training throughput.
N)r,   r-   Úadapter_attn_dimÚ	input_dimr0   Ú
hidden_dimr.   rz   Únormr°   Úlinear_1ÚReLUÚact_fnÚlinear_2rµ   s   &&€rG   r-   ÚHubertAttnAdapterLayer.__init__Ç  s   ø€ ô
 	‰ÑÔØ×0Ñ0ˆŒØ ×,Ñ,ˆŒä—L’L §¡Ó1ˆŒ	ÜŸ	š	 $§/¡/°4·>±>ÓBˆŒÜ—g’g“iˆŒÜŸ	š	 $§.¡.°$·/±/ÓBˆŽrP   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r    rM   )rÂ   ÚFloatTensor)rÅ   rZ   s   "€rG   rÆ   Ú#HubertAttnAdapterLayer.__annotate__Õ  s   ø€ ÷ ñ ¡U×%6Ñ%6ñ rP   c                óŽ   € V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV# rs   )rD  rE  rG  rH  rL   s   &&rG   rN   ÚHubertAttnAdapterLayer.forwardÕ  s@   € ØŸ	™	 -Ó0ˆàŸ™ mÓ4ˆØŸ™ MÓ2ˆØŸ™ mÓ4ˆàÐrP   )rG  rC  rB  rE  rH  rD  rQ   rY   s   @@rG   r?  r?  Æ  s   ù‡ € õC÷÷ ð rP   r?  c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )Ú!HubertEncoderLayerStableLayerNormiß  c                ó  <€ \         SV `  4        \        VP                  VP                  VP
                  R VR7      V n        \        P                  ! VP                  4      V n
        \        P                  ! VP                  VP                  R7      V n        \        V4      V n        \        P                  ! VP                  VP                  R7      V n        \#        VRR4      e   \%        V4      V n        R# RV n        R# )Fr  r¬   rA  N)r,   r-   rÔ   r0   r  r  r  r.   r²   r  r´   rz   r¯   r{   rû   r  r  r5   r?  Úadapter_layerrµ   s   &&€rG   r-   Ú*HubertEncoderLayerStableLayerNorm.__init__à  sÂ   ø€ Ü‰ÑÔÜ(Ø×(Ñ(Ø×0Ñ0Ø×,Ñ,ØØô
ˆŒô —z’z &×"7Ñ"7Ó8ˆŒÜŸ,š, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ-¨fÓ5ˆÔÜ "§¢¨V×-?Ñ-?ÀV×EZÑEZÔ [ˆÔä�6Ð-¨tÓ4Ò@Ü!7¸Ó!?ˆDÖà!%ˆDÖrP   c                ór   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          /# r  r  )rÅ   rZ   s   "€rG   rÆ   Ú.HubertEncoderLayerStableLayerNorm.__annotate__ó  s<   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ Ñ+Õ,ñ	rP   c                ó"  € TpV P                  V4      pV P                  ! V3R V/VB w  rV P                  V4      pWA,           pWP                  V P	                  V4      4      ,           pV P
                  e   WP                  V4      ,           pV# r  )r{   r  r´   r  r  rR  r  s   &&&,  rG   rN   Ú)HubertEncoderLayerStableLayerNorm.forwardó  s�   € ð &ˆØŸ™¨Ó6ˆØŸ>š>¨-ÑaÈÐaÐZ`ÑaÑˆØŸ™ ]Ó3ˆØ%Õ5ˆØ%×(9Ñ(9¸$×:OÑ:OÐP]Ó:^Ó(_Õ_ˆà×ÑÒ)Ø)×,>Ñ,>¸}Ó,MÕMˆMàÐrP   )rR  r  r´   r  r  r{   rs   rQ   rY   s   @@rG   rP  rP  ß  s   ù‡ € õ&÷&÷ ò rP   rP  c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚHubertEncoderStableLayerNormi  c                ó¢  <€ \         SV `  4        Wn        \        V4      V n        \
        P                  ! VP                  VP                  R 7      V n	        \
        P                  ! VP                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\!        V4      NK  	  up4      V n        RV n        R# u upi r"  )r,   r-   rE   r   r#  r.   rz   r0   r¯   r{   r²   r  r´   r•   r’   r$  rP  r%  r—   r&  s   && €rG   r-   Ú%HubertEncoderStableLayerNorm.__init__  s™   ø€ Ü‰ÑÔØŒÜ;¸FÓCˆÔÜŸ,š, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z’z &×"7Ñ"7Ó8ˆŒÜ—m’mÜ@EÀf×F^ÑF^Ô@_Ó`Ñ@_¸1Ô.¨vÖ6Ñ@_Ñ`ó
ˆŒð ',ˆÖ#ùò ar(  c                ó0   <€ V ^8„  d   QhRS[ S[,          /# )r    rÀ   )r   r   )rÅ   rZ   s   "€rG   rÆ   Ú)HubertEncoderStableLayerNorm.__annotate__  s    ø€ ÷ #@ñ #@ñ Ñ+Õ,ñ	#@rP   c                óV  € Ve9   VP                  R4      P                  ^^VP                  ^,          4      p^ W( &   \        V P                  VVR7      pV P                  V4      pW,           pV P                  V4      p\        4       ;'       g    \        V 4      pV P                   Fb  p\        P                  ! . 4      pV P                  ;'       d    W€P                  P                  8  p	V	'       d   V'       g   KW  V! V3RV/VB pKd  	  V P                  V4      p\        VR7      # r-  )r0  r1  rë   r
   rE   r#  r´   r   r	   r%  rÂ   r4  r¤   r5  r{   r   r6  s
   &&&,      rG   rN   Ú$HubertEncoderStableLayerNorm.forward  s  € ð Ò%à$2×$<Ñ$<¸RÓ$@×$GÑ$GÈÈ1Èm×NaÑNaÐbcÕNdÓ$eÐ!Ø45ˆMÐ0Ñ1ä2Ø—;‘;Ø'Ø)ô
ˆð #×1Ñ1°-Ó@ÐØ%Õ;ˆØŸ™ ]Ó3ˆä0Ó2×RÐRÔ6LÈTÓ6Rˆà—[”[ˆEä"'§*¢*¨R£.Ðà!Ÿ]™]×ZÐZÐ/BÇ[Á[×EZÑEZÑ/ZˆNß!§[¡[ñ !& mÑ ]ÀNÐ ]ÐV\Ñ ]’ñ !ð Ÿ™¨Ó6ˆä°Ô?Ð?rP   r=  rs   rQ   rY   s   @@rG   rY  rY    s   ù‡ € õ	,÷#@÷ #@ò #@rP   rY  c            	       óÐ   a a€ ] tR tRt oRtRtRtRR.tRtRt	Rt
RtR]].R	]! ]^R
R7      /t]P$                  ! 4       V 3R l4       tV3R lR ltV3R lR ltV3R ltRtVtV ;t# )ÚHubertPreTrainedModeli8  Úhubertr¥   Úaudior  ÚParametrizedConv1dTrM   Ú
attentionsÚencoder)ÚindexÚ
layer_namec                ó¼  <€ \         SV `  V4       \        V\        P                  4      '       EdI   \        4       '       dç   ^ RIp\        VR4      '       dx   \        VR4      '       df   VP                  P                  VP                  VP                  .^ R7      ;_uu_ 4        \        P                  ! VP                  4       RRR4       MyVP                  P                  VP                  ^ R7      ;_uu_ 4        \        P                  ! VP                  4       RRR4       M \        P                  ! VP                  4       VP                  e#   \        P                   ! VP                  4       R# R# \        V\"        4      '       d7   \        VR4      '       d#   \        P$                  ! VP&                  4       R# R# \        V\(        4      '       dZ   \        VR4      '       dF   \        P*                  ! VP,                  RV P.                  P0                  ^,           ,          4       R# R# R#   + '       g   i     Lÿ; i  + '       g   i     EL; i)zInitialize the weightsNr?   r>   r&   Úmasked_spec_embedÚlayer_weightsg      ð?)r,   Ú_init_weightsr  r.   r/   r   r9   r8   r:   r;   r?   r>   ÚinitÚkaiming_normal_r(   rh   Úzeros_ÚHubertModelÚuniform_rj  ÚHubertForSequenceClassificationÚ	constant_rk  rE   r$  )rD   rº   r9   rF   s   && €rG   rl  Ú#HubertPreTrainedModel._init_weightsH  s’  ø€ ô 	‰Ñ˜fÔ%Ü�fœbŸi™i×(Ó(Ü)×+Ò+Û ä˜6 :×.Ò.´7¸6À:×3NÒ3NØ"Ÿ™×:Ñ:¸F¿O¹OÈVÏ_É_Ð;]ÐmnÐ:×oÕoÜ×,Ò,¨V¯]©]Ô;÷ pÐoð #Ÿ™×:Ñ:¸6¿=¹=ÐXYÐ:×ZÕZÜ×,Ò,¨V¯]©]Ô;÷ [ÐZô ×$Ò$ V§]¡]Ô3à�{‰{Ò&Ü—’˜FŸK™KÖ(ñ 'ä˜¤×,Ò,Ü�vÐ2×3Ò3Ü—’˜f×6Ñ6Ö7ñ 4ä˜Ô ?×@Ò@Ü�v˜×/Ò/Ü—’˜v×3Ñ3°S¸D¿K¹K×<YÑ<YÐ\]Õ<]Õ5^Ö_ñ 0ñ A÷ p×oú÷ [×ZÐZús   Â%!H7Ã>!I
È7I	É
I	c                óD   <€ V ^8„  d   QhRS[ P                  S[,          /# )r    Úinput_lengths)rÂ   Ú
LongTensorrÚ   )rÅ   rZ   s   "€rG   rÆ   Ú"HubertPreTrainedModel.__annotate__b  s    ø€ ÷ ñ ¹e×>NÑ>NÑQTÕ>Tñ rP   c                ó˜   € R p\        V P                  P                  V P                  P                  4       F  w  r4V! WV4      pK  	  V# )z8
Computes the output length of the convolutional layers
c                 óN   € \         P                  ! W,
          VR R7      ^,           # )Úfloor)Úrounding_mode)rÂ   Údiv)Úinput_lengthr!   rg   s   &&&rG   Ú_conv_out_lengthÚPHubertPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_lengthg  s    € ô —9’9˜\Õ7¸ÈwÔWÐZ[Õ[Ð[rP   )ÚziprE   rl   rm   )rD   rv  r  r!   rg   s   &&   rG   Ú _get_feat_extract_output_lengthsÚ6HubertPreTrainedModel._get_feat_extract_output_lengthsb  sG   € ò
	\ô
 $' t§{¡{×'>Ñ'>ÀÇÁ×@WÑ@WÖ#XÑˆKÙ,¨]ÈÓPŠMñ $Yð ÐrP   c                ó:   <€ V ^8„  d   QhRS[ RS[P                  /# )r    Úfeature_vector_lengthr¾   )rÚ   rÂ   rw  )rÅ   rZ   s   "€rG   rÆ   rx  q  s"   ø€ ÷ 
ñ 
Éð 
Ñ]b×]mÑ]mñ 
rP   c                óþ  € V P                  VP                  R4      4      P                  \        P                  4      pVP
                  ^ ,          p\        P                  ! WA3VP                  VP                  R7      p^V\        P                  ! VP
                  ^ ,          VP                  R7      V^,
          3&   VP                  R.4      P                  R4      P                  R.4      P                  4       pV# )rJ   )Údtyper3  )r3  r   )r‚  Úsumr2  rÂ   Úlongrë   Úzerosr‡  r3  ÚarangeÚflipÚcumsumrÛ   )rD   r…  r¾   Úoutput_lengthsÚ
batch_sizes   &&&  rG   Ú"_get_feature_vector_attention_maskÚ8HubertPreTrainedModel._get_feature_vector_attention_maskq  sÔ   € Ø×>Ñ>¸~×?QÑ?QÐRTÓ?UÓV×YÑYÔZ_×ZdÑZdÓeˆØ#×)Ñ)¨!Õ,ˆ
äŸšØÐ/°~×7KÑ7KÐTb×TiÑTiô
ˆð uvˆœŸš ^×%9Ñ%9¸!Õ%<À^×EZÑEZÔ[Ð]kÐnoÕ]oÐpÑqØ'×,Ñ,¨b¨TÓ2×9Ñ9¸"Ó=×BÑBÀBÀ4ÓH×MÑMÓOˆØÐrP   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ©r    rE   r   )rÅ   rZ   s   "€rG   rÆ   rx  8  s   ø‡ ‚ áÑò rP   © )rR   rS   rT   rU   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnr  rP  r   rÔ   Ú_can_record_outputsrÂ   Úno_gradrl  r‚  r�  Ú__annotate_func__rV   rW   rX   rY   s   @@rG   ra  ra  8  s•   ù‡ € ð !ÐØ$€OØÐØ-Ð/CÐDÐØ&*Ð#ØÐØ€NØÐàÐ,Ð.OÐPØ‘n _¸AÈ)ÔTðÐð
 ‡]‚]ƒ_ô`ó ð`÷2ð ÷
ð 
÷s … rP   ra  c                ó¸   € V ^8„  d   QhR\         \        \        3,          R\        R\        R\        P                  R,          R\        R\
        P                  /# )r    rë   Ú	mask_probÚmask_lengthr¾   NÚ	min_masksrç   )rè   rÚ   rÄ   rÂ   rw  ÚnpÚndarray)rÅ   s   "rG   rÆ   rÆ   ~  sc   € ÷ tñ tÜ””c��?ðtäðtô ðtô ×$Ñ$ tÕ+ð	tô
 ðtô ‡Z�ZñtrP   c           	     ó^  aaaaa€ V w  poS^8  d   \        R4      hSS8”  d   \        RS RS R24      h\        P                  P                  ^4      P	                  4       oVVVVV3R lpVe.   VP                  4       P                  R	4      P                  4       M\        V4       Uu. uF  pSNK  	  upp\        P                  ! VS3\        R7      p	. p
V! S4      pV^ 8X  d   V	# V FÌ  pV! V4      p\        P                  P                  \        P                  ! VS^,
          ,
          4      VRR7      p\        V4      ^ 8X  d   S^,
          pM	V^ ,          p\        P                  ! V\        P                  ! W½,
          \        P                   R7      V,          .4      pV
P#                  V4       KÎ  	  \        P$                  ! V
4      p
\        P&                  ! V
R
,          W[S34      p
V
P)                  W[S,          4      p
\        P                  ! S4      R,          p\        P&                  ! VW[S34      P)                  W[S,          4      pV
V,           p
V
P+                  4       S^,
          8”  d   S^,
          WªS^,
          8„  &   \        P,                  ! Wš^R	4       V	# u upi )a2  
Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on
CPU as part of the preprocessing during training.

Args:
    shape: The shape for which to compute masks. This should be of a tuple of size 2 where
           the first element is the batch size and the second element is the length of the axis to span.
    mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
                independently generated mask spans of length `mask_length` is computed by
                `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
                actual percentage will be smaller.
    mask_length: size of the mask
    min_masks: minimum number of masked spans
    attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
                    each batch dimension.
z&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: Ú`c                óì   <€ \        SV ,          S,          S,           4      p\        VS4      pVS,          S8”  d
   SS,          pV S^,
          ,
          V8  d   \        V S^,
          ,
          ^ 4      pV# )z;Given input length, compute how many spans should be masked)rÚ   Úmax)r~  Únum_masked_spanÚepsilonr¢  r¡  r£  Úsequence_lengths   & €€€€€rG   Úcompute_num_masked_spanÚ6_compute_mask_indices.<locals>.compute_num_masked_span¤  so   ø€ ä˜i¨,Õ6¸ÕDÀwÕNÓOˆÜ˜o¨yÓ9ˆð ˜[Õ(¨?Ô:Ø-°Õ<ˆOð ˜;¨�?Õ+¨oÔ=Ü! ,°+Àµ/Õ"BÀAÓFˆOàÐrP   ©r‡  F)Úreplacer   )r`   r`   N)NNr`   )r”   r¤  Úrandomr4  ÚitemÚdetachrˆ  Útolistr’   rŠ  rÛ   Úchoicer‹  ÚlenÚconcatenateÚonesÚint32ÚappendÚarrayÚbroadcast_torï   r©  Úput_along_axis)rë   r¡  r¢  r¾   r£  r�  r­  r  rv  Úspec_aug_maskÚspec_aug_mask_idxsÚmax_num_masked_spanr~  rª  Úspec_aug_mask_idxÚdummy_mask_idxÚoffsetsr«  r¬  s   &ff&f            @@rG   Ú_compute_mask_indicesrÄ  ~  sŽ  ü€ ð0 #(Ñ€J�à�Q„ÜÐAÓBÐBà�_Ô$ÜØ]Ð^iÐ]jØ& Ð&7°qð:ó
ð 	
ô �i‰i�n‰n˜QÓ×$Ñ$Ó&€G÷ñ ð$ Ò%ð 	×ÑÓ×#Ñ# BÓ'×.Ñ.Ô0ä',¨ZÔ'8Ó9Ñ'8 !‹oÑ'8Ñ9ð ô —H’H˜j¨/Ð:Ä$ÔG€MØÐá1°/ÓBÐà˜aÔØÐã%ˆá1°,Ó?ˆô ŸI™I×,Ñ,Ü�IŠI�l k°A¥oÕ6Ó7¸ÐRWð -ó 
Ðô Ð Ó! QÔ&ð -¨qÕ0‰Nà.¨qÕ1ˆNäŸNšNØ¤§¢Ð(;Õ(MÔUW×U]ÑU]Ô ^ÐaoÕ oÐpó
Ðð 	×!Ñ!Ð"3Ö4ñ/ &ô2 ŸšÐ"4Ó5Ðô ŸšØ˜:Õ&¨È+Ð(VóÐð ,×3Ñ3°JÐVaÕ@aÓbÐô �iŠi˜Ó$ ]Õ3€GÜ�oŠo˜g¨
ÈÐ'UÓV×^Ñ^Ø¨+Õ5ó€Gð ,¨gÕ5Ðð ×ÑÓ /°AÕ"5Ô5ØGVÐYZÕGZÐ°À!Õ0CÑCÑDô ×Ò�m¸¸BÔ?àÐùòw :s   Â+J*c                   ó–   a a€ ] tR tRt oV3R lV 3R lltRV3R lR llt]]! RR7      ]RV3R lR	 ll4       4       4       t	R
t
VtV ;t# )rp  iõ  c                ó    <€ V ^8„  d   QhRS[ /# r“  r   )rÅ   rZ   s   "€rG   rÆ   ÚHubertModel.__annotate__÷  s   ø€ ÷ ñ ™|ñ rP   c                óÊ  <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        VP                  R 8”  g   VP                  R 8”  dH   \        P                  ! \        P                  ! VP                  4      P                  4       4      V n        VP                   '       d   \#        V4      V n        M\'        V4      V n        V P)                  4        R# )rê   N)r,   r-   rE   r�   Úfeature_extractorrª   Úfeature_projectionÚmask_time_probÚmask_feature_probr.   Ú	ParameterrÂ   rÃ   r0   rq  rj  Údo_stable_layer_normrY  rf  r   Ú	post_initrµ   s   &&€rG   r-   ÚHubertModel.__init__÷  s¥   ø€ Ü‰Ñ˜Ô ØŒÜ!5°fÓ!=ˆÔÜ"9¸&Ó"AˆÔð × Ñ  3Ô&¨&×*BÑ*BÀSÔ*HÜ%'§\¢\´%·,²,¸v×?QÑ?QÓ2R×2[Ñ2[Ó2]Ó%^ˆDÔ"à×&×&Ð&Ü7¸Ó?ˆD�Lä(¨Ó0ˆDŒLð 	�‰ÖrP   c                ó„   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          /# )r    rM   Úmask_time_indicesNr¾   )rÂ   rK  rw  )rÅ   rZ   s   "€rG   rÆ   rÇ  	  sE   ø€ ÷ ,ñ ,á×(Ñ(ð,ñ !×,Ñ,¨tÕ3ð,ñ ×(Ñ(¨4Õ/ñ	,rP   c                óò  € \        V P                  RR4      '       g   V# VP                  4       w  rEpVe)   V P                  P	                  VP
                  4      W&   MÑV P                  P                  ^ 8”  d·   V P                  '       d¥   \        WE3V P                  P                  V P                  P                  VV P                  P                  R7      p\        P                  ! W!P                  \        P                  R7      pV P                  P	                  VP
                  4      W&   V P                  P                  ^ 8”  d­   V P                  '       d›   \        WF3V P                  P                  V P                  P                   V P                  P"                  R7      p\        P                  ! WqP                  \        P                  R7      pVR,          P%                  RVR4      p^ W&   V# )zŠ
Masks extracted features along time axis and/or along feature axis according to
[SpecAugment](https://huggingface.co/papers/1904.08779).
Úapply_spec_augmentT)r¡  r¢  r¾   r£  )r3  r‡  )r¡  r¢  r£  r£   r   )r5   rE   rË   rj  r2  r‡  rË  r¤   rÄ  Úmask_time_lengthÚmask_time_min_masksrÂ   r*  r3  rÛ   rÌ  Úmask_feature_lengthÚmask_feature_min_masksÚexpand)rD   rM   rÒ  r¾   r�  r¬  r0   Úmask_feature_indicess   &&&&    rG   Ú_mask_hidden_statesÚHubertModel._mask_hidden_states	  sŠ  € ô �t—{‘{Ð$8¸$×?Ò?Ø Ð ð 4A×3EÑ3EÓ3GÑ0ˆ
 [àÒ(à/3×/EÑ/E×/HÑ/HÈ×I\ÑI\Ó/]ˆMÒ,Ø�[‰[×'Ñ'¨!Ô+°··°Ü 5ØÐ-ØŸ+™+×4Ñ4Ø ŸK™K×8Ñ8Ø-ØŸ+™+×9Ñ9ô!Ðô !&§¢Ð->×G[ÑG[Ôch×cmÑcmÔ nÐØ/3×/EÑ/E×/HÑ/HÈ×I\ÑI\Ó/]ˆMÑ,à�;‰;×(Ñ(¨1Ô,°··°ä#8ØÐ)ØŸ+™+×7Ñ7Ø ŸK™K×;Ñ;ØŸ+™+×<Ñ<ô	$Ð ô $)§<¢<Ð0D×MaÑMaÔin×isÑisÔ#tÐ Ø#7¸Õ#@×#GÑ#GÈÈOÐ]_Ó#`Ð Ø23ˆMÑ/àÐrP   F)Útie_last_hidden_statesc                óä   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          RS[S[,          /# )	r    r¥   Nr¾   rÒ  ræ   Úoutput_hidden_statesÚreturn_dictrç   )rÂ   rÃ   rK  rÛ   rè   r   )rÅ   rZ   s   "€rG   rÆ   rÇ  :  sˆ   ø€ ÷ E
ñ E
á—l‘l TÕ)ðE
ñ Ÿ™ tÕ+ðE
ñ !×,Ñ,¨tÕ3ð	E
ñ
   $�;ðE
ñ # T�kðE
ñ ˜D•[ðE
ñ 
‘Õ	 ñE
rP   c                ó8  € Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pV P	                  V4      pVP                  ^^4      pVe$   V P                  VP                  ^,          V4      pV P                  V4      p	V P                  W“R7      p	V P                  V	VVVVR7      p
V
^ ,          p	V'       g   V	3V
R,          ,           # \        V	V
P                  V
P                  R7      # )a©  
mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
    masked extracted features in *config.proj_codevector_dim* space.

Example:

```python
>>> from transformers import AutoProcessor, HubertModel
>>> from datasets import load_dataset

>>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft")
>>> model = HubertModel.from_pretrained("facebook/hubert-large-ls960-ft")


>>> def map_to_array(example):
...     example["speech"] = example["audio"]["array"]
...     return example


>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
>>> ds = ds.map(map_to_array)

>>> input_values = processor(ds["speech"][0], return_tensors="pt").input_values  # Batch size 1
>>> hidden_states = model(input_values).last_hidden_state
```)rÒ  )r¾   ræ   rß  rà  :rJ   NN)r/  rM   re  )rE   ræ   rß  rà  rÉ  rK   r�  rë   rÊ  rÛ  rf  r   rM   re  )rD   r¥   r¾   rÒ  ræ   rß  rà  rÀ   Úextract_featuresrM   Úencoder_outputss   &&&&&&&,   rG   rN   ÚHubertModel.forward7  s'  € ðN 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà×1Ñ1°,Ó?ÐØ+×5Ñ5°a¸Ó;ÐàÒ%à!×DÑDÐEU×E[ÑE[Ð\]ÕE^Ð`nÓoˆNà×/Ñ/Ð0@ÓAˆØ×0Ñ0°Ð0ÓdˆàŸ,™,ØØ)Ø/Ø!5Ø#ð 'ó 
ˆð (¨Õ*ˆçØ!Ð# o°bÕ&9Õ9Ð9äØ+Ø)×7Ñ7Ø&×1Ñ1ô
ð 	
rP   )rE   rf  rÉ  rÊ  rj  ©NN)NNNNN)rR   rS   rT   rU   r-   rÛ  r   r   r   rN   rV   rW   rX   rY   s   @@rG   rp  rp  õ  sQ   ù‡ € ÷ó ÷$,ò ,ð\  Ù¨EÔ2Ø÷E
ñ E
ó ó 3ó  ÷E
ð E
rP   rp  zn
    Hubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).
    )Úcustom_introc                   ó~   a a€ ] tR tRt oR
V3R lV 3R llltR tR tR t]]	RV3R lR ll4       4       t
R	tVtV ;t# )ÚHubertForCTCi‚  c                ó.   <€ V ^8„  d   QhRS[ R,          /# )r    Útarget_langN)r  )rÅ   rZ   s   "€rG   rÆ   ÚHubertForCTC.__annotate__ˆ  s   ø€ ÷ ñ ©C°$­Jñ rP   c                óÌ  <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  4      V n        W n        VP                  f   \        RV P                   R24      h\        VR4      '       d   VP                  '       d   VP                  MVP                  p\        P                   ! W1P                  4      V n        V P%                  4        R# )a  
target_lang (`str`, *optional*):
    Language id of adapter weights. Adapter weights are stored in the format adapter.<lang>.safetensors or
    adapter.<lang>.bin. Only relevant when using an instance of [`HubertForCTC`] with adapters. Uses 'eng' by
    default.
NzYou are trying to instantiate zõ with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `HubertForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.Úadd_adapter)r,   r-   rp  rb  r.   r²   Úfinal_dropoutr´   rê  Ú
vocab_sizer”   rF   r8   rí  Úoutput_hidden_sizer0   r°   Úlm_headrÏ  )rD   rE   rê  rð  rF   s   &&& €rG   r-   ÚHubertForCTC.__init__ˆ  sÁ   ø€ ô 	‰Ñ˜Ô ä! &Ó)ˆŒÜ—z’z &×"6Ñ"6Ó7ˆŒà&Ôà×ÑÒ$ÜØ0°·±Ð0@ð AHð Hóð ô *1°¸×)GÒ)GÈF×L^×L^ÐL^ˆF×%Ò%Ðdj×dvÑdvð 	ô —y’yÐ!3×5FÑ5FÓGˆŒð 	�‰ÖrP   c                óZ  € \        4       \        P                  ! R4      8X  d   R# V P                  pVe)   \	        V P
                  RR4      f   \        RV R24      hVf1   \	        V P
                  RR4      e   \        P                  R4       R# Ve   V P                  VRR7       R# R# )	a  
This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when
passing `target_lang=...` to `from_pretrained(...)`.

This method is **not** supposed to be called by the user and is prone to be changed in the future.
ÚmetaNrA  zCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T)Ú
force_load)
r   rÂ   r3  rê  r5   rE   r”   ÚloggerÚinfoÚload_adapter)rD   rÀ   rê  s   &, rG   Útie_weightsÚHubertForCTC.tie_weights¥  sœ   € ô 6Ó7¼5¿<º<ÈÓ;OÔOÙð ×&Ñ&ˆàÒ"¤w¨t¯{©{Ð<NÐPTÓ'UÒ']ÜÐ:¸;¸-ÐGtÐuÓvÐvØÒ ¤W¨T¯[©[Ð:LÈdÓ%SÒ%_Ü�K‰KÐCÖDØÒ$Ø×Ñ˜k°dÐÖ;ñ %rP   c                óN   € V P                   P                  P                  4        R# ©z�
Calling this function will disable the gradient computation for the feature encoder so that its parameter will
not be updated during training.
N©rb  rÉ  r    ©rD   s   &rG   Úfreeze_feature_encoderÚ#HubertForCTC.freeze_feature_encoder½  ó   € ð
 	�‰×%Ñ%×8Ñ8Ö:rP   c                óV   € V P                   P                  4        F
  pRVn        K  	  R# ©zº
Calling this function will disable the gradient computation for the base model so that its parameters will not
be updated during training. Only the classification head will be updated.
FN©rb  rœ   r�   rž   s   & rG   Úfreeze_base_modelÚHubertForCTC.freeze_base_modelÄ  ó#   € ð
 —[‘[×+Ñ+Ö-ˆEØ"'ˆEÖó .rP   c                ó¾   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[,          /# ©r    r¥   Nr¾   ÚlabelsrÀ   rç   )rÂ   rÃ   r   r   rè   r   )rÅ   rZ   s   "€rG   rÆ   rë  Î  sc   ø€ ÷ :
ñ :
á—l‘l TÕ)ð:
ñ Ÿ™ tÕ+ð:
ñ —‘˜tÕ#ð	:
ñ
 Ñ+Õ,ð:
ñ 
‘Õ	ñ:
rP   c                óx  € VeL   VP                  4       V P                  P                  8¼  d#   \        RV P                  P                   24      hV P                  ! V3RV/VB pVP
                  pV P                  V4      pV P                  V4      pRpVEen   Ve   TM%\        P                  ! V\        P                  R7      pV P                  VP                  R
4      4      P                  \        P                  4      p	V^ 8¬  p
V
P                  R
4      pVP                  V
4      p\        P                   P#                  VR
\        P$                  R7      P'                  ^ ^4      p\        P(                  P*                  P-                  RR7      ;_uu_ 4        \        P                   P/                  VVV	VV P                  P0                  V P                  P2                  V P                  P4                  R7      pRRR4       \7        W‡VP8                  VP:                  R	7      #   + '       g   i     L2; i)a°  
labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
    Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
    the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
    All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
    config.vocab_size - 1]`.
Nz$Label values must be <= vocab_size: r¾   r¯  )r*   r‡  F)Úenabled)ÚblankÚ	reductionÚzero_infinity©ÚlossÚlogitsrM   re  r   )r©  rE   rï  r”   rb  r/  r´   rñ  rÂ   Ú	ones_liker‰  r‚  rˆ  r2  Úmasked_selectr.   rÍ   Úlog_softmaxÚfloat32rK   ÚbackendsÚcudnnÚflagsÚctc_lossÚpad_token_idÚctc_loss_reductionÚctc_zero_infinityr   rM   re  )rD   r¥   r¾   r
  rÀ   ÚoutputsrM   r  r  rv  Úlabels_maskÚtarget_lengthsÚflattened_targetsÚ	log_probss   &&&&,         rG   rN   ÚHubertForCTC.forwardÌ  sß  € ð  Ò &§*¡*£,°$·+±+×2HÑ2HÔ"HÜÐCÀDÇKÁK×DZÑDZÐC[Ð\Ó]Ð]à—+’+Øñ
à)ð
ð ñ
ˆð  ×1Ñ1ˆØŸ™ ]Ó3ˆà—‘˜mÓ,ˆàˆØÓð #1Ò"<‘Ä%Ç/Â/ÐR^Ôfk×fpÑfpÔBqð ð !×AÑAÀ.×BTÑBTÐUWÓBXÓY×\Ñ\Ô]b×]gÑ]gÓhˆMð ! A™+ˆKØ(Ÿ_™_¨RÓ0ˆNØ &× 4Ñ 4°[Ó AÐô Ÿ™×1Ñ1°&¸bÌÏÉÐ1ÓV×`Ñ`ÐabÐdeÓfˆIä—‘×%Ñ%×+Ñ+°EÐ+×:Õ:Ü—}‘}×-Ñ-ØØ%Ø!Ø"ØŸ+™+×2Ñ2Ø"Ÿk™k×<Ñ<Ø"&§+¡+×"?Ñ"?ð .ó �÷ ;ô Ø°G×4IÑ4IÐV]×VhÑVhô
ð 	
÷ ;×:ús   ÆA#H)È)H9	)r´   rb  rñ  rê  rs   rå  )rR   rS   rT   rU   r-   rù  rÿ  r  r   r   rN   rV   rW   rX   rY   s   @@rG   rè  rè  ‚  sD   ù‡ € ÷õ ò:<ò0;ò(ð Ø÷:
ñ :
ó ó ÷:
ð :
rP   rè  z•
    Hubert Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                   ór   a a€ ] tR tRt oV 3R ltR tR t]]]	RV3R lR ll4       4       4       t
RtVtV ;t# )	rr  i  c                ó:  <€ \         SV `  V4       \        VR 4      '       d   VP                  '       d   \	        R4      h\        V4      V n        VP                  ^,           pVP                  '       d7   \        P                  ! \        P                  ! V4      V,          4      V n        \        P                  ! VP                  VP                   4      V n        \        P                  ! VP                   VP$                  4      V n        V P)                  4        R# )rí  z]Sequence classification does not support the use of Hubert adapters (config.add_adapter=True)N)r,   r-   r8   rí  r”   rp  rb  r$  Úuse_weighted_layer_sumr.   rÍ  rÂ   r¸  rk  r°   r0   Úclassifier_proj_sizeÚ	projectorÚ
num_labelsÚ
classifierrÏ  )rD   rE   Ú
num_layersrF   s   && €rG   r-   Ú(HubertForSequenceClassification.__init__  sÉ   ø€ Ü‰Ñ˜Ô ä�6˜=×)Ò)¨f×.@×.@Ð.@ÜØoóð ô " &Ó)ˆŒØ×-Ñ-°Õ1ˆ
Ø×(×(Ð(Ü!#§¢¬e¯jªj¸Ó.DÀzÕ.QÓ!RˆDÔÜŸš 6×#5Ñ#5°v×7RÑ7RÓSˆŒÜŸ)š) F×$?Ñ$?À×ARÑARÓSˆŒð 	�‰ÖrP   c                óN   € V P                   P                  P                  4        R# rü  rý  rþ  s   &rG   rÿ  Ú6HubertForSequenceClassification.freeze_feature_encoder#  r  rP   c                óV   € V P                   P                  4        F
  pRVn        K  	  R# r  r  rž   s   & rG   r  Ú1HubertForSequenceClassification.freeze_base_model*  r  rP   c                ó¾   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[,          RS[S[,          /# r	  )rÂ   rÃ   r   r   rè   r   )rÅ   rZ   s   "€rG   rÆ   Ú,HubertForSequenceClassification.__annotate__5  sd   ø€ ÷ 9
ñ 9
á—l‘l TÕ)ð9
ñ Ÿ™ tÕ+ð9
ñ —‘˜tÕ#ð	9
ñ
 Ñ+Õ,ð9
ñ 
Ñ)Õ	)ñ9
rP   c                ó(  € V P                   P                  '       d   RVR&   V P                  ! V3RV/VB pV P                   P                  '       dz   VP                  p\        P
                  ! V^R7      p\        P                  P                  V P                  RR7      pWgP                  R^^4      ,          P                  ^R7      pMVP                  pV P                  V4      pVf   VP                  ^R7      pM”V P                  VP                   ^,          V4      p	V	P#                  R4      P%                  ^^VP                   ^,          4      p
RWj( &   VP                  ^R7      V	P                  ^R7      P                  R^4      ,          pV P'                  V4      pRpVeG   \)        4       pV! VP                  RV P                   P*                  4      VP                  R4      4      p\-        VVVP                  VP.                  R7      # )	a²  
input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
    Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
    into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
    (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
    To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
    into a tensor of type `torch.FloatTensor`. See [`HubertProcessor.__call__`] for details.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Trß  r¾   rÈ   Nrê   r  r   )rE   r&  rb  rM   rÂ   Ústackr.   rÍ   rÎ   rk  rì   rˆ  r/  r(  Úmeanr�  rë   r0  r1  r*  r   r)  r   re  )rD   r¥   r¾   r
  rÀ   r  rM   Únorm_weightsÚpooled_outputÚpadding_maskÚexpand_padding_maskr  r  Úloss_fcts   &&&&,         rG   rN   Ú'HubertForSequenceClassification.forward2  sá  € ð, �;‰;×-×-Ð-Ø-1ˆFÐ)Ñ*à—+’+Øñ
à)ð
ð ñ
ˆð �;‰;×-×-Ð-Ø#×1Ñ1ˆMÜ!ŸKšK¨¸1Ô=ˆMÜŸ=™=×0Ñ0°×1CÑ1CÈÐ0ÓLˆLØ*×->Ñ->¸rÀ1ÀaÓ-HÕH×MÑMÐRSÐMÓT‰Mà#×5Ñ5ˆMàŸ™ }Ó5ˆØÒ!Ø)×.Ñ.°1Ð.Ó5‰Mà×BÑBÀ=×CVÑCVÐWXÕCYÐ[iÓjˆLØ".×"8Ñ"8¸Ó"<×"CÑ"CÀAÀqÈ-×J]ÑJ]Ð^_ÕJ`Ó"aÐØ25ˆMÐ.Ñ/Ø)×-Ñ-°!Ð-Ó4°|×7GÑ7GÈAÐ7GÓ7N×7SÑ7SÐTVÐXYÓ7ZÕZˆMà—‘ Ó/ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯K©K×,BÑ,BÓCÀVÇ[Á[ÐQSÃ_ÓUˆDä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rP   )r*  rb  rk  r(  rå  )rR   rS   rT   rU   r-   rÿ  r  r   r   r   rN   rV   rW   rX   rY   s   @@rG   rr  rr    sB   ù‡ € õò";ò(ð Ø Ø÷9
ñ 9
ó ó !ó ÷9
ð 9
rP   rr  )Nrê   )Nr   )rè  rr  rp  ra  )HÚcollections.abcr   Únumpyr¤  rÂ   Útorch.nnr.   r   Ú r   rm  Úactivationsr   Úbackbone_utilsr   Úintegrations.deepspeedr   Úintegrations.fsdpr	   Úmasking_utilsr
   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   r   Úprocessing_utilsr   r7   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_hubertr   Ú
get_loggerrR   rö  rÁ   r   rA   rd   rw   rƒ   r�   rª   rÒ   rÔ   rû   r  r   r?  rP  rY  ra  rÄ  rp  rè  rr  Ú__all__r”  rP   rG   Ú<module>rO     sã  ðõ* %ã Û Ý Ý %å &Ý !Ý 9Ý @Ý 7Ý 6Ý BÝ 9ß YÑ Yß rÑ rÝ &ß @Ñ @ß Iß EÝ .ð 
×	Ò	˜HÓ	%€ô/ B§I¡Iô /ôd˜Ÿ™ô ôÐ!;ô ô*Ð9ô ô6Ð9ô ô0#˜2Ÿ9™9ô #ôL˜bŸi™iô ÷$%ô8R)�b—i‘iô R)ôj˜Ÿ	™	ô ô0Ð3ô ôD+@�B—I‘Iô +@ô\˜RŸY™Yô ô2$Ð(Bô $ôN/@ 2§9¡9ô /@ðd ôB˜Oó Bó ðB÷Jtðn ôI
Ð'ó I
ó ðI
ñX ðôô
A
Ð(ó A
óð
A
ñH ðôô]
Ð&;ó ]
óð]
ò@ f‚rP   