+
    œ°³j·å  ã                   óD  € R t ^ RIt^ RIt^ RIHt ^ RIt^ RIHt ^ RIHtH	t	H
t
 ^RIHt ^RIHt ^RIHt ^RIHt ^R	IHtHtHtHtHtHt ^R
IHt ^RIHt ^RIH t H!t! ^RI"H#t# ]!PH                  ! ]%4      t&. R;Ot'] ! RR7      ] ! R R]4      4       4       t( ! R R]PR                  4      t* ! R R]PR                  4      t+ ! R R]PR                  4      t, ! R R]PR                  4      t- ! R R]PR                  4      t. ! R R]PR                  4      t/ ! R R]PR                  4      t0 ! R  R!]PR                  4      t1 ! R" R#]4      t2 ! R$ R%]PR                  4      t3 ! R& R']PR                  4      t4 ! R( R)]PR                  4      t5 ! R* R+]PR                  4      t6 ! R, R-]PR                  4      t7]  ! R. R/]4      4       t8]  ! R0 R1]84      4       t9] ! R2R7       ! R3 R4]84      4       t:]  ! R5 R6]84      4       t;]  ! R7 R8]84      4       t<]  ! R9 R:]84      4       t=. R<Ot>R# )=zPyTorch CANINE model.N)Ú	dataclass)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚModelOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úapply_chunking_to_forward)Úauto_docstringÚlogging©ÚCanineConfiga  
    Output type of [`CanineModel`]. Based on [`~modeling_outputs.BaseModelOutputWithPooling`], but with slightly
    different `hidden_states` and `attentions`, as these also include the hidden states and attentions of the shallow
    Transformer encoders.
    )Úcustom_introc                   ó@   a € ] tR t^1t o RtRtRtRtRtV 3R lt	Rt
V tR# )ÚCanineModelOutputWithPoolingaW  
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
    Sequence of hidden-states at the output of the last layer of the model (i.e. the output of the final
    shallow Transformer encoder).
pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
    Hidden-state of the first token of the sequence (classification token) at the last layer of the deep
    Transformer encoder, further processed by a Linear layer and a Tanh activation function. The Linear layer
    weights are trained from the next sentence prediction (classification) objective during pretraining.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
    Tuple of `torch.FloatTensor` (one for the input to each encoder + one for the output of each layer of each
    encoder) of shape `(batch_size, sequence_length, hidden_size)` and `(batch_size, sequence_length //
    config.downsampling_rate, hidden_size)`. Hidden-states of the model at the output of each layer plus the
    initial input to each Transformer encoder. The hidden states of the shallow encoders have length
    `sequence_length`, but the hidden states of the deep encoder have length `sequence_length` //
    `config.downsampling_rate`.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of the 3 Transformer encoders of shape `(batch_size,
    num_heads, sequence_length, sequence_length)` and `(batch_size, num_heads, sequence_length //
    config.downsampling_rate, sequence_length // config.downsampling_rate)`. Attentions weights after the
    attention softmax, used to compute the weighted average in the self-attention heads.
Nc                óò   <€ V ^8„  d   Qh/ S[ P                  R,          ;R&   S[ P                  R,          ;R&   S[S[ P                  ,          R,          ;R&   S[S[ P                  ,          R,          ;R&   # )é   NÚlast_hidden_stateÚpooler_outputÚhidden_statesÚ
attentions)ÚtorchÚFloatTensorÚtuple)ÚformatÚ__classdict__s   "€Úk/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/canine/modeling_canine.pyÚ__annotate__Ú)CanineModelOutputWithPooling.__annotate__1   su   ø‡ ‚ ñ> ×(Ñ(¨4Õ/Ñ6ñ? ñ@ ×$Ñ$ tÕ+Ñ2ñA ñB ™×*Ñ*Õ+¨dÕ2Ñ9ñC ñD ‘e×'Ñ'Õ(¨4Õ/Ñ6òE ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r   r   r   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r$   s   @r%   r   r   1   s)   ø‡ € ñð, 37ÐØ.2€MØ59€MØ26€J÷E ƒ r(   r   c                   óp   a a€ ] tR t^Vt oRtV 3R ltV3R lR ltV3R lR ltR
V3R lR lltR	t	Vt
V ;t# )ÚCanineEmbeddingsz<Construct the character, position and token_type embeddings.c           	     ó  <€ \         SV `  4        Wn        VP                  VP                  ,          p\        VP                  4       F3  pR V 2p\        W\        P                  ! VP                  V4      4       K5  	  \        P                  ! VP                  VP                  4      V n
        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                   4      V n        \        P$                  ! \&        P(                  ! VP*                  4      P-                  R4      RR7      V n        R# )ÚHashBucketCodepointEmbedder_©ÚepsF)Ú
persistentN©é   éÿÿÿÿ)ÚsuperÚ__init__ÚconfigÚhidden_sizeÚnum_hash_functionsÚrangeÚsetattrr   Ú	EmbeddingÚnum_hash_bucketsÚchar_position_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚBufferr    ÚarangeÚmax_position_embeddingsÚexpandÚposition_ids)Úselfr>   Úshard_embedding_sizeÚiÚnameÚ	__class__s   &&   €r%   r=   ÚCanineEmbeddings.__init__Y   s  ø€ Ü‰ÑÔàŒð  &×1Ñ1°V×5NÑ5NÕNÐÜ�v×0Ñ0Ö1ˆAØ1°!°Ð5ˆDÜ�D¤§¢¨V×-DÑ-DÐFZÓ [Ö\ñ 2ô )+¯ª°V×5LÑ5LÈf×N`ÑN`Ó(aˆÔ%Ü%'§\¢\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒô ŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÔuˆÖr(   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   Ú
num_hashesÚnum_buckets©Úint)r#   r$   s   "€r%   r&   ÚCanineEmbeddings.__annotate__l   s   ø€ ÷ ñ ¹#ð ÉCñ r(   c                óÜ   € V\        \        4      8”  d   \        R\        \        4       24      h\        RV p. pV F+  pV^,           V,          V,          pVP                  V4       K-  	  V# )aW  
Converts ids to hash bucket ids via multiple hashing.

Args:
    input_ids: The codepoints or other IDs to be hashed.
    num_hashes: The number of hash functions to use.
    num_buckets: The number of hash buckets (i.e. embeddings in each table).

Returns:
    A list of tensors, each of which is the hash bucket IDs from one hash function.
z`num_hashes` must be <= N)ÚlenÚ_PRIMESÚ
ValueErrorÚappend)rR   Ú	input_idsrY   rZ   ÚprimesÚresult_tensorsÚprimeÚhasheds   &&&&    r%   Ú_hash_bucket_tensorsÚ%CanineEmbeddings._hash_bucket_tensorsl   sk   € ð œœG›Ô$ÜÐ7¼¼G»°~ÐFÓGÐGä˜˜*Ð%ˆàˆÛˆEØ  1•}¨Õ-°Õ<ˆFØ×!Ñ! &Ö)ñ ð Ðr(   c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[ /# )r   Úembedding_sizerY   rZ   r[   )r#   r$   s   "€r%   r&   r]   ƒ   s#   ø€ ÷ 3ñ 3¹Sð 3Écð 3Ñ`cñ 3r(   c                ó  € W#,          ^ 8w  d   \        RV RV R24      hV P                  WVR7      p. p\        V4       F,  w  rxRV 2p	\        W	4      ! V4      p
VP	                  V
4       K.  	  \
        P                  ! VRR7      # )zDConverts IDs (e.g. codepoints) into embeddings via multiple hashing.zExpected `embedding_size` (z) % `num_hashes` (z) == 0)rY   rZ   r5   ©Údimr;   )ra   rh   Ú	enumerateÚgetattrrb   r    Úcat)rR   rc   rk   rY   rZ   Úhash_bucket_tensorsÚembedding_shardsrT   Úhash_bucket_idsrU   Úshard_embeddingss   &&&&&      r%   Ú_embed_hash_bucketsÚ$CanineEmbeddings._embed_hash_bucketsƒ   s�   € àÕ&¨!Ô+ÜÐ:¸>Ð:JÐJ\Ð]gÐ\hÐhnÐoÓpÐpà"×7Ñ7¸	ÐfqÐ7ÓrÐØÐÜ"+Ð,?Ö"@ÑˆAØ1°!°Ð5ˆDÜ& tÔ2°?ÓCÐØ×#Ñ#Ð$4Ö5ñ #Aô
 �yŠyÐ)¨rÔ2Ð2r(   c          
      óÔ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  /# )r   rc   NÚtoken_type_idsrQ   Úinputs_embedsÚreturn)r    Ú
LongTensorr!   )r#   r$   s   "€r%   r&   r]   ‘   so   ø€ ÷ !ñ !á×#Ñ# dÕ*ð!ñ ×(Ñ(¨4Õ/ð!ñ ×&Ñ&¨Õ-ð	!ñ
 ×(Ñ(¨4Õ/ð!ñ 
×	Ñ	ñ!r(   c                ój  € Ve   VP                  4       pMVP                  4       R R pV^,          pVf   V P                  RR V13,          pVf<   \        P                  ! V\        P                  V P                  P
                  R7      pVfP   V P                  WP                  P                  V P                  P                  V P                  P                  4      pV P                  V4      pWG,           pV P                  V4      p	W‰,          pV P                  V4      pV P                  V4      pV# )NºNNN©ÚdtypeÚdevicer;   )ÚsizerQ   r    ÚzerosÚlongr�   rv   r>   r?   r@   rD   rG   rE   rH   rL   )
rR   rc   ry   rQ   rz   Úinput_shapeÚ
seq_lengthrG   Ú
embeddingsÚposition_embeddingss
   &&&&&     r%   ÚforwardÚCanineEmbeddings.forward‘   s  € ð Ò Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  •^ˆ
àÒØ×,Ñ,¨Q°°°¨^Õ<ˆLàÒ!Ü"Ÿ[š[¨¼E¿J¹JÈt×O`ÑO`×OgÑOgÔhˆNàÒ Ø ×4Ñ4ØŸ;™;×2Ñ2°D·K±K×4RÑ4RÐTX×T_ÑT_×TpÑTpóˆMð !%× :Ñ :¸>Ó JÐØ"Õ:ˆ
à"×;Ñ;¸LÓIÐØÕ)ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr(   )rH   rE   r>   rL   rQ   rG   )NNNN)r*   r+   r,   r-   r.   r=   rh   rv   r‰   r0   r1   Ú__classcell__©rV   r$   s   @@r%   r3   r3   V   s1   ù‡ € ÙFõv÷&ð ÷.3ð 3÷!÷ !ò !r(   r3   c                   óH   a a€ ] tR t^µt oRtV 3R ltV3R lR ltRtVtV ;t	# )ÚCharactersToMoleculeszeConvert character sequence to initial molecule sequence (i.e. downsample) using strided convolutions.c                óL  <€ \         SV `  4        \        P                  ! VP                  VP                  VP
                  VP
                  R 7      V n        \        VP                  ,          V n	        \        P                  ! VP                  VP                  R7      V n
        R# )©Úin_channelsÚout_channelsÚkernel_sizeÚstrider6   N)r<   r=   r   ÚConv1dr?   Údownsampling_rateÚconvr   Ú
hidden_actÚ
activationrH   rI   ©rR   r>   rV   s   &&€r%   r=   ÚCharactersToMolecules.__init__¸   st   ø€ Ü‰ÑÔä—I’IØ×*Ñ*Ø×+Ñ+Ø×0Ñ0Ø×+Ñ+ô	
ˆŒ	ô ! ×!2Ñ!2Õ3ˆŒäŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŽr(   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Úchar_encodingr{   ©r    ÚTensor)r#   r$   s   "€r%   r&   Ú"CharactersToMolecules.__annotate__Å   s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r(   c                ó,  € VR,          p\         P                  ! V^^4      pV P                  V4      p\         P                  ! V^^4      pV P                  V4      pVR ^ R1R 3,          p\         P                  ! W$.^R7      pV P                  V4      pV# )r~   rm   ©r~   ºé    r:   Nr~   r;   )r    Ú	transposer—   r™   rq   rH   )rR   r�   Úcls_encodingÚdownsampledÚdownsampled_truncatedÚresults   &&    r%   r‰   ÚCharactersToMolecules.forwardÅ   s�   € à$ YÕ/ˆô Ÿš¨°q¸!Ó<ˆØ—i‘i Ó.ˆÜ—o’o k°1°aÓ8ˆØ—o‘o kÓ2ˆð !,¨A¨q°¨t°Q¨JÕ 7Ðô —’˜LÐ@ÀaÔHˆà—‘ Ó'ˆàˆr(   )rH   r™   r—   ©
r*   r+   r,   r-   r.   r=   r‰   r0   r1   r‹   rŒ   s   @@r%   rŽ   rŽ   µ   s   ù‡ € ÙoõU÷÷ ð r(   rŽ   c                   óL   a a€ ] tR t^át oRtV 3R ltRV3R lR lltRtVtV ;t	# )ÚConvProjectionz�
Project representations from hidden_size*2 back to hidden_size across a window of w = config.upsampling_kernel_size
characters.
c                óœ  <€ \         SV `  4        Wn        \        P                  ! VP
                  ^,          VP
                  VP                  ^R7      V n        \        VP                  ,          V n
        \        P                  ! VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        R# )r   r�   r6   N)r<   r=   r>   r   r•   r?   Úupsampling_kernel_sizer—   r   r˜   r™   rH   rI   rJ   rK   rL   rš   s   &&€r%   r=   ÚConvProjection.__init__ç   sŽ   ø€ Ü‰ÑÔØŒÜ—I’IØ×*Ñ*¨QÕ.Ø×+Ñ+Ø×5Ñ5Øô	
ˆŒ	ô ! ×!2Ñ!2Õ3ˆŒÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr(   c                óv   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  /# )r   ÚinputsÚfinal_seq_char_positionsNr{   rž   )r#   r$   s   "€r%   r&   ÚConvProjection.__annotate__ô   s:   ø€ ÷ "ñ "á—‘ð"ñ #(§,¡,°Õ"5ð"ñ 
�‰ñ	"r(   c                ó¬  € \         P                  ! V^^4      pV P                  P                  ^,
          pV^,          pW4,
          p\        P
                  ! WE3^ 4      pV P                  V! V4      4      p\         P                  ! V^^4      pV P                  V4      pV P                  V4      pV P                  V4      pTpVe   \        R4      hTp	V	# )r:   z,CanineForMaskedLM is currently not supported)r    r¥   r>   r¯   r   ÚConstantPad1dr—   r™   rH   rL   ÚNotImplementedError)
rR   r²   r³   Ú	pad_totalÚpad_begÚpad_endÚpadr©   Úfinal_char_seqÚ	query_seqs
   &&&       r%   r‰   ÚConvProjection.forwardô   sÄ   € ô —’ ¨¨AÓ.ˆð
 —K‘K×6Ñ6¸Õ:ˆ	Ø˜q•.ˆØÕ%ˆä×Ò Ð1°1Ó5ˆà—‘™3˜v›;Ó'ˆÜ—’ ¨¨AÓ.ˆØ—‘ Ó(ˆØ—‘ Ó'ˆØ—‘˜fÓ%ˆØˆà#Ò/ô
 &Ð&TÓUÐUà&ˆIàÐr(   )rH   r™   r>   r—   rL   ©Nr«   rŒ   s   @@r%   r­   r­   á   s   ù‡ € ñõ
>÷"÷ "ò "r(   r­   c                   óH   a a€ ] tR tRt oV 3R ltRV3R lR lltRtVtV ;t# )ÚCanineSelfAttentioni  c                óþ  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d9   \	        VR4      '       g'   \        RVP                   RVP                   R24      hVP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  V P                  4      V n        \        P                  ! VP                  4      V n        R# )r¤   rk   zThe hidden size (z6) is not a multiple of the number of attention heads (Ú)N)r<   r=   r?   Únum_attention_headsÚhasattrra   r\   Úattention_head_sizeÚall_head_sizer   ÚLinearÚqueryÚkeyÚvaluerJ   Úattention_probs_dropout_probrL   rš   s   &&€r%   r=   ÚCanineSelfAttention.__init__  s  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z’z &×"EÑ"EÓFˆŽr(   c                óÜ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  R,          RS[R,          RS[S[ P                  S[ P                  R,          3,          /# )r   Úfrom_tensorÚ	to_tensorÚattention_maskNÚoutput_attentionsr{   )r    rŸ   r!   Úboolr"   )r#   r$   s   "€r%   r&   Ú CanineSelfAttention.__annotate__,  sk   ø€ ÷ ;ñ ;á—\‘\ð;ñ —<‘<ð;ñ ×)Ñ)¨DÕ0ð	;ñ
   $�;ð;ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñ;r(   c                óî  € VP                   w  rVpV P                  V4      P                  VRV P                  V P                  4      P                  ^^4      pV P                  V4      P                  VRV P                  V P                  4      P                  ^^4      p	V P                  V4      P                  VRV P                  V P                  4      P                  ^^4      p
\        P                  ! W¨P                  RR4      4      pV\        P                  ! V P                  4      ,          pVe‘   VP                  ^8X  d_   \        P                  ! V^R7      pRVP                  4       ,
          \        P                  ! VP                   4      P"                  ,          pW³P%                  VP                   4      ,           p\&        P(                  P+                  VRR7      pV P-                  V4      p\        P                  ! WÉ4      pVP/                  ^ ^^^4      P1                  4       pVP3                  4       RR V P4                  3,           pVP                  ! V!  pV'       d   WÜ3pV# V3pV# )r:   Nrm   g      ð?r;   éþÿÿÿ)ÚshaperÊ   ÚviewrÄ   rÆ   r¥   rË   rÉ   r    ÚmatmulÚmathÚsqrtÚndimÚ	unsqueezeÚfloatÚfinfor€   ÚminÚtor   Ú
functionalÚsoftmaxrL   ÚpermuteÚ
contiguousr‚   rÇ   )rR   rÏ   rÐ   rÑ   rÒ   Ú
batch_sizer†   Ú_Ú	key_layerÚvalue_layerÚquery_layerÚattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss   &&&&&           r%   r‰   ÚCanineSelfAttention.forward,  s  € ð %0×$5Ñ$5Ñ!ˆ
 ð �H‰H�YÓß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �J‰J�yÓ!ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �J‰J�{Ó#ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ô !Ÿ<š<¨×5HÑ5HÈÈRÓ5PÓQÐà+¬d¯iªi¸×8PÑ8PÓ.QÕQÐØÒ%Ø×"Ñ" aÔ'ä!&§¢°ÀQÔ!G�ð #&¨×(<Ñ(<Ó(>Õ">Ä%Ç+Â+ÐN^×NdÑNdÓBe×BiÑBiÕ!i�à/×2CÑ2CÐDT×DZÑDZÓ2[Õ[Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆäŸš _ÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÕ"SÐØ%×*Ò*Ð,CÑDˆç6G�=Ð2ˆàˆð O\ÐM]ˆàˆr(   )rÇ   rÆ   rL   rÊ   rÄ   rÉ   rË   ©NF©	r*   r+   r,   r-   r=   r‰   r0   r1   r‹   rŒ   s   @@r%   rÁ   rÁ     s   ù‡ € õG÷$;÷ ;ò ;r(   rÁ   c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚCanineSelfOutputij  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  4      V n
        R# ©r6   N)r<   r=   r   rÈ   r?   ÚdenserH   rI   rJ   rK   rL   rš   s   &&€r%   r=   ÚCanineSelfOutput.__init__k  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr(   c                ó¢   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  RS[ S[P                  S[P                  3,          /# ©r   r   Úinput_tensorr{   ©r"   r    r!   )r#   r$   s   "€r%   r&   ÚCanineSelfOutput.__annotate__q  sL   ø€ ÷ ñ Ù"¡5×#4Ñ#4Õ5ðÙEJ×EVÑEVðá	‰u× Ñ ¡%×"3Ñ"3Ð3Õ	4ñr(   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# r¿   ©r÷   rL   rH   ©rR   r   rû   s   &&&r%   r‰   ÚCanineSelfOutput.forwardq  s7   € ð Ÿ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }Õ'CÓDˆØÐr(   ©rH   r÷   rL   rò   rŒ   s   @@r%   rô   rô   j  s   ù‡ € õ>÷÷ ð r(   rô   c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
ÚCanineAttentioniz  aB  
Additional arguments related to local attention:

    - **local** (`bool`, *optional*, defaults to `False`) -- Whether to apply local attention.
    - **always_attend_to_first_position** (`bool`, *optional*, defaults to `False`) -- Should all blocks be able to
      attend
    to the `to_tensor`'s first position (e.g. a [CLS] position)? - **first_position_attends_to_all** (`bool`,
    *optional*, defaults to `False`) -- Should the *from_tensor*'s first position be able to attend to all
    positions within the *from_tensor*? - **attend_from_chunk_width** (`int`, *optional*, defaults to 128) -- The
    width of each block-wise chunk in `from_tensor`. - **attend_from_chunk_stride** (`int`, *optional*, defaults to
    128) -- The number of elements to skip when moving to the next block in `from_tensor`. -
    **attend_to_chunk_width** (`int`, *optional*, defaults to 128) -- The width of each block-wise chunk in
    *to_tensor*. - **attend_to_chunk_stride** (`int`, *optional*, defaults to 128) -- The number of elements to
    skip when moving to the next block in `to_tensor`.
c                ó>   <€ V ^8„  d   QhRS[ RS[ RS[RS[RS[RS[/# )r   Úalways_attend_to_first_positionÚfirst_position_attends_to_allÚattend_from_chunk_widthÚattend_from_chunk_strideÚattend_to_chunk_widthÚattend_to_chunk_stride)rÓ   r\   )r#   r$   s   "€r%   r&   ÚCanineAttention.__annotate__‹  sI   ø€ ÷ =ñ =ñ *.ð	=ñ
 (,ð=ñ "%ð=ñ #&ð=ñ  #ð=ñ !$ñ=r(   c	                óü   <€ \         S	V `  4        \        V4      V n        \	        V4      V n        W n        WV8  d   \        R 4      hWx8  d   \        R4      hW0n        W@n	        WPn
        W`n        Wpn        W€n        R# )ze`attend_from_chunk_width` < `attend_from_chunk_stride` would cause sequence positions to get skipped.z``attend_to_chunk_width` < `attend_to_chunk_stride`would cause sequence positions to get skipped.N)r<   r=   rÁ   rR   rô   ÚoutputÚlocalra   r  r  r  r	  r
  r  ©
rR   r>   r  r  r  r  r	  r
  r  rV   s
   &&&&&&&&&€r%   r=   ÚCanineAttention.__init__‹  s„   ø€ ô 	‰ÑÔÜ'¨Ó/ˆŒ	Ü& vÓ.ˆŒð Œ
Ø"Ô=ÜØwóð ð !Ô9ÜØróð ð 0OÔ,Ø-JÔ*Ø'>Ô$Ø(@Ô%Ø%:Ô"Ø&<Ö#r(   c                óÒ   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  R,          RS[R,          RS[ S[P                  S[P                  R,          3,          /# ©r   r   rÑ   NrÒ   r{   ©r"   r    r!   rÓ   )r#   r$   s   "€r%   r&   r  «  sn   ø€ ÷ Gñ Gá™U×.Ñ.Õ/ðGñ ×)Ñ)¨DÕ0ðGñ   $�;ð	Gñ
 
‰u× Ñ ¡%×"3Ñ"3°dÕ":Ð:Õ	;ñGr(   c                óf  € V P                   '       g   V P                  WW#4      pV^ ,          pEM7VP                  ^,          ;rgT;r‰. p
V P                  '       d   V
P	                  R4       ^pM^ p\        W¶V P                  4       F1  p\        WlV P                  ,           4      pV
P	                  WÍ34       K3  	  . pV P                  '       d   VP	                  ^ V34       \        ^ WpP                  4       F1  p\        W|V P                  ,           4      pVP	                  WÍ34       K3  	  \        V
4      \        V4      8w  d   \        RV
 RV
 R24      h. p. p\        W®4       Fã  w  w  ppw  ppVRVV1R3,          pV	RVV1R3,          pVRVV1VV13,          pV P                  '       dL   VRVV1R3,          p\        P                   ! VV.^R7      pV	R	,          p\        P                   ! VV.^R7      pV P                  VVVV4      pVP	                  V^ ,          4       V'       g   KË  VP	                  V^,          4       Kå  	  \        P                   ! V^R7      pV P#                  WQ4      pV3pV P                   '       g   VXR,          ,           pV# V\%        X4      ,           pV# )
r¤   z/Expected to have same number of `from_chunks` (z) and `to_chunks` (z). Check strides.r~   r£   rm   ºr:   NN)r¤   r:   r¢   )r  rR   r×   r  rb   rA   r	  rà   r  r  r
  r_   ra   Úzipr  r    rq   r  r"   )rR   r   rÑ   rÒ   Úself_outputsÚattention_outputÚfrom_seq_lengthÚto_seq_lengthrÏ   rÐ   Úfrom_chunksÚ
from_startÚchunk_startÚ	chunk_endÚ	to_chunksÚattention_output_chunksÚattention_probs_chunksÚfrom_endÚto_startÚto_endÚfrom_tensor_chunkÚto_tensor_chunkÚattention_mask_chunkÚcls_attention_maskÚcls_positionÚattention_outputs_chunkrï   s   &&&&                       r%   r‰   ÚCanineAttention.forward«  sÌ  € ð �z�zˆzØŸ9™9 ]À>ÓeˆLØ+¨A�Òà.;×.AÑ.AÀ!Õ.DÐDˆOØ&3Ð3ˆKð ˆKØ×1×1Ð1Ø×"Ñ" 6Ô*ð ‘
à�
Ü$ ZÀ$×B_ÑB_Ö`�Ü ¸t×?[Ñ?[Õ1[Ó\�	Ø×"Ñ" KÐ#;Ö<ñ  að
 ˆIØ×1×1Ð1Ø× Ñ  ! ]Ð!3Ô4Ü$ Q¨×7RÑ7RÖS�Ü ¸T×=WÑ=WÕ/WÓX�	Ø× Ñ  +Ð!9Ö:ñ  Tô �;Ó¤3 y£>Ô1Ü ØEÀkÀ]ð S$Ø$/ =Ð0AðCóð ð ')Ð#Ø%'Ð"Ü>AÀ+Ö>YÑ:Ñ&�˜XÑ(:¨°6Ø$/°°:¸hÐ3FÈÐ0IÕ$JÐ!Ø"+¨A¨x¸¨ÀÐ,AÕ"B�ð (6°a¸ÀHÐ9LÈhÐW]ÈoÐ6]Õ'^Ð$Ø×7×7Ð7Ø)7¸¸:ÀhÐ;NÐPSÐ8SÕ)TÐ&Ü+0¯9ª9Ð6HÐJ^Ð5_ÐefÔ+gÐ(à#,¨YÕ#7�LÜ&+§i¢i°¸Ð0OÐUVÔ&W�Oà*.¯)©)Ø% Ð8LÐN_ó+Ð'ð (×.Ñ.Ð/FÀqÕ/IÔJß$Ñ$Ø*×1Ñ1Ð2IÈ!Õ2LÖMñ% ?Zô(  %ŸyšyÐ)@ÀaÔHÐàŸ;™;Ð'7ÓGÐØ#Ð%ˆØ�z�zˆzØ ¨RÕ 0Õ0ˆGð ˆð ¤Ð&<Ó =Õ=ˆGØˆr(   )	r  r	  r  r  r
  r  r  r  rR   ©FFFé€   r.  r.  r.  rñ   r«   rŒ   s   @@r%   r  r  z  s'   ù‡ € ñ÷ =õ =÷@G÷ Gò Gr(   r  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚCanineIntermediateiõ  c                ó"  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n        R # VP                  V n        R # r¿   )r<   r=   r   rÈ   r?   Úintermediate_sizer÷   Ú
isinstancer˜   Ústrr   Úintermediate_act_fnrš   s   &&€r%   r=   ÚCanineIntermediate.__init__ö  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬×-Ò-Ü'-¨f×.?Ñ.?Õ'@ˆDÖ$à'-×'8Ñ'8ˆDÖ$r(   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   r   r{   )r    r!   )r#   r$   s   "€r%   r&   ÚCanineIntermediate.__annotate__þ  s'   ø€ ÷ ñ ¡U×%6Ñ%6ð ¹5×;LÑ;Lñ r(   c                óJ   € V P                  V4      pV P                  V4      pV# r¿   ©r÷   r5  ©rR   r   s   &&r%   r‰   ÚCanineIntermediate.forwardþ  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr(   r;  rò   rŒ   s   @@r%   r0  r0  õ  s   ù‡ € õ9÷÷ ð r(   r0  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚCanineOutputi  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R 7      V n        \        P                  ! VP                  4      V n        R# rö   )r<   r=   r   rÈ   r2  r?   r÷   rH   rI   rJ   rK   rL   rš   s   &&€r%   r=   ÚCanineOutput.__init__  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr(   c                óx   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  RS[P                  /# rú   rü   )r#   r$   s   "€r%   r&   ÚCanineOutput.__annotate__  s:   ø€ ÷ ñ ¡U©5×+<Ñ+<Õ%=ð ÉU×M^ÑM^ð Ñch×ctÑctñ r(   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# r¿   rÿ   r   s   &&&r%   r‰   ÚCanineOutput.forward  s5   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }Õ'CÓDˆØÐr(   r  rò   rŒ   s   @@r%   r?  r?    s   ù‡ € õ>÷÷ ð r(   r?  c                   óN   a a€ ] tR tRt oV 3R ltRV3R lR lltR tRtVtV ;t	# )ÚCanineLayeri  c	           
     óÂ   <€ \         S	V `  4        VP                  V n        ^V n        \	        VVVVVVVV4      V n        \        V4      V n        \        V4      V n	        R# ©r:   N)
r<   r=   Úchunk_size_feed_forwardÚseq_len_dimr  Ú	attentionr0  Úintermediater?  r  r  s
   &&&&&&&&&€r%   r=   ÚCanineLayer.__init__  se   ø€ ô 	‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ(ØØØ+Ø)Ø#Ø$Ø!Ø"ó	
ˆŒô /¨vÓ6ˆÔÜ" 6Ó*ˆŽr(   c                óÒ   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  R,          RS[R,          RS[ S[P                  S[P                  R,          3,          /# r  r  )r#   r$   s   "€r%   r&   ÚCanineLayer.__annotate__.  sh   ø€ ÷ ñ á™U×.Ñ.Õ/ðñ ×)Ñ)¨DÕ0ðñ   $�;ð	ñ
 
‰u× Ñ ¡%×"3Ñ"3°dÕ":Ð:Õ	;ñr(   c                ó¾   € V P                  VVVR 7      pV^ ,          pVR,          p\        V P                  V P                  V P                  V4      pV3V,           pV# ))rÒ   r  )rL  r   Úfeed_forward_chunkrJ  rK  )rR   r   rÑ   rÒ   Úself_attention_outputsr  rï   Úlayer_outputs   &&&&    r%   r‰   ÚCanineLayer.forward.  sq   € ð "&§¡ØØØ/ð "0ó "
Ðð
 2°!Õ4Ðà(¨Õ,ˆä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð  �/ GÕ+ˆàˆr(   c                óJ   € V P                  V4      pV P                  W!4      pV# r¿   )rM  r  )rR   r  Úintermediate_outputrT  s   &&  r%   rR  ÚCanineLayer.feed_forward_chunkD  s)   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6ÓIˆØÐr(   )rL  rJ  rM  r  rK  rñ   )
r*   r+   r,   r-   r=   r‰   rR  r0   r1   r‹   rŒ   s   @@r%   rG  rG    s   ù‡ € õ+÷6ò ÷,ò r(   rG  c                   óL   a a€ ] tR tRt oRV 3R lltRV3R lR lltRtVtV ;t# )ÚCanineEncoderiJ  c	                óä   <€ \         S
V `  4        Wn        \        P                  ! \        VP                  4       U	u. uF  p	\        VVVVVVVV4      NK  	  up	4      V n        R V n	        R# u up	i )FN)
r<   r=   r>   r   Ú
ModuleListrA   Únum_hidden_layersrG  ÚlayerÚgradient_checkpointing)rR   r>   r  r  r  r  r	  r
  r  rç   rV   s   &&&&&&&&& €r%   r=   ÚCanineEncoder.__init__K  s}   ø€ ô 	‰ÑÔØŒÜ—]’]ô ˜v×7Ñ7Ô8óñ 9�Aô ØØØ3Ø1Ø+Ø,Ø)Ø*ö	ñ 9ñó
ˆŒ
ð ',ˆÖ#ùòs   ½A-c                ó¾   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  R,          RS[R,          RS[R,          RS[R,          RS[ S[,          /# )r   r   rÑ   NrÒ   Úoutput_hidden_statesÚreturn_dictr{   )r"   r    r!   rÓ   r   )r#   r$   s   "€r%   r&   ÚCanineEncoder.__annotate__i  sn   ø€ ÷ 
ñ 
á™U×.Ñ.Õ/ð
ñ ×)Ñ)¨DÕ0ð
ñ   $�;ð	
ñ
 # T�kð
ñ ˜D•[ð
ñ 
‘Õ	 ñ
r(   c                ó–  € V'       d   RMR pV'       d   RMR p\        V P                  4       FB  w  r‰V'       d
   Wa3,           pV	! WV4      p
V
^ ,          pV'       g   K2  Wz^,          3,           pKD  	  V'       d
   Wa3,           pV'       g1   \        ;QJ d    . R WV3 4       F  NK  	  5# ! R WV3 4       4      # \        VVVR7      # )Nc              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5ir¿   r)   ©Ú.0Úvs   & r%   Ú	<genexpr>Ú(CanineEncoder.forward.<locals>.<genexpr>‚  s   é € ÐmÑ$[˜qŸšÓ$[ùó   ‚Œ
)r   r   r   r)   )ro   r^  r"   r   )rR   r   rÑ   rÒ   rb  rc  Úall_hidden_statesÚall_self_attentionsrT   Úlayer_moduleÚlayer_outputss   &&&&&&     r%   r‰   ÚCanineEncoder.forwardi  sÂ   € ÷ #7™B¸DÐß$5™b¸4Ðä(¨¯©Ö4‰OˆAß#Ø$5Ð8HÕ$HÐ!á(¨ÐHYÓZˆMà)¨!Õ,ˆMß Ñ Ø&9È1Õ=MÐ<OÕ&OÒ#ñ  5÷  Ø 1Ð4DÕ DÐçß”5Ñm ]ÐGZÑ$[Óm—5Ðm�5Ñm ]ÐGZÑ$[ÓmÓmÐmÜØ+Ø+Ø*ô
ð 	
r(   )r>   r_  r^  r-  )NFFTrò   rŒ   s   @@r%   rZ  rZ  J  s   ù‡ € ÷,÷<
÷ 
ò 
r(   rZ  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚCaninePooleriŠ  c                ó¸   <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! 4       V n        R # r¿   )r<   r=   r   rÈ   r?   r÷   ÚTanhr™   rš   s   &&€r%   r=   ÚCaninePooler.__init__‹  s9   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'š'›)ˆŽr(   c                ó^   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  /# r8  rü   )r#   r$   s   "€r%   r&   ÚCaninePooler.__annotate__�  s,   ø€ ÷ ñ ¡U©5×+<Ñ+<Õ%=ð Á%×BSÑBSñ r(   c                ó\   € VR,          pV P                  V4      pV P                  V4      pV# )r~   )r~   r¤   )r÷   r™   )rR   r   Úfirst_token_tensorÚpooled_outputs   &&  r%   r‰   ÚCaninePooler.forward�  s2   € ð +¨4Õ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr(   )r™   r÷   rò   rŒ   s   @@r%   rs  rs  Š  s   ù‡ € õ$÷
÷ ð r(   rs  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚCaninePredictionHeadTransformi™  c                ó‚  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n
        MVP                  V n
        \        P                  ! VP                  VP                  R 7      V n        R# rö   )r<   r=   r   rÈ   r?   r÷   r3  r˜   r4  r   Útransform_act_fnrH   rI   rš   s   &&€r%   r=   Ú&CaninePredictionHeadTransform.__init__š  s~   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬×-Ò-Ü$*¨6×+<Ñ+<Õ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŽr(   c                ó^   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  /# r8  rü   )r#   r$   s   "€r%   r&   Ú*CaninePredictionHeadTransform.__annotate__£  s,   ø€ ÷ ñ ¡U©5×+<Ñ+<Õ%=ð Á%×BSÑBSñ r(   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# r¿   )r÷   r€  rH   r<  s   &&r%   r‰   Ú%CaninePredictionHeadTransform.forward£  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐr(   )rH   r÷   r€  rò   rŒ   s   @@r%   r~  r~  ™  s   ù‡ € õU÷÷ ð r(   r~  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚCanineLMPredictionHeadiª  c                ó  <€ \         SV `  4        \        V4      V n        \        P
                  ! VP                  VP                  R R7      V n        \        P                  ! \        P                  ! VP                  4      4      V n        R# )T)ÚbiasN)r<   r=   r~  Ú	transformr   rÈ   r?   Ú
vocab_sizeÚdecoderÚ	Parameterr    rƒ   r‰  rš   s   &&€r%   r=   ÚCanineLMPredictionHead.__init__«  s[   ø€ Ü‰ÑÔÜ6°vÓ>ˆŒô —y’y ×!3Ñ!3°V×5FÑ5FÈTÔRˆŒä—L’L¤§¢¨V×->Ñ->Ó!?Ó@ˆŽ	r(   c                ó^   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  /# r8  rü   )r#   r$   s   "€r%   r&   Ú#CanineLMPredictionHead.__annotate__·  s,   ø€ ÷ ñ ¡U©5×+<Ñ+<Õ%=ð Á%×BSÑBSñ r(   c                óJ   € V P                  V4      pV P                  V4      pV# r¿   )rŠ  rŒ  r<  s   &&r%   r‰   ÚCanineLMPredictionHead.forward·  s$   € ØŸ™ }Ó5ˆØŸ™ ]Ó3ˆØÐr(   )r‰  rŒ  rŠ  rò   rŒ   s   @@r%   r‡  r‡  ª  s   ù‡ € õA÷÷ ð r(   r‡  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚCanineOnlyMLMHeadi½  c                óD   <€ \         SV `  4        \        V4      V n        R # r¿   )r<   r=   r‡  Úpredictionsrš   s   &&€r%   r=   ÚCanineOnlyMLMHead.__init__¾  s   ø€ Ü‰ÑÔÜ1°&Ó9ˆÖr(   c                ó^   <€ V ^8„  d   QhRS[ S[P                  ,          RS[P                  /# )r   Úsequence_outputr{   )r"   r    rŸ   )r#   r$   s   "€r%   r&   ÚCanineOnlyMLMHead.__annotate__Â  s,   ø€ ÷ !ñ !á™uŸ|™|Õ,ð!ñ 
�‰ñ!r(   c                ó(   € V P                  V4      pV# r¿   ©r–  )rR   r™  Úprediction_scoress   && r%   r‰   ÚCanineOnlyMLMHead.forwardÂ  s   € ð !×,Ñ,¨_Ó=ÐØ Ð r(   rœ  rò   rŒ   s   @@r%   r”  r”  ½  s   ù‡ € õ:÷!÷ !ð !r(   r”  c                   óF   a a€ ] tR tRt oRtRtV 3R ltV3R ltRtVt	V ;t
# )ÚCaninePreTrainedModeliÊ  ÚcanineTc                ó  <€ \         SV `  V4       \        V\        4      '       db   \        P
                  ! VP                  \        P                  ! VP                  P                  R,          4      P                  R4      4       R# R# )r:   Nr;   r9   )r<   Ú_init_weightsr3  r3   ÚinitÚcopy_rQ   r    rN   r×   rP   )rR   ÚmodulerV   s   &&€r%   r£  Ú#CaninePreTrainedModel._init_weightsÐ  s^   ø€ Ü‰Ñ˜fÔ%Ü�fÔ.×/Ò/Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÖhñ 0r(   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # )r   r>   r   )r#   r$   s   "€r%   r&   Ú"CaninePreTrainedModel.__annotate__Ê  s   ø‡ ‚ áÑò r(   r)   )r*   r+   r,   r-   Úbase_model_prefixÚsupports_gradient_checkpointingr£  r/   r0   r1   r‹   rŒ   s   @@r%   r   r   Ê  s    ù‡ € ð !ÐØ&*Ð#õi÷ … r(   r   c                   ó€   a a€ ] tR tRt oRV 3R lltR tV3R lR ltV3R lR lt]RV3R lR	 ll4       t	R
t
VtV ;t# )ÚCanineModeliÖ  c                óà  <€ \         SV `  V4       Wn        \        P                  ! V4      p^Vn        \        V4      V n        \        VRRRVP                  VP                  VP                  VP                  R7      V n
        \        V4      V n        \        V4      V n        \        V4      V n        \        V4      V n        V'       d   \#        V4      MRV n        V P'                  4        R# )z^
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
TF)r  r  r  r  r	  r
  r  N)r<   r=   r>   ÚcopyÚdeepcopyr]  r3   Úchar_embeddingsrZ  Úlocal_transformer_strideÚinitial_char_encoderrŽ   Úchars_to_moleculesÚencoderr­   Ú
projectionÚfinal_char_encoderrs  ÚpoolerÚ	post_init)rR   r>   Úadd_pooling_layerÚshallow_configrV   s   &&& €r%   r=   ÚCanineModel.__init__Ø  sÇ   ø€ ô
 	‰Ñ˜Ô ØŒÜŸš vÓ.ˆØ+,ˆÔ(ä/°Ó7ˆÔä$1ØØØ,1Ø*/Ø$*×$CÑ$CØ%+×%DÑ%DØ"(×"AÑ"AØ#)×#BÑ#Bô	%
ˆÔ!ô #8¸Ó"?ˆÔä$ VÓ,ˆŒÜ(¨Ó0ˆŒä"/°Ó"?ˆÔç.?”l 6Ô*ÀTˆŒð 	�‰Ör(   c                ó:  € VP                   ^ ,          VP                   ^,          rCVP                   ^,          p\        P                  ! W#^V34      P                  4       p\        P                  ! W4^3\        P
                  VP                  R7      pWb,          pV# )a  
Create 3D attention mask from a 2D tensor mask.

Args:
    from_tensor: 2D or 3D Tensor of shape [batch_size, from_seq_length, ...].
    to_mask: int32 Tensor of shape [batch_size, to_seq_length].

Returns:
    float Tensor of shape [batch_size, from_seq_length, to_seq_length].
)r‚   r€   r�   )r×   r    ÚreshaperÞ   ÚonesÚfloat32r�   )rR   rÏ   Úto_maskræ   r  r  Úbroadcast_onesÚmasks   &&&     r%   Ú)_create_3d_attention_mask_from_input_maskÚ5CanineModel._create_3d_attention_mask_from_input_maskú  sƒ   € ð '2×&7Ñ&7¸Õ&:¸K×<MÑ<MÈaÕ<P�OàŸ™ aÕ(ˆä—-’- °a¸Ð)GÓH×NÑNÓPˆô
 Ÿš¨*ÀqÐ)IÔQV×Q^ÑQ^Ðgn×guÑguÔvˆð Õ'ˆàˆr(   c                ó:   <€ V ^8„  d   QhRS[ P                  RS[/# )r   Úchar_attention_maskr–   ©r    rŸ   r\   )r#   r$   s   "€r%   r&   ÚCanineModel.__annotate__  s    ø€ ÷ 3ñ 3¹e¿l¹lð 3Ñ_bñ 3r(   c                óÜ   € VP                   w  r4\        P                  ! W^V34      p\        P                  P	                  W"R7      ! VP                  4       4      pVP                  ^R7      # )z[Downsample 2D character attention mask to 2D molecule attention mask using MaxPool1d layer.)r“   r”   rm   )r×   r    r¾  r   Ú	MaxPool1drÞ   Úsqueeze)rR   rÇ  r–   ræ   Úchar_seq_lenÚpoolable_char_maskÚpooled_molecule_masks   &&&    r%   Ú_downsample_attention_maskÚ&CanineModel._downsample_attention_mask  sl   € ð $7×#<Ñ#<Ñ ˆ
Ü"Ÿ]š]Ð+>ÈQÐP\Ð@]Ó^Ðô  %Ÿx™x×1Ñ1Ð>OÐ1ÔjØ×$Ñ$Ó&ó 
Ðð
 $×+Ñ+°Ð+Ó2Ð2r(   c                óT   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  /# )r   Ú	moleculesÚchar_seq_lengthr{   rÈ  )r#   r$   s   "€r%   r&   rÉ  $  s2   ø€ ÷ Añ A©5¯<©<ð AÉ#ð AÑRW×R^ÑR^ñ Ar(   c                ó  € V P                   P                  pVR,          p\        P                  ! WCRR7      pVRRR1R3,          pW#,          p\        P                  ! VWs,           RR7      p\        P                  ! WX.RR7      # )zDRepeats molecules to make them the same length as the char sequence.r~   )Úrepeatsrn   Nrm   )r~   r  r~   rÖ   r;   )r>   r–   r    Úrepeat_interleaverq   )	rR   rÓ  rÔ  ÚrateÚmolecules_without_extra_clsÚrepeatedÚlast_moleculeÚremainder_lengthÚremainder_repeateds	   &&&      r%   Ú_repeat_moleculesÚCanineModel._repeat_molecules$  s‚   € ð �{‰{×,Ñ,ˆà&/°Õ&9Ð#ä×*Ò*Ð+FÐZ\Ô]ˆð " ! R¡S¨! )Õ,ˆØ*Õ1ÐÜ"×4Ò4Øà$Õ+Øô	
Ðô �yŠy˜(Ð7¸RÔ@Ð@r(   c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[,          /	# )r   rc   NrÑ   ry   rQ   rz   rÒ   rb  rc  r{   )r    r|   r!   rÓ   r"   r   )r#   r$   s   "€r%   r&   rÉ  >  s¹   ø€ ÷ \
ñ \
á×#Ñ# dÕ*ð\
ñ ×)Ñ)¨DÕ0ð\
ñ ×(Ñ(¨4Õ/ð	\
ñ
 ×&Ñ&¨Õ-ð\
ñ ×(Ñ(¨4Õ/ð\
ñ   $�;ð\
ñ # T�kð\
ñ ˜D•[ð\
ñ 
Ñ-Õ	-ñ\
r(   c	                ó  € Ve   TMV P                   P                  pVe   TMV P                   P                  pV'       d   RMR p
V'       d   RMR pVe   TMV P                   P                  pVe   Ve   \	        R4      hVe#   V P                  W4       VP                  4       pM#Ve   VP                  4       R R pM\	        R4      hVw  rÞVe   VP                  MVP                  pVf   \        P                  ! WÞ3VR7      pVf(   \        P                  ! V\        P                  VR7      pV P                  W P                   P                  R7      pV P                  VVVVR7      pT P                  Ve   TMTV4      pV P!                  VVVVR7      pVP"                  pV P%                  V4      p\'        V P                   VR,          VR7      pV P)                  VVVVVR	7      pV^ ,          pV P*                  e   V P+                  V4      MR pV P-                  VVR,          R
7      p\        P.                  ! VV.RR7      pV P1                  V4      p\'        V P                   VVR7      pV P3                  VVVVR7      pVP"                  pV'       dI   V'       d   VP4                  MV^,          pV
VP4                  ,           V,           VP4                  ,           p
V'       dI   V'       d   VP6                  MVR,          pVVP6                  ,           V,           VP6                  ,           pV'       gD   VV3pT\8        ;QJ d    . R W«3 4       F  NK  	  5,          pV# ! R W«3 4       4      ,          pV# \;        VVV
VR7      # )NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embeds)r�   r   )r–   )rc   rQ   ry   rz   )rÑ   rÒ   rb  )r>   rz   rÑ   )rÑ   rÒ   rb  rc  )rÔ  rm   c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5ir¿   r)   rg  s   & r%   rj  Ú&CanineModel.forward.<locals>.<genexpr>Ò  s   é € ÐaÑ'O !ŸAšAÓ'Oùrl  )r   r   r   r   r)   r;   r¢   )r>   rÒ   rb  rc  ra   Ú%warn_if_padding_and_no_attention_maskr‚   r�   r    r¿  rƒ   r„   rÐ  r–   r±  rÄ  r³  r   r´  r	   rµ  r¸  rÞ  rq   r¶  r·  r   r   r"   r   ) rR   rc   rÑ   ry   rQ   rz   rÒ   rb  rc  Úkwargsrm  rn  r…   ræ   r†   r�   Úmolecule_attention_maskÚinput_char_embeddingsrÇ  Úinit_chars_encoder_outputsÚinput_char_encodingÚinit_molecule_encodingÚencoder_outputsÚmolecule_sequence_outputr{  Úrepeated_moleculesÚconcatr™  Úfinal_chars_encoder_outputsÚdeep_encoder_hidden_statesÚdeep_encoder_self_attentionsr  s    &&&&&&&&&,                      r%   r‰   ÚCanineModel.forward=  s¹  € ð 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	÷ #7™B¸DÐß$5™b¸4ÐØ%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒ  ]Ò%>ÜÐcÓdÐdØÒ"Ø×6Ñ6°yÔQØ#Ÿ.™.Ó*‰KØÒ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
Ø%.Ò%:�×!Ò!À×@TÑ@TˆàÒ!Ü"ŸZšZ¨*Ð)AÈ6ÔRˆNØÒ!Ü"Ÿ[š[¨¼E¿J¹JÈvÔVˆNð #'×"AÑ"AØ¯k©k×.KÑ.Kð #Bó #
Ðð
 !%× 4Ñ 4ØØ%Ø)Ø'ð	 !5ó !
Ðð #×LÑLØ"Ò.‰I°MÀ>ó
Ðð &*×%>Ñ%>Ø!Ø.Ø/Ø!5ð	 &?ó &
Ð"ð 9×JÑJÐð  "&×!8Ñ!8Ð9LÓ!MÐä";Ø—;‘;Ø0°Õ;Ø2ô#
Ðð Ÿ,™,Ø"Ø2Ø/Ø!5Ø#ð 'ó 
ˆð $3°1Õ#5Ð ØAEÇÁÒAX˜Ÿ™Ð$<Ô=Ð^bˆð "×3Ñ3Ð4LÐ^iÐjlÕ^mÐ3ÓnÐô —’Ð/Ð1CÐDÈ"ÔMˆð Ÿ/™/¨&Ó1ˆä2Ø—;‘;Ø)Ø)ô
ˆð '+×&=Ñ&=ØØ)Ø/Ø!5ð	 '>ó '
Ð#ð 6×GÑGˆçßJU¨×)FÒ)FÐ[jÐklÕ[mÐ&à!Ø,×:Ñ:õ;à,õ-ð .×;Ñ;õ<ð ÷ ßIT¨?×+EÒ+EÐZiÐjlÕZmÐ(à#Ø,×7Ñ7õ8à.õ/ð .×8Ñ8õ9ð  ÷ Ø% }Ð5ˆFØ—e”eÑaÐ(9Ñ'OÓa—eÕaˆFØˆMð ÑaÐ(9Ñ'OÓaÓaÕaˆFØˆMä+Ø-Ø'Ø+Ø*ô	
ð 	
r(   )r±  r´  r>   rµ  r·  r³  r¸  r¶  )T)NNNNNNNN)r*   r+   r,   r-   r=   rÄ  rÐ  rÞ  r   r‰   r0   r1   r‹   rŒ   s   @@r%   r­  r­  Ö  sH   ù‡ € ÷ òD÷63ð 3÷Að Að2 ÷\
ñ \
ó ÷\
ð \
r(   r­  zž
    CANINE Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚCanineForSequenceClassificationiÝ  c                ó2  <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R # r¿   ©r<   r=   Ú
num_labelsr­  r¡  r   rJ   rK   rL   rÈ   r?   Ú
classifierr¹  rš   s   &&€r%   r=   Ú(CanineForSequenceClassification.__init__ä  ói   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &Ó)ˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör(   c                ó\  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[R,          RS[S[,          /
# ©r   rc   NrÑ   ry   rQ   rz   ÚlabelsrÒ   rb  rc  r{   )r    r|   r!   rÓ   r"   r   )r#   r$   s   "€r%   r&   Ú,CanineForSequenceClassification.__annotate__ð  sÏ   ø€ ÷ D
ñ D
á×#Ñ# dÕ*ðD
ñ ×)Ñ)¨DÕ0ðD
ñ ×(Ñ(¨4Õ/ð	D
ñ
 ×&Ñ&¨Õ-ðD
ñ ×(Ñ(¨4Õ/ðD
ñ × Ñ  4Õ'ðD
ñ   $�;ðD
ñ # T�kðD
ñ ˜D•[ðD
ñ 
Ñ)Õ	)ñD
r(   c
                ó–  € V	e   T	MV P                   P                  p	V P                  VVVVVVVV	R7      pV^,          pV P                  V4      pV P	                  V4      pRpVEe˜   V P                   P
                  f–   V P                  ^8X  d   RV P                   n        MrV P                  ^8”  dQ   VP                  \        P                  8X  g    VP                  \        P                  8X  d   RV P                   n        MRV P                   n        V P                   P
                  R8X  dK   \        4       pV P                  ^8X  d'   V! VP                  4       VP                  4       4      pMŽV! WÖ4      pM…V P                   P
                  R8X  d>   \        4       pV! VP                  RV P                  4      VP                  R4      4      pM-V P                   P
                  R8X  d   \        4       pV! WÖ4      pV	'       g"   V3VR,          ,           pVe   V3V,           # T# \!        VVVP"                  VP$                  R7      # )	ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
N©rÑ   ry   rQ   rz   rÒ   rb  rc  Ú
regressionÚsingle_label_classificationÚmulti_label_classificationºr   NN©ÚlossÚlogitsr   r   r;   )r>   rc  r¡  rL   rø  Úproblem_typer÷  r€   r    r„   r\   r   rÌ  r   rØ   r   r   r   r   )rR   rc   rÑ   ry   rQ   rz   rý  rÒ   rb  rc  rå  rï   r{  r  r  Úloss_fctr  s   &&&&&&&&&&,      r%   r‰   Ú'CanineForSequenceClassification.forwardï  sß  € ð( &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—+‘+ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð   �
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ßØ�Y ¨¥Õ,ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r(   ©r¡  rø  rL   r÷  ©	NNNNNNNNN©
r*   r+   r,   r-   r=   r   r‰   r0   r1   r‹   rŒ   s   @@r%   rô  rô  Ý  s,   ù‡ € õ	ð ÷D
ñ D
ó ÷D
ð D
r(   rô  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚCanineForMultipleChoicei7  c                óü   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  4      V n        \        P                  ! VP                  ^4      V n
        V P                  4        R# rI  )r<   r=   r­  r¡  r   rJ   rK   rL   rÈ   r?   rø  r¹  rš   s   &&€r%   r=   Ú CanineForMultipleChoice.__init__9  sV   ø€ Ü‰Ñ˜Ô ä! &Ó)ˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰Ör(   c                ó\  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[R,          RS[S[,          /
# rü  )r    r|   r!   rÓ   r"   r   )r#   r$   s   "€r%   r&   Ú$CanineForMultipleChoice.__annotate__D  sÏ   ø€ ÷ W
ñ W
á×#Ñ# dÕ*ðW
ñ ×)Ñ)¨DÕ0ðW
ñ ×(Ñ(¨4Õ/ð	W
ñ
 ×&Ñ&¨Õ-ðW
ñ ×(Ñ(¨4Õ/ðW
ñ × Ñ  4Õ'ðW
ñ   $�;ðW
ñ # T�kðW
ñ ˜D•[ðW
ñ 
Ñ*Õ	*ñW
r(   c
                ó¨  € V	e   T	MV P                   P                  p	Ve   VP                  ^,          MVP                  ^,          pVe"   VP                  RVP	                  R4      4      MRpVe"   VP                  RVP	                  R4      4      MRpVe"   VP                  RVP	                  R4      4      MRpVe"   VP                  RVP	                  R4      4      MRpVe2   VP                  RVP	                  R4      VP	                  R4      4      MRpV P                  VVVVVVVV	R7      pV^,          pV P                  V4      pV P                  V4      pVP                  RV4      pRpVe   \        4       pV! Wö4      pV	'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  R7      # )a›  
input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
    1]`:

    - 0 corresponds to a *sentence A* token,
    - 1 corresponds to a *sentence B* token.

    [What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
    config.max_position_embeddings - 1]`.

    [What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
    model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
    `input_ids` above)
Nr   r  r  r;   rÖ   )r>   rc  r×   rØ   r‚   r¡  rL   rø  r   r   r   r   )rR   rc   rÑ   ry   rQ   rz   rý  rÒ   rb  rc  rå  Únum_choicesrï   r{  r  Úreshaped_logitsr  r	  r  s   &&&&&&&&&&,        r%   r‰   ÚCanineForMultipleChoice.forwardC  s×  € ðX &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆØ,5Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆà>GÒ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÒG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ò(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —+‘+ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð   �
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÒÜ'Ó)ˆHÙ˜OÓ4ˆDçØ%Ð'¨'°"­+Õ5ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r(   )r¡  rø  rL   r  r  rŒ   s   @@r%   r  r  7  s,   ù‡ € õð ÷W
ñ W
ó ÷W
ð W
r(   r  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚCanineForTokenClassificationiž  c                ó2  <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R # r¿   rö  rš   s   &&€r%   r=   Ú%CanineForTokenClassification.__init__   rú  r(   c                ó\  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[R,          RS[S[,          /
# rü  )r    r|   r!   rÓ   r"   r   )r#   r$   s   "€r%   r&   Ú)CanineForTokenClassification.__annotate__¬  sÏ   ø€ ÷ O
ñ O
á×#Ñ# dÕ*ðO
ñ ×)Ñ)¨DÕ0ðO
ñ ×(Ñ(¨4Õ/ð	O
ñ
 ×&Ñ&¨Õ-ðO
ñ ×(Ñ(¨4Õ/ðO
ñ × Ñ  4Õ'ðO
ñ   $�;ðO
ñ # T�kðO
ñ ˜D•[ðO
ñ 
Ñ&Õ	&ñO
r(   c
                óÞ  € V	e   T	MV P                   P                  p	V P                  VVVVVVVV	R7      pV^ ,          pV P                  V4      pV P	                  V4      pRpVe=   \        4       pV! VP                  RV P                  4      VP                  R4      4      pV	'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  R7      # )a¸  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.

Example:

```python
>>> from transformers import AutoTokenizer, CanineForTokenClassification
>>> import torch

>>> tokenizer = AutoTokenizer.from_pretrained("google/canine-s")
>>> model = CanineForTokenClassification.from_pretrained("google/canine-s")

>>> inputs = tokenizer(
...     "HuggingFace is a company based in Paris and New York", add_special_tokens=False, return_tensors="pt"
... )

>>> with torch.no_grad():
...     logits = model(**inputs).logits

>>> predicted_token_class_ids = logits.argmax(-1)

>>> # Note that tokens are classified rather then input words which means that
>>> # there might be more predicted token classes than words.
>>> # Multiple token classes might account for the same word
>>> predicted_tokens_classes = [model.config.id2label[t.item()] for t in predicted_token_class_ids[0]]
>>> predicted_tokens_classes  # doctest: +SKIP
```

```python
>>> labels = predicted_token_class_ids
>>> loss = model(**inputs, labels=labels).loss
>>> round(loss.item(), 2)  # doctest: +SKIP
```Nr   r  r  r;   )r>   rc  r¡  rL   rø  r   rØ   r÷  r   r   r   )rR   rc   rÑ   ry   rQ   rz   rý  rÒ   rb  rc  rå  rï   r™  r  r  r	  r  s   &&&&&&&&&&,      r%   r‰   Ú$CanineForTokenClassification.forward«  sò   € ð` &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—+‘+ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð " !�*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÒÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDçØ�Y ¨¥Õ,ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r(   r  r  r  rŒ   s   @@r%   r  r  ž  s,   ù‡ € õ	ð ÷O
ñ O
ó ÷O
ð O
r(   r  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚCanineForQuestionAnsweringiþ  c                óè   <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R # r¿   )
r<   r=   r÷  r­  r¡  r   rÈ   r?   Ú
qa_outputsr¹  rš   s   &&€r%   r=   Ú#CanineForQuestionAnswering.__init__   sS   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &Ó)ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör(   c                ó„  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[R,          RS[R,          RS[S[,          /# )r   rc   NrÑ   ry   rQ   rz   Ústart_positionsÚend_positionsrÒ   rb  rc  r{   )r    r|   r!   rÓ   r"   r   )r#   r$   s   "€r%   r&   Ú'CanineForQuestionAnswering.__annotate__  sØ   ø€ ÷ =
ñ =
á×#Ñ# dÕ*ð=
ñ ×)Ñ)¨DÕ0ð=
ñ ×(Ñ(¨4Õ/ð	=
ñ
 ×&Ñ&¨Õ-ð=
ñ ×(Ñ(¨4Õ/ð=
ñ ×)Ñ)¨DÕ0ð=
ñ ×'Ñ'¨$Õ.ð=
ñ   $�;ð=
ñ # T�kð=
ñ ˜D•[ð=
ñ 
Ñ-Õ	-ñ=
r(   c                ó@  € V
e   T
MV P                   P                  p
V P                  VVVVVVV	V
R7      pV^ ,          pV P                  V4      pVP	                  ^RR7      w  ppVP                  R4      pVP                  R4      pR pVeÅ   VeÁ   \        VP                  4       4      ^8”  d   VP                  R4      p\        VP                  4       4      ^8”  d   VP                  R4      pVP                  ^4      pVP                  ^ V4       VP                  ^ V4       \        VR7      pV! Wö4      pV! VV4      pVV,           ^,          pV
'       g#   VV3VR,          ,           pVe   V3V,           # T# \        VVVVP                  VP                  R7      # )Nr   rm   )Úignore_indexr  )r  Ústart_logitsÚ
end_logitsr   r   r;   )r>   rc  r¡  r#  ÚsplitrÌ  r_   r‚   Úclamp_r   r   r   r   )rR   rc   rÑ   ry   rQ   rz   r&  r'  rÒ   rb  rc  rå  rï   r™  r  r+  r,  Ú
total_lossÚignored_indexr	  Ú
start_lossÚend_lossr  s   &&&&&&&&&&&,           r%   r‰   Ú"CanineForQuestionAnswering.forward
  s¥  € ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—+‘+ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð " !�*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/ˆØ×'Ñ'¨Ó+ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ×"Ñ" 1 mÔ4Ø× Ñ   MÔ2ä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÕ/°1Õ4ˆJçØ" JÐ/°'¸"µ+Õ=ˆFØ/9Ò/E�Z�M FÕ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r(   )r¡  r÷  r#  )
NNNNNNNNNNr  rŒ   s   @@r%   r!  r!  þ  s(   ù‡ € õð ÷=
ñ =
ó ÷=
ð =
r(   r!  )é   é+   é;   é=   éI   éa   ég   éq   é‰   é•   é�   é­   éµ   éÁ   éÓ   éß   )r  r!  rô  r  rG  r­  r   )?r.   r¯  rÚ   Údataclassesr   r    r   Útorch.nnr   r   r   Ú r   r¤  Úactivationsr   Úmasking_utilsr	   Úmodeling_layersr
   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   Úconfiguration_caniner   Ú
get_loggerr*   Úloggerr`   r   ÚModuler3   rŽ   r­   rÁ   rô   r  r0  r?  rG  rZ  rs  r~  r‡  r”  r   r­  rô  r  r  r!  Ú__all__r)   r(   r%   Ú<module>rS     sE  ðñ ã Û Ý !ã Ý ß AÑ Aå &Ý !Ý 6Ý 9÷÷ õ .Ý 6ß ,Ý .ð 
×	Ò	˜HÓ	%€ò U€ñ ðôð ô7 ;ó 7ó óð7ô:\�r—y‘yô \ô~)˜BŸI™Iô )ôX5�R—Y‘Yô 5ôpN˜"Ÿ)™)ô Nôb�r—y‘yô ô x�b—i‘iô xôv˜Ÿ™ô ô�2—9‘9ô ô5Ð,ô 5ôp=
�B—I‘Iô =
ô@�2—9‘9ô ô B§I¡Iô ô"˜RŸY™Yô ô&
!˜Ÿ	™	ô 
!ð ôi˜Oó ió ðið ôC
Ð'ó C
ó ðC
ñL ðôôQ
Ð&;ó Q
óðQ
ðh ôc
Ð3ó c
ó ðc
ðL ô\
Ð#8ó \
ó ð\
ð~ ôI
Ð!6ó I
ó ðI
òX‚r(   