+
    œ°³jáÑ  ã                   óª  € R t ^ RIt^ RIt^ RIHt ^ RIHtHtHt ^RIH	t
 ^RIHt ^RIHt ^RIHt ^RIHtHtHtHtHtHt ^R	IHt ^R
IHt ^RIHtHtHtH t H!t!H"t" ^RI#H$t$ ]"PJ                  ! ]&4      t'Rs(R t)R t*RDR lt+RDR lt,RDR lt-R t. ! R R]P^                  P`                  4      t1 ! R R]P^                  P`                  4      t2 ! R R4      t3RER lt4R t5RFR lt6 ! R R]Pn                  4      t8 ! R R]Pn                  4      t9 ! R  R!]Pn                  4      t: ! R" R#]Pn                  4      t; ! R$ R%]Pn                  4      t< ! R& R']Pn                  4      t= ! R( R)]4      t> ! R* R+]Pn                  4      t? ! R, R-]Pn                  4      t@ ! R. R/]Pn                  4      tA ! R0 R1]Pn                  4      tB] ! R2 R3]4      4       tC] ! R4 R5]C4      4       tD] ! R6 R7]C4      4       tE ! R8 R9]Pn                  4      tF]! R:R;7       ! R< R=]C4      4       tG] ! R> R?]C4      4       tH] ! R@ RA]C4      4       tI] ! RB RC]C4      4       tJ. RGOtKR# )HzPyTorch MRA model.N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úapply_chunking_to_forward)Úauto_docstringÚis_cuda_platformÚis_kernels_availableÚis_ninja_availableÚis_torch_cuda_availableÚlogging©Ú	MraConfigc                  ó\   € \        4       '       g   \        R 4      h^RIHp  V ! R^R7      sR# )zFkernels is not installed, please install it with `pip install kernels`©Ú
get_kernelzkernels-community/mra)ÚversionN)r   ÚImportErrorÚintegrations.hub_kernelsr   Úmra_cuda_kernelr   s    Úe/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/mra/modeling_mra.pyÚload_cuda_kernelsr"   4   s'   € ä×!Ò!ÜÐbÓcÐcÝ6á Ð!8À!ÔD‚Oó    c                óJ  € \        V P                  4       4      ^8w  d   \        R4      h\        VP                  4       4      ^8w  d   \        R4      hV P                  ^4      ^ 8w  d   \        R4      hV P                  ^4      ^ 8w  d   \        R4      hV P                  RR7      P                  P                  RR4      pVP                  4       pVP                  4       pVP                  4       p\        P                  WAW#4      w  rVVP                  RR4      R,          pWV3# )	z0
Computes maximum values for softmax stability.
z.sparse_qk_prod must be a 4-dimensional tensor.ú'indices must be a 2-dimensional tensor.z>The size of the second dimension of sparse_qk_prod must be 32.z=The size of the third dimension of sparse_qk_prod must be 32.©Údiméþÿÿÿéÿÿÿÿ©ºNNNr+   Nr+   )
ÚlenÚsizeÚ
ValueErrorÚmaxÚvaluesÚ	transposeÚ
contiguousÚintr    Ú	index_max)Úsparse_qk_prodÚindicesÚquery_num_blockÚkey_num_blockÚ
index_valsÚmax_valsÚmax_vals_scatters   &&&&   r!   Ú
sparse_maxr<   =   s  € ô ˆ>×ÑÓ Ó! QÔ&ÜÐIÓJÐJä
ˆ7�<‰<‹>Ó˜aÔÜÐBÓCÐCà×Ñ˜1Ó Ô#ÜÐYÓZÐZà×Ñ˜1Ó Ô#ÜÐXÓYÐYà×#Ñ#¨Ð#Ó+×2Ñ2×<Ñ<¸RÀÓD€JØ×&Ñ&Ó(€Jà�k‰k‹m€GØ× Ñ Ó"€Gä!0×!:Ñ!:¸:ÐP_Ó!oÑ€HØ'×1Ñ1°"°bÓ9¸-ÕHÐàÐ%Ð%r#   c                ó.  € \        V P                  4       4      ^8w  d   \        R4      h\        VP                  4       4      ^8w  d   \        R4      hV P                  ^ ,          VP                  ^ ,          8w  d   \        R4      hV P                  w  r4WB,          p\        P
                  ! VP                  ^ 4      \        P                  VP                  R7      pV P                  W5V4      p WR,          W,          P                  4       R3,          p V # )zF
Converts attention mask to a sparse mask for high resolution logits.
z$mask must be a 2-dimensional tensor.r%   zBmask and indices must have the same size in the zero-th dimension.©ÚdtypeÚdevicer+   ©r+   N)	r,   r-   r.   ÚshapeÚtorchÚarangeÚlongr@   Úreshape)Úmaskr6   Ú
block_sizeÚ
batch_sizeÚseq_lenÚ	num_blockÚ	batch_idxs   &&&    r!   Úsparse_maskrM   Y   sÒ   € ô ˆ4�9‰9‹;Ó˜1ÔÜÐ?Ó@Ð@ä
ˆ7�<‰<‹>Ó˜aÔÜÐBÓCÐCà‡z�z�!…}˜Ÿ™ aÕ(Ô(ÜÐ]Ó^Ð^àŸ*™*Ñ€JØÕ%€Iä—’˜WŸ\™\¨!›_´E·J±JÀwÇ~Á~ÔV€IØ�<‰<˜
¨zÓ:€DØ˜'Õ" WÕ%8×$>Ñ$>Ó$@À!ÐCÕD€Dà€Kr#   c                ó”  € V P                  4       w  rEpVP                  4       w  rxpWS,          ^ 8w  d   \        R4      hWƒ,          ^ 8w  d   \        R4      hV P                  WEV,          W64      P                  RR	4      p VP                  WHV,          W64      P                  RR	4      p\	        V P                  4       4      ^8w  d   \        R4      h\	        VP                  4       4      ^8w  d   \        R4      h\	        VP                  4       4      ^8w  d   \        R4      hV P                  ^4      ^ 8w  d   \        R4      hVP                  ^4      ^ 8w  d   \        R4      hV P                  4       p VP                  4       pVP                  4       pVP                  4       p\        P                  WVP                  4       4      # )
z/
Performs Sampled Dense Matrix Multiplication.
zTquery_size (size of first dimension of dense_query) must be divisible by block_size.úPkey_size (size of first dimension of dense_key) must be divisible by block_size.z+dense_query must be a 4-dimensional tensor.ú)dense_key must be a 4-dimensional tensor.r%   z.The third dimension of dense_query must be 32.z,The third dimension of dense_key must be 32.r)   r(   )	r-   r.   rF   r1   r,   r2   r3   r    Úmm_to_sparse)	Údense_queryÚ	dense_keyr6   rH   rI   Ú
query_sizer'   Ú_Úkey_sizes	   &&&&     r!   rQ   rQ   p   s�  € ð #.×"2Ñ"2Ó"4Ñ€J˜CØ —~‘~Ó'Ñ€A�àÕ !Ô#ÜÐoÓpÐpàÕ Ô!ÜÐkÓlÐlà×%Ñ% jÀ
Õ2JÈJÓ\×fÑfÐgiÐkmÓn€KØ×!Ñ! *¸*Õ.DÀjÓV×`Ñ`ÐacÐegÓh€Iä
ˆ;×ÑÓÓ !Ô#ÜÐFÓGÐGä
ˆ9�>‰>ÓÓ Ô!ÜÐDÓEÐEä
ˆ7�<‰<‹>Ó˜aÔÜÐBÓCÐCà×Ñ˜Ó˜bÔ ÜÐIÓJÐJà‡~�~�aÓ˜BÔÜÐGÓHÐHà×(Ñ(Ó*€KØ×$Ñ$Ó&€Ià�k‰k‹m€GØ× Ñ Ó"€Gä×'Ñ'¨ÀÇÁÃÓNÐNr#   c                óh  € VP                  4       w  rVpWd,          ^ 8w  d   \        R4      hV P                  ^4      V8w  d   \        R4      hV P                  ^4      V8w  d   \        R4      hVP                  WVV,          WG4      P                  RR	4      p\	        V P                  4       4      ^8w  d   \        R4      h\	        VP                  4       4      ^8w  d   \        R4      h\	        VP                  4       4      ^8w  d   \        R4      hVP                  ^4      ^ 8w  d   \        R4      hV P                  4       p VP                  4       pVP                  4       pVP                  4       p\        P                  WW#4      pVP                  RR	4      P                  WSV,          V4      pV# )
zH
Performs matrix multiplication of a sparse matrix with a dense matrix.
rO   zQThe size of the second dimension of sparse_query must be equal to the block_size.zPThe size of the third dimension of sparse_query must be equal to the block_size.ú,sparse_query must be a 4-dimensional tensor.rP   r%   z8The size of the third dimension of dense_key must be 32.r)   r(   )	r-   r.   rF   r1   r,   r2   r3   r    Úsparse_dense_mm)	Úsparse_queryr6   rS   r7   rH   rI   rV   r'   Údense_qk_prods	   &&&&&    r!   rY   rY   ˜   s�  € ð !*§¡Ó 0Ñ€J˜#àÕ Ô!ÜÐkÓlÐlà×Ñ˜Ó˜zÔ)ÜÐlÓmÐmà×Ñ˜Ó˜zÔ)ÜÐkÓlÐlà×!Ñ! *¸*Õ.DÀjÓV×`Ñ`ÐacÐegÓh€Iä
ˆ<×ÑÓÓ 1Ô$ÜÐGÓHÐHä
ˆ9�>‰>ÓÓ Ô!ÜÐDÓEÐEä
ˆ7�<‰<‹>Ó˜aÔÜÐBÓCÐCà‡~�~�aÓ˜BÔÜÐSÓTÐTà×*Ñ*Ó,€Là�k‰k‹m€GØ× Ñ Ó"€GØ×$Ñ$Ó&€Iä#×3Ñ3°LÈ9Óf€MØ!×+Ñ+¨B°Ó3×;Ñ;¸JÐZdÕHdÐfiÓj€MØÐr#   c                 óv   € W,          V,          \         P                  ! WR R7      ,           P                  4       # )Úfloor©Úrounding_mode)rC   ÚdivrE   )r6   Údim_1_blockÚdim_2_blocks   &&&r!   Útranspose_indicesrc   À   s*   € ØÕ" kÕ1´E·I²I¸gÐbiÔ4jÕj×pÑpÓrÐrr#   c                   óT   a € ] tR t^Ät o ]R 4       t]R 4       t]RR l4       tRtV t	R# )ÚMraSampledDenseMatMulc                óN   € \        WW44      pV P                  WV4       W@n        V# ©N)rQ   Úsave_for_backwardrH   )ÚctxrR   rS   r6   rH   r5   s   &&&&& r!   ÚforwardÚMraSampledDenseMatMul.forwardÅ   s)   € ä% k¸gÓRˆØ×Ñ˜k°gÔ>Ø#ŒØÐr#   c                ó  € V P                   w  r#pV P                  pVP                  ^4      V,          pVP                  ^4      V,          p\        WFV4      p\	        VP                  RR4      W‚V4      p	\	        WW64      p
W©RR3# ©é   Nr)   r(   )Úsaved_tensorsrH   r-   rc   rY   r1   )ri   ÚgradrR   rS   r6   rH   r7   r8   Ú	indices_TÚgrad_keyÚ
grad_querys   &&         r!   ÚbackwardÚMraSampledDenseMatMul.backwardÌ   s‚   € à*-×*;Ñ*;Ñ'ˆ Ø—^‘^ˆ
Ø%×*Ñ*¨1Ó-°Õ;ˆØ!Ÿ™ qÓ)¨ZÕ7ˆÜ% gÀÓNˆ	Ü" 4§>¡>°"°bÓ#9¸9ÐS`ÓaˆÜ$ T°IÓOˆ
Ø T¨4Ð/Ð/r#   c                ó.   € \         P                  WW#4      # rg   )re   Úapply)rR   rS   r6   rH   s   &&&&r!   Úoperator_callÚ#MraSampledDenseMatMul.operator_call×   s   € ä$×*Ñ*¨;À7ÓWÐWr#   © N©é    ©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ústaticmethodrj   rt   rx   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r!   re   re   Ä   sA   ø‡ € Øñó ðð ñ0ó ð0ð óXó öXr#   re   c                   óP   a € ] tR t^Üt o ]R 4       t]R 4       t]R 4       tRtV t	R# )ÚMraSparseDenseMatMulc                óN   € \        WW44      pV P                  WV4       W@n        V# rg   )rY   rh   r7   )ri   rZ   r6   rS   r7   r5   s   &&&&& r!   rj   ÚMraSparseDenseMatMul.forwardÝ   s*   € ä(¨À	Ó[ˆØ×Ñ˜l°YÔ?Ø-ÔØÐr#   c                óü   € V P                   w  r#pV P                  pVP                  ^4      VP                  R4      ,          p\        W5V4      p\	        VP                  RR4      WqV4      p\        WV4      p	V	RVR3# rm   )ro   r7   r-   rc   rY   r1   rQ   )
ri   rp   rZ   r6   rS   r7   r8   rq   rr   rs   s
   &&        r!   rt   ÚMraSparseDenseMatMul.backwardä   s~   € à+.×+<Ñ+<Ñ(ˆ˜yØ×-Ñ-ˆØ!Ÿ™ qÓ)¨\×->Ñ->¸rÓ-BÕBˆÜ% gÀÓNˆ	Ü" <×#9Ñ#9¸"¸bÓ#AÀ9ÐTaÓbˆÜ! $°7Ó;ˆ
Ø˜4 ¨4Ð/Ð/r#   c                ó.   € \         P                  WW#4      # rg   )rˆ   rw   )rZ   r6   rS   r7   s   &&&&r!   rx   Ú"MraSparseDenseMatMul.operator_callî   s   € ä#×)Ñ)¨,ÀÓ\Ð\r#   rz   Nr}   r…   s   @r!   rˆ   rˆ   Ü   sA   ø‡ € Øñó ðð ñ0ó ð0ð ñ]ó ö]r#   rˆ   c                   ó0   a € ] tR t^ót o ]R 4       tRtV tR# )ÚMraReduceSumc                ój  € V P                  4       w  rErg\        V P                  4       4      ^8w  d   \        R4      h\        VP                  4       4      ^8w  d   \        R4      hV P                  4       w   rvpVP                  4       w  rEV P                  ^R7      P	                  WE,          V4      p \
        P                  ! VP                  ^ 4      \
        P                  VP                  R7      p\
        P                  ! WRR7      P                  4       VR,          V,          ,           P	                  WE,          4      p	\
        P                  ! WB,          V3V P                  V P                  R7      p
V
P                  ^ W�4      P	                  WBV4      pVP	                  WBV,          4      pV# )é   rX   r%   r&   r>   r]   r^   rA   )r-   r,   r.   ÚsumrF   rC   rD   rE   r@   r`   Úzerosr?   Ú	index_add)rZ   r6   r7   r8   rI   rK   rH   rU   rL   Úglobal_idxesÚtempÚoutputs   &&&&        r!   rx   ÚMraReduceSum.operator_callô   s\  € à/;×/@Ñ/@Ó/BÑ,ˆ
˜zäˆ|× Ñ Ó"Ó# qÔ(ÜÐKÓLÐLäˆw�|‰|‹~Ó !Ô#ÜÐFÓGÐGà*×/Ñ/Ó1Ñˆˆ1˜!Ø '§¡£Ñˆ
à#×'Ñ'¨AÐ'Ó.×6Ñ6°zÕ7MÈzÓZˆä—L’L §¡¨a£¼¿
¹
È7Ï>É>ÔZˆ	ä�IŠI�g¸GÔD×IÑIÓKÈiÐX_ÕN`ÐcrÕNrÕrß
‰'�*Õ(Ó
)ð 	ô �{Š{ØÕ)¨:Ð6¸l×>PÑ>PÐYe×YlÑYlô
ˆð —‘  <Ó>×FÑFÀzÐdnÓoˆà—‘ 
¸jÕ,HÓIˆØˆr#   rz   N)r~   r   r€   r�   r‚   rx   rƒ   r„   r…   s   @r!   r�   r�   ó   s   ø‡ € Øñó ör#   r�   c                óX  € V P                  4       w  rVpWb,          pRp	VeÍ   VP                  WXV4      P                  R	R7      p
V P                  WXW'4      P                  R
R7      V
R,          R,           ,          pVP                  WXW'4      P                  R
R7      V
R,          R,           ,          pVe8   VP                  WXW'4      P                  R
R7      V
R,          R,           ,          p	M¢V\        P                  ! WX\        P
                  V P                  R7      ,          p
V P                  WXW'4      P                  R
R7      pVP                  WXW'4      P                  R
R7      pVe#   VP                  WXW'4      P                  R
R7      p	\        P                  ! W¼P                  R	R
4      4      \        P                  ! V4      ,          pVP                  R	RR7      P                  pVe7   VRV
R,          V
R,          ,          R8  P                  4       ,          ,
          pWÚWé3# )z'
Compute low resolution approximation.
Nr&   ç�íµ ÷Æ°>r>   T)r'   Úkeepdimsç     ˆÃ@g      à?r)   r(   ©r+   r+   N©r+   Nr+   )r-   rF   r“   rC   ÚonesÚfloatr@   ÚmeanÚmatmulr1   ÚmathÚsqrtr/   r0   )ÚqueryÚkeyrH   rG   ÚvaluerI   rJ   Úhead_dimÚnum_block_per_rowÚ	value_hatÚtoken_countÚ	query_hatÚkey_hatÚlow_resolution_logitÚlow_resolution_logit_row_maxs   &&&&&          r!   Úget_low_resolution_logitr±     sþ  € ð %*§J¡J£LÑ!€J˜àÕ-Ðà€IØÒØ—l‘l :À*ÓM×QÑQÐVXÐQÓYˆØ—M‘M *ÀÓV×ZÑZÐ_aÐZÓbØ˜
Õ# dÕ*õ
ˆ	ð —+‘+˜j¸ZÓR×VÑVÐ[]ÐVÓ^Ø˜
Õ# dÕ*õ
ˆð ÒØŸ™ jÀZÓZ×^Ñ^ÐceÐ^ÓfØ˜JÕ'¨$Õ.õˆIøð !¤5§:¢:¨jÔSX×S^ÑS^Ðgl×gsÑgsÔ#tÕtˆØ—M‘M *ÀÓV×[Ñ[Ð`bÐ[Ócˆ	Ø—+‘+˜j¸ZÓR×WÑWÐ\^ÐWÓ_ˆØÒØŸ™ jÀZÓZ×_Ñ_ÐdfÐ_ÓgˆIä Ÿ<š<¨	×3DÑ3DÀRÈÓ3LÓMÔPT×PYÒPYÐZbÓPcÕcÐà#7×#;Ñ#;ÀÈTÐ#;Ó#R×#YÑ#YÐ àÒà  3¨;°zÕ+BÀ[ÐQ[ÕE\Õ+\Ð`cÑ*c×)jÑ)jÓ)lÕ#lÕlð 	ð  Ð.JÐUÐUr#   c                óÂ  € V P                   w  rVpV^ 8”  dp   V^,          p\        P                  ! WfV P                  R7      p	\        P                  ! \        P
                  ! W˜) R7      VR7      p
W
R,          R,          ,           p V^ 8”  d;   V RRV1R3,          R,           V RRV1R3&   V RRRV13,          R,           V RRRV13&   \        P                  ! V P                  VR4      VRRRR7      pVP                  pVR	8X  dC   VP                  P                  RR
7      P                  pWR,          8¬  P                  4       pWÎ3# VR8X  d   RpWÎ3# \        V R24      h)zR
Compute the indices of the subset of components to be used in the approximation.
©r@   )ÚdiagonalNr+   g     ˆ³@TF)r'   ÚlargestÚsortedÚfullr&   Úsparsez# is not a valid approx_model value.)Nr+   r+   r)   )r+   NN)rB   rC   r    r@   ÚtrilÚtriuÚtopkrF   r6   r0   Úminr¡   r.   )r¯   Ú
num_blocksÚapprox_modeÚinitial_prior_first_n_blocksÚinitial_prior_diagonal_n_blocksrI   Útotal_blocks_per_rowrU   ÚoffsetÚ	temp_maskÚdiagonal_maskÚ
top_k_valsr6   Ú	thresholdÚhigh_resolution_masks   &&&&&          r!   Úget_block_idxesrÈ   8  sŽ  € ð +?×*DÑ*DÑ'€J aà&¨Ô*Ø0°AÕ5ˆÜ—J’JÐ3ÐRf×RmÑRmÔnˆ	ÜŸ
š
¤5§:¢:¨iÀ'Ô#JÐU[Ô\ˆØ3ÀJÕ6OÐRUÕ6UÕUÐà# aÔ'à  Ð$AÐ%AÐ$AÀ1Ð!DÕEÈÕKð 	˜QÐ =Ð!=Ð =¸qÐ@ÑAð !  AÐ'DÐ(DÐ'DÐ!DÕEÈÕKð 	˜Q Ð#@Ð$@Ð#@Ð@ÑAô —’Ø×$Ñ$ Z°Ó4°jÀbÐRVÐ_dô€Jð × Ñ €Gà�fÔØ×%Ñ%×)Ñ)¨bÐ)Ó1×8Ñ8ˆ	Ø 4À-Õ8PÑ P×WÑWÓYÐð Ð(Ð(ð 
˜Ô	 Ø#Ðð Ð(Ð(ô ˜K˜=Ð(KÐLÓMÐMr#   c	           	     ó¬  € \         f%   \        P                  ! V 4      P                  4       # V P	                  4       w  ršr¼Wš,          pW¶,          ^ 8w  d   \        R4      hW¶,          pV P                  WÛV4      p VP                  WÛV4      pVP                  WÛV4      pVe.   WR,          ,          p WR,          ,          pW#R,          ,          pVR8X  d   \        WWcV4      w  ppppMJVR8X  d9   \        P                  ! 4       ;_uu_ 4        \        WWc4      w  ppppRRR4       M\        R4      h\        P                  ! 4       ;_uu_ 4        XX,
          p\        VVVVV4      w  ppRRR4       \        P                  WXVR7      \        P                  ! V4      ,          p\        VVWî4      w  ppVV,
          pVe)   VR^\!        VV4      R,          ,
          ,          ,
          p\        P"                  ! V4      p\$        P                  VVW.4      p\&        P                  VVWî4      pVR8X  Ed¥   \        P"                  ! XX,
          RX,          ,
          4      XR,          ,          p\        P(                  ! VX4      R,          P+                  ^^V^4      P                  WÛV4      pVP-                  RR7      R,          P+                  ^^V4      P                  WÛ4      pVP+                  ^^V4      P                  WÛ4      V,
          p Ve
   V V,          p \        P"                  ! V V ^ 8*  P/                  4       ,          4      p!VV!R,          ,          pVV!,          p\        P"                  ! V ) V ^ 8„  P/                  4       ,          4      p"VV"R,          ,          pVV",          pVV,           VR,          VR,          ,           R	,           ,          p#M*VR8X  d   VVR,          R	,           ,          p#M\        R
4      hVe   V#VR,          ,          p#V#P                  WšW¼4      p#V##   + '       g   i     EL; i  + '       g   i     ELß; i)z(
Use Mra to approximate self-attention.
Nz4sequence length must be divisible by the block_size.r·   r¸   z&approx_mode must be "full" or "sparse")rH   r�   r&   r›   z-config.approx_mode must be "full" or "sparse"rž   )r+   r+   r+   NrŸ   r*   r)   )r    rC   Ú
zeros_likeÚrequires_grad_r-   r.   rF   r±   Úno_gradÚ	ExceptionrÈ   re   rx   r¤   r¥   r<   rM   Úexprˆ   r�   r£   Úrepeatr“   r¡   )$r¦   r§   r¨   rG   r½   r¾   rH   r¿   rÀ   rI   Únum_headrJ   r©   Ú
meta_batchrª   r¯   r¬   r°   r«   rU   Úlow_resolution_logit_normalizedr6   rÇ   Úhigh_resolution_logitr:   r;   Úhigh_resolution_attnÚhigh_resolution_attn_outÚhigh_resolution_normalizerÚlow_resolution_attnÚlow_resolution_attn_outÚlow_resolution_normalizerÚlog_correctionÚlow_resolution_corrÚhigh_resolution_corrÚcontext_layers$   &&&&&&&&&                           r!   Úmra2_attentionrÞ   ^  s  € ô ÒÜ×Ò Ó&×5Ñ5Ó7Ð7à.3¯j©j«lÑ+€J˜'ØÕ&€JàÕ˜qÔ ÜÐOÓPÐPàÕ-Ðà�M‰M˜*¨xÓ8€EØ
�+‰+�j¨8Ó
4€CØ�M‰M˜*¨xÓ8€EàÒØ˜ZÕ(Õ(ˆØ˜Õ$Õ$ˆØ˜ZÕ(Õ(ˆà�fÔÜUmØ˜
¨%óV
ÑRÐ˜kÐ+GÉð 
˜Ô	 Ü�]Š]�_�_ÜQiØ˜JóRÑNÐ  +Ð/KÈQ÷ ˆ_ô
 Ð@ÓAÐAä	�Š��Ø*>ÐA]Õ*]Ð'Ü(7Ø+ØØØ(Ø+ó)
Ñ%ˆÐ%÷ 
ô 2×?Ñ?Ø�G¨
ð @ó ä�	Š	�(ÓõÐô ",Ð,AÀ7ÐL]Ó!qÑ€HÐØ1Ð4DÕDÐØÒØ 5¸¸qÄ;ÈtÐU\ÓC]Ð^kÕClÕ?lÕ8mÕ mÐÜ Ÿ9š9Ð%:Ó;ÐÜ3×AÑAØ˜g uó Ðô ".×!;Ñ!;Ø˜gÐ'8ó"Ðð �fÕä�IŠIÐ*Ð-IÕIÈCÐRfÕLfÕfÓgØ˜*Õ%õ&ð 	ô �LŠLÐ,¨iÓ8¸ÕGß‰V�A�q˜* aÓ(ß‰W�Z¨(Ó3ð 	 ð  ×#Ñ#¨Ð#Ó+¨JÕ7×>Ñ>¸qÀ!ÀZÓP×XÑXÐYcÓmð 	"ð 6×<Ñ<¸QÀÀ:ÓN×VÑVÐWaÓkÐnvÕvˆØÒØ+¨dÕ2ˆNä#Ÿiši¨¸.ÈAÑ:M×9TÑ9TÓ9VÕ(VÓWÐØ"9Ð<OÐPZÕ<[Õ"[ÐØ$=Ð@SÕ$SÐ!ä$Ÿyšy¨.¨¸NÈQÑ<N×;UÑ;UÓ;WÕ)WÓXÐØ#;Ð>RÐS]Õ>^Õ#^Ð Ø%?ÐBVÕ%VÐ"à1Ð4KÕKØ& zÕ2Ð5NÈzÕ5ZÕZÐ]aÕaõ
‰ð 
˜Ô	 Ø0Ð4NÈzÕ4ZÐ]aÕ4aÕb‰äÐGÓHÐHàÒØ%¨¨ZÕ(8Õ8ˆà!×)Ñ)¨*ÀÓR€MàÐ÷S �_ˆ_ú÷ 
�ˆús   ÄP.ÅQÐ.P?	ÑQ	c                   ó@   a a€ ] tR tRt oRtV 3R ltRR ltRtVtV ;t	# )ÚMraEmbeddingsiÑ  zGConstruct the embeddings from word, position and token_type embeddings.c                ó�  <€ \         SV `  4        \        P                  ! VP                  VP
                  VP                  R 7      V n        \        P                  ! VP                  ^,           VP
                  4      V n	        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R7      V n        \        P                  ! VP                  4      V n        \        P"                  ! \$        P&                  ! VP                  4      P)                  R4      ^,           4      V n        \        P"                  ! \$        P,                  ! V P*                  P/                  4       \$        P0                  V P*                  P2                  R7      RR7      V n        R# ))Úpadding_idx©Úepsr>   F)Ú
persistentN©rn   r)   )ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚBufferrC   rD   ÚexpandÚposition_idsr”   r-   rE   r@   Útoken_type_ids©ÚselfÚconfigÚ	__class__s   &&€r!   rè   ÚMraEmbeddings.__init__Ô  s&  ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¢<°×0NÑ0NÐQRÕ0RÐTZ×TfÑTfÓ#gˆÔ Ü%'§\¢\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒô ŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐefÕ&fÓgˆÔÜ ŸišiÜ�KŠK˜×)Ñ)×.Ñ.Ó0¼¿
¹
È4×K\ÑK\×KcÑKcÔdÐqvô
ˆÖr#   c                óx  € Ve   VP                  4       pMVP                  4       R R pV^,          pVf   V P                  RR V13,          pVf�   \        V R4      '       d4   V P                  RR V13,          pVP	                  V^ ,          V4      pTpM;\
        P                  ! V\
        P                  V P                  P                  R7      pVf   V P                  V4      pV P                  V4      p	WI,           p
V P                  V4      pW«,          p
V P                  V
4      p
V P                  V
4      p
V
# )Nr+   rú   r>   r)   )r-   rù   Úhasattrrú   rø   rC   r”   rE   r@   rí   rñ   rï   rò   rö   )rü   Ú	input_idsrú   rù   Úinputs_embedsÚinput_shapeÚ
seq_lengthÚbuffered_token_type_idsÚ buffered_token_type_ids_expandedrñ   Ú
embeddingsrï   s   &&&&&       r!   rj   ÚMraEmbeddings.forwardã  s,  € ØÒ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  •^ˆ
àÒØ×,Ñ,¨Q°°°¨^Õ<ˆLð
 Ò!Ü�tÐ-×.Ò.Ø*.×*=Ñ*=¸aÀÀ*À¸nÕ*MÐ'Ø3J×3QÑ3QÐR]Ð^_ÕR`ÐblÓ3mÐ0Ø!A‘ä!&§¢¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÒ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐØ"Õ:ˆ
à"×6Ñ6°|ÓDÐØÕ)ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr#   )rò   rö   rï   rù   rñ   rú   rí   )NNNN©
r~   r   r€   r�   Ú__doc__rè   rj   rƒ   r„   Ú__classcell__©rþ   r†   s   @@r!   rà   rà   Ñ  s   ù‡ € ÙQõ
÷ ô  r#   rà   c                   ó<   a a€ ] tR tRt oV 3R ltRR ltRtVtV ;t# )ÚMraSelfAttentioni  c                ó&  <€ \         SV `  4        VP                  VP                  ,          ^ 8w  d9   \	        VR4      '       g'   \        RVP                   RVP                   R24      h\        RJp\        4       '       d4   \        4       '       d$   \        4       '       d   V'       g    \        4        VP                  V n        \        VP                  VP                  ,          4      V n        V P                  V P                  ,          V n        \"        P$                  ! VP                  V P                   4      V n        \"        P$                  ! VP                  V P                   4      V n        \"        P$                  ! VP                  V P                   4      V n        \"        P,                  ! VP.                  4      V n        VP2                  ^ ,          VP4                  ,          V n        \9        V P6                  \        VP2                  ^ ,          ^,          4      4      V n        VP:                  V n        VP<                  V n        VP>                  V n        R#   \         d$   p\        P                  RT 24        Rp?ELÖRp?ii ; i)é    Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (Ú)NzGCould not load the custom kernel for multi-scale deformable attention: ) rç   rè   rë   Únum_attention_headsr  r.   r    r   r   r   r"   rÍ   ÚloggerÚwarningr3   Úattention_head_sizeÚall_head_sizer   ÚLinearr¦   r§   r¨   rô   Úattention_probs_dropout_probrö   rî   Úblock_per_rowrK   r¼   r¾   r¿   rÀ   )rü   rý   Úkernel_loadedÚerþ   s   &&  €r!   rè   ÚMraSelfAttention.__init__  sò  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Õ:¸aÔ?ÌÐPVÐXh×HiÒHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ô
 (¨tÐ3ˆÜ"×$Ò$Ô)9×);Ò);Ô@R×@TÒ@T×]jðnÜ!Ô#ð $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÕ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÕPˆÔä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z’z &×"EÑ"EÓFˆŒà ×8Ñ8¸BÕ>À&×BVÑBVÕVˆŒÜ˜TŸ^™^¬S°&×2PÑ2PÐTVÕ2VÐ[\Õ1\Ó-]Ó^ˆŒà!×-Ñ-ˆÔØ,2×,OÑ,OˆÔ)Ø/5×/UÑ/UˆÖ,øô% ô nÜ—‘Ð!hÐijÐhkÐl×mÒmûðnús   Â+
I" É"JÉ-JÊJc                ó”  € VP                   w  r4pV P                  V4      P                  VRV P                  V P                  4      P                  ^^4      pV P                  V4      P                  VRV P                  V P                  4      P                  ^^4      pV P                  V4      P                  VRV P                  V P                  4      P                  ^^4      pRVR,          ,           pVP                  4       P                  ^V P                  ^4      P                  W0P                  ,          V4      P                  4       p^ p	V P                  V	8  dË   W0P                  WIV P                  ,
          3p
\        P                  ! V\        P                  ! W¦P                  R7      .RR7      p\        P                  ! V\        P                  ! W§P                  R7      .RR7      p\        P                  ! V\        P                  ! W¨P                  R7      .RR7      p\!        VP#                  4       VP#                  4       VP#                  4       VP#                  4       V P$                  V P&                  V P(                  V P*                  R7      pV P                  V	8  d   VRRRRV P                  13,          pVP                  W0P                  W@P                  4      pVP-                  ^ ^^^4      P/                  4       pVP1                  4       RR	 V P2                  3,           pVP                  ! V!  pV3pV# )
rn   g      ð?r�   r³   r&   )r¾   r¿   rÀ   r+   Nr)   r(   )rB   r¦   Úviewr  r  r1   r§   r¨   ÚsqueezerÏ   rF   r3   rC   Úcatr”   r@   rÞ   r¡   rK   r¾   r¿   rÀ   Úpermuter2   r-   r  )rü   Úhidden_statesÚattention_maskrI   rJ   rU   Úquery_layerÚ	key_layerÚvalue_layerÚgpu_warp_sizeÚpad_sizerÝ   Únew_context_layer_shapeÚoutputss   &&&           r!   rj   ÚMraSelfAttention.forward'  s¹  € Ø!.×!4Ñ!4Ñˆ
˜Qà�J‰J�}Ó%ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �H‰H�]Ó#ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �J‰J�}Ó%ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð ˜~°Õ7Õ7ˆà×"Ñ"Ó$ß‰V�A�t×/Ñ/°Ó3ß‰W�Z×":Ñ":Õ:¸GÓDß‰S‹Uð	 	ð ˆà×#Ñ# mÔ3Ø!×#;Ñ#;¸WÐVZ×VnÑVnÕFnÐnˆHäŸ)š) [´%·+²+¸h×OaÑOaÔ2bÐ$cÐikÔlˆKÜŸ	š	 9¬e¯kªk¸(×K[ÑK[Ô.\Ð"]ÐceÔfˆIÜŸ)š) [´%·+²+¸h×OaÑOaÔ2bÐ$cÐikÔlˆKä&Ø×ÑÓØ�O‰OÓØ×ÑÓØ× Ñ Ó"Ø�N‰NØ×(Ñ(Ø)-×)JÑ)JØ,0×,PÑ,Pô	
ˆð ×#Ñ# mÔ3Ø)¨!¨Q°Ð3M°T×5MÑ5MÐ3MÐ*MÕNˆMà%×-Ñ-¨j×:RÑ:RÐT[×]uÑ]uÓvˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÕ"SÐØ%×*Ò*Ð,CÑDˆà Ð"ˆàˆr#   )r  r¾   r  rö   rÀ   r¿   r§   r  rK   r¦   r¨   rg   ©	r~   r   r€   r�   rè   rj   rƒ   r„   r  r  s   @@r!   r  r    s   ù‡ € õV÷@<ô <r#   r  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMraSelfOutputig  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  4      V n
        R# ©rã   N)rç   rè   r   r  rë   Údenserò   ró   rô   rõ   rö   rû   s   &&€r!   rè   ÚMraSelfOutput.__init__h  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr#   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# ©é   r$  Úinput_tensorÚreturn©rC   ÚTensor)Úformatr†   s   "€r!   Ú__annotate__ÚMraSelfOutput.__annotate__n  ó1   ø€ ÷ ñ ¡U§\¡\ð ÁÇÁð ÑRW×R^ÑR^ñ r#   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# rg   ©r3  rö   rò   ©rü   r$  r8  s   &&&r!   rj   ÚMraSelfOutput.forwardn  ó5   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }Õ'CÓDˆØÐr#   ©rò   r3  rö   r.  r  s   @@r!   r0  r0  g  ó   ù‡ € õ>÷÷ ð r#   r0  c                   ó<   a a€ ] tR tRt oV 3R ltRR ltRtVtV ;t# )ÚMraAttentioniu  c                ód   <€ \         SV `  4        \        V4      V n        \	        V4      V n        R # rg   )rç   rè   r  rü   r0  r˜   rû   s   &&€r!   rè   ÚMraAttention.__init__v  s&   ø€ Ü‰ÑÔÜ$ VÓ,ˆŒ	Ü# FÓ+ˆŽr#   c                ó|   € V P                  W4      pV P                  V^ ,          V4      pV3VR,          ,           pV# ©r  ºrn   NN)rü   r˜   )rü   r$  r%  Úself_outputsÚattention_outputr,  s   &&&   r!   rj   ÚMraAttention.forward{  s<   € Ø—y‘y Ó?ˆØŸ;™; |°A¥¸ÓFÐØ#Ð%¨°RÕ(8Õ8ˆØˆr#   )r˜   rü   rg   r.  r  s   @@r!   rH  rH  u  s   ù‡ € õ,÷
ô r#   rH  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMraIntermediateiƒ  c                ó"  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n        R # VP                  V n        R # rg   )rç   rè   r   r  rë   Úintermediate_sizer3  Ú
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnrû   s   &&€r!   rè   ÚMraIntermediate.__init__„  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬×-Ò-Ü'-¨f×.?Ñ.?Õ'@ˆDÖ$à'-×'8Ñ'8ˆDÖ$r#   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r7  r$  r9  r:  )r<  r†   s   "€r!   r=  ÚMraIntermediate.__annotate__Œ  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r#   c                óJ   € V P                  V4      pV P                  V4      pV# rg   ©r3  rX  ©rü   r$  s   &&r!   rj   ÚMraIntermediate.forwardŒ  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr#   r^  r.  r  s   @@r!   rR  rR  ƒ  s   ù‡ € õ9÷÷ ð r#   rR  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )Ú	MraOutputi“  c                ó0  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        P                  ! VP
                  VP                  R 7      V n        \        P                  ! VP                  4      V n        R# r2  )rç   rè   r   r  rT  rë   r3  rò   ró   rô   rõ   rö   rû   s   &&€r!   rè   ÚMraOutput.__init__”  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŽr#   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# r6  r:  )r<  r†   s   "€r!   r=  ÚMraOutput.__annotate__š  r?  r#   c                óx   € V P                  V4      pV P                  V4      pV P                  W,           4      pV# rg   rA  rB  s   &&&r!   rj   ÚMraOutput.forwardš  rD  r#   rE  r.  r  s   @@r!   rb  rb  “  rF  r#   rb  c                   óB   a a€ ] tR tRt oV 3R ltRR ltR tRtVtV ;t	# )ÚMraLayeri¡  c                óÖ   <€ \         SV `  4        VP                  V n        ^V n        \	        V4      V n        VP                  V n        \        V4      V n        \        V4      V n
        R# ©rn   N)rç   rè   Úchunk_size_feed_forwardÚseq_len_dimrH  Ú	attentionÚadd_cross_attentionrR  Úintermediaterb  r˜   rû   s   &&€r!   rè   ÚMraLayer.__init__¢  sW   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ% fÓ-ˆŒØ#)×#=Ñ#=ˆÔ Ü+¨FÓ3ˆÔÜ Ó'ˆŽr#   c                ó¸   € V P                  W4      pV^ ,          pVR,          p\        V P                  V P                  V P                  V4      pV3V,           pV# rL  )ro  r   Úfeed_forward_chunkrm  rn  )rü   r$  r%  Úself_attention_outputsrO  r,  Úlayer_outputs   &&&    r!   rj   ÚMraLayer.forward«  s_   € Ø!%§¡°Ó!NÐØ1°!Õ4Ðà(¨Õ,ˆä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð  �/ GÕ+ˆàˆr#   c                óJ   € V P                  V4      pV P                  W!4      pV# rg   )rq  r˜   )rü   rO  Úintermediate_outputrv  s   &&  r!   rt  ÚMraLayer.feed_forward_chunk¸  s)   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6ÓIˆØÐr#   )rp  ro  rm  rq  r˜   rn  rg   )
r~   r   r€   r�   rè   rj   rt  rƒ   r„   r  r  s   @@r!   rj  rj  ¡  s   ù‡ € õ(ô÷ò r#   rj  c                   ó<   a a€ ] tR tRt oV 3R ltRR ltRtVtV ;t# )Ú
MraEncoderi¾  c                óÖ   <€ \         SV `  4        Wn        \        P                  ! \        VP                  4       Uu. uF  p\        V4      NK  	  up4      V n        R V n	        R# u upi )FN)
rç   rè   rý   r   Ú
ModuleListÚrangeÚnum_hidden_layersrj  ÚlayerÚgradient_checkpointing)rü   rý   rU   rþ   s   && €r!   rè   ÚMraEncoder.__init__¿  sR   ø€ Ü‰ÑÔØŒÜ—]’]¼eÀF×D\ÑD\Ô>]Ó#^Ñ>]¸¤H¨VÖ$4Ñ>]Ñ#^Ó_ˆŒ
Ø&+ˆÖ#ùò $_s   ½A&c                óB  € V'       d   RMR p\        V P                  4       F'  w  rgV'       d
   WQ3,           pV! W4      pV^ ,          pK)  	  V'       d
   WQ3,           pV'       g/   \        ;QJ d    . R W3 4       F  NK  	  5# ! R W3 4       4      # \        VVR7      # )Nc              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5irg   rz   )Ú.0Úvs   & r!   Ú	<genexpr>Ú%MraEncoder.forward.<locals>.<genexpr>Ú  s   é € ÐXÑ$F˜qŸšÓ$Fùs   ‚Œ
)Úlast_hidden_stater$  rz   )Ú	enumerater�  Útupler
   )	rü   r$  r%  Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚiÚlayer_moduleÚlayer_outputss	   &&&&&    r!   rj   ÚMraEncoder.forwardÅ  s•   € ÷ #7™B¸DÐä(¨¯©Ö4‰OˆAß#Ø$5Ð8HÕ$HÐ!á(¨ÓGˆMà)¨!Õ,ŠMñ  5÷  Ø 1Ð4DÕ DÐçß”5ÑX ]Ñ$FÓX—5ÐX�5ÑX ]Ñ$FÓXÓXÐXÜ1Ø+Ø+ô
ð 	
r#   )rý   r‚  r�  )NFTr.  r  s   @@r!   r|  r|  ¾  s   ù‡ € õ,÷
ô 
r#   r|  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMraPredictionHeadTransformiâ  c                ó‚  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        VP                  \        4      '       d   \        VP                  ,          V n
        MVP                  V n
        \        P                  ! VP                  VP                  R 7      V n        R# r2  )rç   rè   r   r  rë   r3  rU  rV  rW  r   Útransform_act_fnrò   ró   rû   s   &&€r!   rè   Ú#MraPredictionHeadTransform.__init__ã  s~   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬×-Ò-Ü$*¨6×+<Ñ+<Õ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸš f×&8Ñ&8¸f×>SÑ>SÔTˆŽr#   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r[  r:  )r<  r†   s   "€r!   r=  Ú'MraPredictionHeadTransform.__annotate__ì  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ r#   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# rg   )r3  r—  rò   r_  s   &&r!   rj   Ú"MraPredictionHeadTransform.forwardì  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐr#   )rò   r3  r—  r.  r  s   @@r!   r•  r•  â  s   ù‡ € õU÷÷ ð r#   r•  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚMraLMPredictionHeadiô  c                ó  <€ \         SV `  4        \        V4      V n        \        P
                  ! VP                  VP                  R R7      V n        \        P                  ! \        P                  ! VP                  4      4      V n        R# )T)ÚbiasN)rç   rè   r•  Ú	transformr   r  rë   rê   ÚdecoderÚ	ParameterrC   r”   r   rû   s   &&€r!   rè   ÚMraLMPredictionHead.__init__õ  s[   ø€ Ü‰ÑÔÜ3°FÓ;ˆŒô —y’y ×!3Ñ!3°V×5FÑ5FÈTÔRˆŒÜ—L’L¤§¢¨V×->Ñ->Ó!?Ó@ˆŽ	r#   c                óJ   € V P                  V4      pV P                  V4      pV# rg   )r¡  r¢  r_  s   &&r!   rj   ÚMraLMPredictionHead.forwardþ  s$   € ØŸ™ }Ó5ˆØŸ™ ]Ó3ˆØÐr#   )r   r¢  r¡  r.  r  s   @@r!   rž  rž  ô  s   ù‡ € õA÷ò r#   rž  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚMraOnlyMLMHeadi  c                óD   <€ \         SV `  4        \        V4      V n        R # rg   )rç   rè   rž  Úpredictionsrû   s   &&€r!   rè   ÚMraOnlyMLMHead.__init__  s   ø€ Ü‰ÑÔÜ.¨vÓ6ˆÖr#   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r7  Úsequence_outputr9  r:  )r<  r†   s   "€r!   r=  ÚMraOnlyMLMHead.__annotate__
  s#   ø€ ÷ !ñ !¡u§|¡|ð !¹¿¹ñ !r#   c                ó(   € V P                  V4      pV# rg   ©rª  )rü   r­  Úprediction_scoress   && r!   rj   ÚMraOnlyMLMHead.forward
  s   € Ø ×,Ñ,¨_Ó=ÐØ Ð r#   r°  r.  r  s   @@r!   r¨  r¨    s   ù‡ € õ7÷!÷ !ð !r#   r¨  c                   óz   a a€ ] tR tRt oRtRt]P                  ! 4       V3R lV 3R ll4       tV3R lt	Rt
VtV ;t# )ÚMraPreTrainedModeli  ÚmraTc                ó4   <€ V ^8„  d   QhRS[ P                  /# )r7  Úmodule)r   ÚModule)r<  r†   s   "€r!   r=  ÚMraPreTrainedModel.__annotate__  s   ø€ ÷ /ñ /¡B§I¡Iñ /r#   c                óÒ  <€ \         SV `  V4       \        V\        4      '       d#   \        P
                  ! VP                  4       R# \        V\        4      '       d‰   \        P                  ! VP                  \        P                  ! VP                  P                  R,          4      P                  R4      ^,           4       \        P
                  ! VP                  4       R# R# )zInitialize the weightsNr)   ræ   )rç   Ú_init_weightsrU  rž  ÚinitÚzeros_r   rà   Úcopy_rù   rC   rD   rB   rø   rú   )rü   r·  rþ   s   &&€r!   r»  Ú MraPreTrainedModel._init_weights  s•   ø€ ô 	‰Ñ˜fÔ%Ü�fÔ1×2Ò2Ü�KŠK˜Ÿ™Ö$Ü˜¤×.Ò.Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÕ9VÓ,W×,^Ñ,^Ð_fÓ,gÐjkÕ,kÔlÜ�KŠK˜×-Ñ-Ö.ñ /r#   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # )r7  rý   r   )r<  r†   s   "€r!   r=  r¹    s   ø‡ ‚ ñ Ñò r#   rz   )r~   r   r€   r�   Úbase_model_prefixÚsupports_gradient_checkpointingrC   rÌ   r»  Ú__annotate_func__rƒ   r„   r  r  s   @@r!   r´  r´    s7   ù‡ € ð ÐØ&*Ð#à
‡]‚]ƒ_÷/ò /ó ð/÷ … r#   r´  c                   ó^   a a€ ] tR tRt oV 3R ltR tR t]RV3R lR ll4       tRt	Vt
V ;t# )	ÚMraModeli!  c                ó’   <€ \         SV `  V4       Wn        \        V4      V n        \        V4      V n        V P                  4        R # rg   )rç   rè   rý   rà   r  r|  ÚencoderÚ	post_initrû   s   &&€r!   rè   ÚMraModel.__init__#  s9   ø€ Ü‰Ñ˜Ô ØŒä'¨Ó/ˆŒÜ! &Ó)ˆŒð 	�‰Ör#   c                ó.   € V P                   P                  # rg   ©r  rí   ©rü   s   &r!   Úget_input_embeddingsÚMraModel.get_input_embeddings-  s   € Ø�‰×.Ñ.Ð.r#   c                ó&   € WP                   n        R # rg   rË  )rü   r¨   s   &&r!   Úset_input_embeddingsÚMraModel.set_input_embeddings0  s   € Ø*/�‰Ö'r#   c                ó   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[S[,          /# )
r7  r  Nr%  rú   rù   r  r�  rŽ  r9  )rC   r;  ÚboolrŒ  r
   )r<  r†   s   "€r!   r=  ÚMraModel.__annotate__4  s    ø€ ÷ G
ñ G
á—<‘< $Õ&ðG
ñ Ÿ™ tÕ+ðG
ñ Ÿ™ tÕ+ð	G
ñ
 —l‘l TÕ)ðG
ñ —|‘| dÕ*ðG
ñ # T�kðG
ñ ˜D•[ðG
ñ 
Ñ3Õ	3ñG
r#   c                óÎ  € Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   Ve   \        R4      hVe#   V P	                  W4       VP                  4       p	M#Ve   VP                  4       R R p	M\        R4      hV	w  r«Ve   VP                  MVP                  pVf   \        P                  ! W«3VR7      pVfy   \        V P                  R4      '       d6   V P                  P                  RR V13,          pVP                  W«4      pTpM'\        P                  ! V	\        P                  VR7      pV P                  VVVVR7      p\        V P                   VR,          VRR	7      pV P!                  VVVVR
7      pV^ ,          pV'       g   V3VR,          ,           # \#        VVP$                  VP&                  VP(                  R7      # )NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsr³   rú   r+   r>   )r  rù   rú   r  F)rý   r  r%  Úallow_is_bidirectional_skip)r%  r�  rŽ  rM  )rŠ  r$  Ú
attentionsÚcross_attentionsr)   )r+   :r  rn   Nr+   )rý   r�  rŽ  r.   Ú%warn_if_padding_and_no_attention_maskr-   r@   rC   r    r  r  rú   rø   r”   rE   r   rÇ  r
   r$  r×  rØ  )rü   r  r%  rú   rù   r  r�  rŽ  Úkwargsr  rI   r  r@   r  r  Úembedding_outputÚencoder_outputsr­  s   &&&&&&&&,         r!   rj   ÚMraModel.forward3  sß  € ð %9Ò$DÑ È$Ï+É+×JjÑJjð 	ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆàÒ  ]Ò%>ÜÐcÓdÐdØÒ"Ø×6Ñ6°yÔQØ#Ÿ.™.Ó*‰KØÒ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
Ø%.Ò%:�×!Ò!À×@TÑ@TˆàÒ!Ü"ŸZšZ¨*Ð)AÈ6ÔRˆNàÒ!Ü�t—‘Ð(8×9Ò9Ø*.¯/©/×*HÑ*HÈÈKÈZÈKÈÕ*XÐ'Ø3J×3QÑ3QÐR\Ó3iÐ0Ø!A‘ä!&§¢¨[ÄÇ
Á
ÐSYÔ!Z�àŸ?™?ØØ%Ø)Ø'ð	 +ó 
Ðô 3Ø—;‘;Ø*¨9Õ5Ø)à(-ô
ˆð Ÿ,™,ØØ)Ø!5Ø#ð	 'ó 
ˆð *¨!Õ,ˆçØ#Ð%¨¸Õ(;Õ;Ð;ä1Ø-Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ô	
ð 	
r#   )rý   r  rÇ  )NNNNNNN)r~   r   r€   r�   rè   rÍ  rÐ  r   rj   rƒ   r„   r  r  s   @@r!   rÅ  rÅ  !  s6   ù‡ € õò/ò0ð ÷G
ñ G
ó ÷G
ð G
r#   rÅ  c                   ój   a a€ ] tR tRt oRRRR/tV 3R ltR tR t]RV3R	 lR
 ll4       t	Rt
VtV ;t# )ÚMraForMaskedLMi~  zcls.predictions.decoder.biaszcls.predictions.biaszcls.predictions.decoder.weightz%mra.embeddings.word_embeddings.weightc                ó†   <€ \         SV `  V4       \        V4      V n        \	        V4      V n        V P                  4        R # rg   )rç   rè   rÅ  rµ  r¨  ÚclsrÈ  rû   s   &&€r!   rè   ÚMraForMaskedLM.__init__…  s4   ø€ Ü‰Ñ˜Ô ä˜FÓ#ˆŒÜ! &Ó)ˆŒð 	�‰Ör#   c                óB   € V P                   P                  P                  # rg   )rá  rª  r¢  rÌ  s   &r!   Úget_output_embeddingsÚ$MraForMaskedLM.get_output_embeddingsŽ  s   € Ø�x‰x×#Ñ#×+Ñ+Ð+r#   c                ó„   € WP                   P                  n        VP                  V P                   P                  n        R # rg   )rá  rª  r¢  r   )rü   Únew_embeddingss   &&r!   Úset_output_embeddingsÚ$MraForMaskedLM.set_output_embeddings‘  s*   € Ø'5�‰×ÑÔ$Ø$2×$7Ñ$7ˆ�‰×ÑÖ!r#   c                óH  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[,          /	# ©r7  r  Nr%  rú   rù   r  Úlabelsr�  rŽ  r9  )rC   r;  rÓ  rŒ  r   )r<  r†   s   "€r!   r=  ÚMraForMaskedLM.__annotate__–  s¨   ø€ ÷ /
ñ /
á—<‘< $Õ&ð/
ñ Ÿ™ tÕ+ð/
ñ Ÿ™ tÕ+ð	/
ñ
 —l‘l TÕ)ð/
ñ —|‘| dÕ*ð/
ñ —‘˜tÕ#ð/
ñ # T�kð/
ñ ˜D•[ð/
ñ 
‘Õ	ñ/
r#   c	           
     óÎ  € Ve   TMV P                   P                  pV P                  VVVVVVVR7      p
V
^ ,          pV P                  V4      pRpVeG   \	        4       pV! VP                  RV P                   P                  4      VP                  R4      4      pV'       g"   V3V
R,          ,           pVe   V3V,           # T# \        VVV
P                  V
P                  R7      # )a{  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
N©r%  rú   rù   r  r�  rŽ  rM  ©ÚlossÚlogitsr$  r×  r)   )
rý   rŽ  rµ  rá  r   r   rê   r   r$  r×  )rü   r  r%  rú   rù   r  rì  r�  rŽ  rÚ  r,  r­  r±  Úmasked_lm_lossÚloss_fctr˜   s   &&&&&&&&&,      r!   rj   ÚMraForMaskedLM.forward•  sò   € ð& &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—(‘(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ó 
ˆð " !�*ˆØ ŸH™H _Ó5ÐàˆØÒÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNçØ'Ð)¨G°B­KÕ7ˆFØ3AÒ3M�^Ð%¨Õ.ÐYÐSYÐYäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r#   )rá  rµ  ©NNNNNNNN)r~   r   r€   r�   Ú_tied_weights_keysrè   rä  rè  r   rj   rƒ   r„   r  r  s   @@r!   rß  rß  ~  sH   ù‡ € ð 	'Ð(>Ø(Ð*QðÐõ
ò,ò8ð ÷/
ñ /
ó ÷/
ð /
r#   rß  c                   ó<   a a€ ] tR tRt oRtV 3R ltR tRtVtV ;t	# )ÚMraClassificationHeadiÉ  z-Head for sentence-level classification tasks.c                ó:  <€ \         SV `  4        \        P                  ! VP                  VP                  4      V n        \        P                  ! VP                  4      V n        \        P                  ! VP                  VP                  4      V n
        Wn        R # rg   )rç   rè   r   r  rë   r3  rô   rõ   rö   Ú
num_labelsÚout_projrý   rû   s   &&€r!   rè   ÚMraClassificationHead.__init__Ì  se   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŒÜŸ	š	 &×"4Ñ"4°f×6GÑ6GÓHˆŒàŽr#   c                óî   € VR,          pV P                  V4      pV P                  V4      p\        V P                  P                  ,          ! V4      pV P                  V4      pV P                  V4      pV# )r+   )r+   r  r+   )rö   r3  r   rý   rV  rü  )rü   ÚfeaturesrÚ  Úxs   &&, r!   rj   ÚMraClassificationHead.forwardÔ  s_   € Ø�WÕˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆÜ�4—;‘;×)Ñ)Ö*¨1Ó-ˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆr#   )rý   r3  rö   rü  r
  r  s   @@r!   rù  rù  É  s   ù‡ € Ù7õ÷ò r#   rù  z›
    MRA Model transformer with a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks.
    )Úcustom_introc                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚMraForSequenceClassificationiÞ  c                ó¨   <€ \         SV `  V4       VP                  V n        \        V4      V n        \        V4      V n        V P                  4        R # rg   )rç   rè   rû  rÅ  rµ  rù  Ú
classifierrÈ  rû   s   &&€r!   rè   Ú%MraForSequenceClassification.__init__å  sA   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ˜FÓ#ˆŒÜ/°Ó7ˆŒð 	�‰Ör#   c                óH  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[,          /	# rë  )rC   r;  rÓ  rŒ  r   )r<  r†   s   "€r!   r=  Ú)MraForSequenceClassification.__annotate__ï  s´   ø€ ÷ @
ñ @
á—<‘< $Õ&ð@
ñ Ÿ™ tÕ+ð@
ñ Ÿ™ tÕ+ð	@
ñ
 —l‘l TÕ)ð@
ñ —|‘| dÕ*ð@
ñ —‘˜tÕ#ð@
ñ # T�kð@
ñ ˜D•[ð@
ñ 
Ñ)Õ	)ñ@
r#   c	           
     ór  € Ve   TMV P                   P                  pV P                  VVVVVVVR7      p
V
^ ,          pV P                  V4      pRpVEe˜   V P                   P                  f–   V P
                  ^8X  d   RV P                   n        MrV P
                  ^8”  dQ   VP                  \        P                  8X  g    VP                  \        P                  8X  d   RV P                   n        MRV P                   n        V P                   P                  R8X  dK   \        4       pV P
                  ^8X  d'   V! VP                  4       VP                  4       4      pMŽV! WÆ4      pM…V P                   P                  R8X  d>   \        4       pV! VP                  RV P
                  4      VP                  R4      4      pM-V P                   P                  R8X  d   \        4       pV! WÆ4      pV'       g"   V3V
R,          ,           pVe   V3V,           # T# \        VVV
P                   V
P"                  R7      # )	ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Nrï  Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrM  rð  r)   )rý   rŽ  rµ  r  Úproblem_typerû  r?   rC   rE   r3   r   r!  r   r   r   r   r$  r×  )rü   r  r%  rú   rù   r  rì  r�  rŽ  rÚ  r,  r­  rò  rñ  rô  r˜   s   &&&&&&&&&,      r!   rj   Ú$MraForSequenceClassification.forwardî  sÎ  € ð& &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—(‘(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ó 
ˆð " !�*ˆØ—‘ Ó1ˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù Ó/�ßØ�Y ¨¥Õ,ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r#   )r  rµ  rû  rö  ©
r~   r   r€   r�   rè   r   rj   rƒ   r„   r  r  s   @@r!   r  r  Þ  s,   ù‡ € õð ÷@
ñ @
ó ÷@
ð @
r#   r  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚMraForMultipleChoicei2  c                ó  <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  VP                  4      V n        \        P
                  ! VP                  ^4      V n        V P                  4        R# rl  )
rç   rè   rÅ  rµ  r   r  rë   Úpre_classifierr  rÈ  rû   s   &&€r!   rè   ÚMraForMultipleChoice.__init__4  s_   ø€ Ü‰Ñ˜Ô ä˜FÓ#ˆŒÜ Ÿiši¨×(:Ñ(:¸F×<NÑ<NÓOˆÔÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰Ör#   c                óH  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[,          /	# rë  )rC   r;  rÓ  rŒ  r   )r<  r†   s   "€r!   r=  Ú!MraForMultipleChoice.__annotate__?  s´   ø€ ÷ W
ñ W
á—<‘< $Õ&ðW
ñ Ÿ™ tÕ+ðW
ñ Ÿ™ tÕ+ð	W
ñ
 —l‘l TÕ)ðW
ñ —|‘| dÕ*ðW
ñ —‘˜tÕ#ðW
ñ # T�kðW
ñ ˜D•[ðW
ñ 
Ñ*Õ	*ñW
r#   c	           
     óî  € Ve   TMV P                   P                  pVe   VP                  ^,          MVP                  ^,          p
Ve"   VP                  RVP	                  R4      4      MRpVe"   VP                  RVP	                  R4      4      MRpVe"   VP                  RVP	                  R4      4      MRpVe"   VP                  RVP	                  R4      4      MRpVe2   VP                  RVP	                  R4      VP	                  R4      4      MRpV P                  VVVVVVVR7      pV^ ,          pVR,          pV P                  V4      p\        P                  ! 4       ! V4      pV P                  V4      pVP                  RV
4      pRpVe   \        4       pV! Wö4      pV'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  R7      # )a›  
input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
    1]`:

    - 0 corresponds to a *sentence A* token,
    - 1 corresponds to a *sentence B* token.

    [What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
    config.max_position_embeddings - 1]`.

    [What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
    model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
    `input_ids` above)
Nrï  rM  rð  r)   r(   )r+   r  )rý   rŽ  rB   r   r-   rµ  r  r   ÚReLUr  r   r   r$  r×  )rü   r  r%  rú   rù   r  rì  r�  rŽ  rÚ  Únum_choicesr,  Úhidden_stateÚpooled_outputrò  Úreshaped_logitsrñ  rô  r˜   s   &&&&&&&&&,         r!   rj   ÚMraForMultipleChoice.forward>  sð  € ðV &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆØ,5Ò,A�i—o‘o aÖ(À}×GZÑGZÐ[\ÕG]ˆà>GÒ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÒMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÒG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ò(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —(‘(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ó 
ˆð ˜q•zˆØ$ TÕ*ˆØ×+Ñ+¨MÓ:ˆÜŸšœ	 -Ó0ˆØ—‘ Ó/ˆà Ÿ+™+ b¨+Ó6ˆàˆØÒÜ'Ó)ˆHÙ˜OÓ4ˆDçØ%Ð'¨'°"­+Õ5ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r#   )r  rµ  r  rö  r  r  s   @@r!   r  r  2  s,   ù‡ € õð ÷W
ñ W
ó ÷W
ð W
r#   r  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚMraForTokenClassificationi™  c                ó2  <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R # rg   )rç   rè   rû  rÅ  rµ  r   rô   rõ   rö   r  rë   r  rÈ  rû   s   &&€r!   rè   Ú"MraForTokenClassification.__init__›  si   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä˜FÓ#ˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör#   c                óH  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[R,          R
S[S[,          /	# rë  )rC   r;  rÓ  rŒ  r   )r<  r†   s   "€r!   r=  Ú&MraForTokenClassification.__annotate__§  s©   ø€ ÷ 8
ñ 8
á—<‘< $Õ&ð8
ñ Ÿ™ tÕ+ð8
ñ Ÿ™ tÕ+ð	8
ñ
 —l‘l TÕ)ð8
ñ —|‘| dÕ*ð8
ñ —‘˜tÕ#ð8
ñ # T�kð8
ñ ˜D•[ð8
ñ 
Ñ&Õ	&ñ8
r#   c	           
     óþ  € Ve   TMV P                   P                  pV P                  VVVVVVVR7      p
V
^ ,          pV P                  V4      pV P	                  V4      pRpVeÎ   \        4       pVeŽ   VP                  R4      ^8H  pVP                  RV P                  4      p\        P                  ! WöP                  R4      \        P                  ! VP                  4      P                  V4      4      pV! VV4      pM2V! VP                  RV P                  4      VP                  R4      4      pV'       g"   V3V
R,          ,           pVe   V3V,           # T# \        VVV
P                  V
P                  R7      # )zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
Nrï  rM  rð  r)   )rý   rŽ  rµ  rö   r  r   r   rû  rC   ÚwhereÚtensorÚignore_indexÚtype_asr   r$  r×  )rü   r  r%  rú   rù   r  rì  r�  rŽ  rÚ  r,  r­  rò  rñ  rô  Úactive_lossÚactive_logitsÚactive_labelsr˜   s   &&&&&&&&&,         r!   rj   Ú!MraForTokenClassification.forward¦  sa  € ð" &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—(‘(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ó 
ˆð " !�*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÒÜ'Ó)ˆHàÒ)Ø,×1Ñ1°"Ó5¸Ñ:�Ø &§¡¨B°·±Ó @�Ü %§¢Ø§¡¨R£´%·,²,¸x×?TÑ?TÓ2U×2]Ñ2]Ð^dÓ2eó!�ñ   ¨}Ó=‘á §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR�çØ�Y ¨¥Õ,ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r#   )r  rö   rµ  rû  rö  r  r  s   @@r!   r   r   ™  s(   ù‡ € õ	ð ÷8
ñ 8
ó ÷8
ð 8
r#   r   c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚMraForQuestionAnsweringiâ  c                óö   <€ \         SV `  V4       ^Vn        VP                  V n        \        V4      V n        \
        P                  ! VP                  VP                  4      V n        V P                  4        R# )r7  N)
rç   rè   rû  rÅ  rµ  r   r  rë   Ú
qa_outputsrÈ  rû   s   &&€r!   rè   Ú MraForQuestionAnswering.__init__ä  s[   ø€ Ü‰Ñ˜Ô àˆÔØ ×+Ñ+ˆŒä˜FÓ#ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör#   c                óp  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          R	S[R,          R
S[R,          RS[S[,          /
# )r7  r  Nr%  rú   rù   r  Ústart_positionsÚend_positionsr�  rŽ  r9  )rC   r;  rÓ  rŒ  r   )r<  r†   s   "€r!   r=  Ú$MraForQuestionAnswering.__annotate__ñ  s¼   ø€ ÷ ;
ñ ;
á—<‘< $Õ&ð;
ñ Ÿ™ tÕ+ð;
ñ Ÿ™ tÕ+ð	;
ñ
 —l‘l TÕ)ð;
ñ —|‘| dÕ*ð;
ñ Ÿ™¨Õ,ð;
ñ —|‘| dÕ*ð;
ñ # T�kð;
ñ ˜D•[ð;
ñ 
Ñ-Õ	-ñ;
r#   c
           
     ó8  € V	e   T	MV P                   P                  p	V P                  VVVVVVV	R7      pV^ ,          pV P                  V4      pVP	                  ^RR7      w  rïVP                  R4      pVP                  R4      pR pVeÄ   VeÀ   \        VP                  4       4      ^8”  d   VP                  R4      p\        VP                  4       4      ^8”  d   VP                  R4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR7      pV! Wæ4      pV! W÷4      pVV,           ^,          pV	'       g"   Wï3VR,          ,           pVe   V3V,           # T# \        VVVVP                  VP                  R7      # )Nrï  r&   )r(  rM  )rñ  Ústart_logitsÚ
end_logitsr$  r×  r)   )rý   rŽ  rµ  r1  Úsplitr!  r,   r-   Úclampr   r   r$  r×  )rü   r  r%  rú   rù   r  r4  r5  r�  rŽ  rÚ  r,  r­  rò  r8  r9  Ú
total_lossÚignored_indexrô  Ú
start_lossÚend_lossr˜   s   &&&&&&&&&&,           r!   rj   ÚMraForQuestionAnswering.forwardð  s   € ð &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà—(‘(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ó 
ˆð " !�*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/ˆØ×'Ñ'¨Ó+ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÕ/°1Õ4ˆJçØ"Ð/°'¸"µ+Õ=ˆFØ/9Ò/E�Z�M FÕ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r#   )rµ  rû  r1  )	NNNNNNNNNr  r  s   @@r!   r/  r/  â  s(   ù‡ € õ
ð ÷;
ñ ;
ó ÷;
ð ;
r#   r/  r{   )NN)r|   r  r  )rß  r  r/  r  r   rj  rÅ  r´  )Lr  r¤   rC   r   Útorch.nnr   r   r   Ú r   r¼  Úactivationsr   Úmasking_utilsr   Úmodeling_layersr	   Úmodeling_outputsr
   r   r   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   r   r   Úconfiguration_mrar   Ú
get_loggerr~   r  r    r"   r<   rM   rQ   rY   rc   ÚautogradÚFunctionre   rˆ   r�   r±   rÈ   rÞ   r¸  rà   r  r0  rH  rR  rb  rj  r|  r•  rž  r¨  r´  rÅ  rß  rù  r  r  r   r/  Ú__all__rz   r#   r!   Ú<module>rO     s|  ðñ ã ã Ý ß AÑ Aå &Ý !Ý 6Ý 9÷÷ õ .Ý 6÷÷ õ )ð 
×	Ò	˜HÓ	%€à€òEò&ô8ô.%OôP%òPsôX˜EŸN™N×3Ñ3ô Xô0]˜5Ÿ>™>×2Ñ2ô ]÷.ñ ô:%VòP#)ôLpôf2�B—I‘Iô 2ôj]�r—y‘yô ]ôB�B—I‘Iô ô
�2—9‘9ô 
ô�b—i‘iô ô �—	‘	ô ôÐ)ô ô: 
�—‘ô  
ôH §¡ô ô$˜"Ÿ)™)ô ô"!�R—Y‘Yô !ð ô/˜ó /ó ð/ð  ôY
Ð!ó Y
ó ðY
ðx ôF
Ð'ó F
ó ðF
ôT˜BŸI™Iô ñ* ðôôK
Ð#5ó K
óðK
ð\ ôc
Ð-ó c
ó ðc
ðL ôE
Ð 2ó E
ó ðE
ðP ôI
Ð0ó I
ó ðI
òX	‚r#   