+ jֺ"Rt^RIt^RIt^RIHt^RIHtHtHt^RIH t ^RI H t ^RI HtHtHt^RIHt^RIHt^R IHt^R IHtHtHtHtHtHtHtHt^R I H!t!^R I"H#t#^R I$H%t%H&t&^RI'H(t(]&PR!]*4t+!RR]PX4t-!RR]PX4t.!RR]PX4t/!RR]PX4t0!RR]PX4t1!RR]PX4t2!RR]PX4t3!RR]4t4!RR ]PX4t5!R!R"]PX4t6!R#R$]PX4t7!R%R&]PX4t8]%!R'R(]!44t9]%!R)R*7!R+R,]944t:]%!R-R.]944t;]%!R/R*7!R0R1]9]44t<]%!R2R*7!R3R4]944t=]%!R5R6]944t>]%!R7R8]944t?]%!R9R:]944t@.R;OtAR#) J": "66|D) ^^J/ \\*- rA)r1r5r.r:r0r,)NNNN) __name__ __module__ __qualname____firstlineno____doc__r'r\__static_attributes____classdictcell__ __classcell__r>rNs@@r?rr/sQ vrArcDaa]tRt^btoV3RltV3RlRltRtVtV;t#) RemBertPoolerc<\SV`4\P!VPVP4Vn\P !4VnR#N)r&r'rLinear hidden_sizedenseTanh activationr;s&&r?r'RemBertPooler.__init__cs9 YYv1163E3EF '')rAcN<V^8dQhRS[PRS[P/#rC hidden_statesrHr7rL)rMrNs"r?rORemBertPooler.__annotate__hs#U\\ellrAc\VR,pVPV4pVPV4pV#)rR)rRr^)rnrp)r<rtfirst_token_tensor pooled_outputs&& r?r\RemBertPooler.forwardhs2+40 #56  6 rA)rprn r_r`rarbr'r\rdrerfrgs@@r?riribs$ rAricLaa]tRt^qtoRV3RlltRV3RlRlltRtVtV;t#)RemBertSelfAttentionc,<\SV`4VPVP,^8wd9\ VR4'g'\ RVP RVP R24hVPVn\ VPVP, 4VnVPVP,Vn\P!VPVP4Vn \P!VPVP4Vn \P!VPVP4Vn \P!VP4VnVP"VnW nR#)r^embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()N)r&r'rmnum_attention_headshasattr ValueErrorrKattention_head_size all_head_sizerrlquerykeyvaluer3attention_probs_dropout_probr5 is_decoder layer_idxr<r=rr>s&&&r?r'RemBertSelfAttention.__init__rs1     : : :a ?PVXhHiHi#F$6$6#78 445Q8  $*#=#= #&v'9'9F  &/+>??,77;;DNNK %+:+P+P(+:+O+O('6$2D.- /"=*,33DNNCHHI.55dnnELLKQ--cr2QBQ8P8PQH055h?II!QOI**^499(CMMaQRSK*)=)D)DY]a]k]k)l& %*_FY*Z*ZAEO..t~~>!<< 5H5HR5PQ+dii8P8P.QQ  %/.@ --//0@b/I,,7 _kB %--aAq9DDF "/"4"4"6s";t?Q?Q>S"S%**,CD o--rA) rrr5rrrrrrrkNNNFr{rgs@@r?r}r}qs#*@.@.@.rAr}cDaa]tRt^toV3RltV3RlRltRtVtV;t#)RemBertSelfOutputc0<\SV`4\P!VPVP4Vn\P !VPVPR7Vn\P!VP4Vn R#r N) r&r'rrlrmrnr1r2r3r4r5r;s&&r?r'RemBertSelfOutput.__init__s` YYv1163E3EF f&8&8f>S>STzz&"<"<= rAch<V^8dQhRS[PRS[PRS[P/#rCrt input_tensorrHru)rMrNs"r?rORemBertSelfOutput.__annotate__1U\\RWR^R^rAcxVPV4pVPV4pVPW,4pV#rkrnr5r1r<rtrs&&&r?r\RemBertSelfOutput.forward5 =1  ]3 }'CD rAr1rnr5r{rgs@@r?rr> rArcLaa]tRt^toRV3RlltRV3RlRlltRtVtV;t#)RemBertAttentioncf<\SV`4\WR7Vn\ V4VnR#)rN)r&r'r}r<routputrs&&&r?r'RemBertAttention.__init__s& (E '/ rAc <V^8dQhRS[PRS[PR,RS[PR,RS[R,RS[R,RS[S[P,/#rr)rMrNs"r?rORemBertAttention.__annotate__sr||))D0 %0047     $;  u|| rAc VPVVVVVR7pVPV^,V4pV3VR,,p V #)rrrrr$NN)r<r) r<rtrrrrr self_outputsattention_outputoutputss &&&&&&, r?r\RemBertAttention.forwardsTyy )"7+/ !  ;;|A F#% R(88rA)rr<rkrr{rgs@@r?rrs0 rArcDaa]tRt^toV3RltV3RlRltRtVtV;t#)RemBertIntermediatec"<\SV`4\P!VPVP 4Vn\VP\4'd\VP,Vn R#VPVn R#rk) r&r'rrlrmintermediate_sizernr hidden_actstrrintermediate_act_fnr;s&&r?r'RemBertIntermediate.__init__s` YYv1163K3KL f'' - -'-f.?.?'@D $'-'8'8D $rAcN<V^8dQhRS[PRS[P/#rsru)rMrNs"r?rO RemBertIntermediate.__annotate__s#U\\ellrAcJVPV4pVPV4pV#rkrnrr<rts&&r?r\RemBertIntermediate.forwards& =1 00? rArr{rgs@@r?rrs9rArcDaa]tRtRtoV3RltV3RlRltRtVtV;t#) RemBertOutputic0<\SV`4\P!VPVP 4Vn\P!VP VPR7Vn\P!VP4Vn R#r) r&r'rrlrrmrnr1r2r3r4r5r;s&&r?r'RemBertOutput.__init__s` YYv779K9KL f&8&8f>S>STzz&"<"<= rAch<V^8dQhRS[PRS[PRS[P/#rru)rMrNs"r?rORemBertOutput.__annotate__ rrAcxVPV4pVPV4pVPW,4pV#rkrrs&&&r?r\RemBertOutput.forward rrArr{rgs@@r?rrrrArcRaa]tRtRtoRV3RlltRV3RlRlltRtRtVtV;t #) RemBertLayeric~<\SV`4VPVn^Vn\ W4VnVP VnVPVnVP'd2VP 'g\V R24h\ WR7Vn \V4Vn \V4Vn R#)r$z> should be used as a decoder model if cross attention is addedrN)r&r'chunk_size_feed_forward seq_len_dimr attentionradd_cross_attentionrcrossattentionr intermediaterrrs&&&r?r'RemBertLayer.__init__s '-'E'E$)&< ++#)#=#=  # # #??? D6)g!hii"26"OD /7#F+ rAc<V^8dQhRS[PRS[PR,RS[PR,RS[PR,RS[R,RS[R,RS[S[P,/#) rCrtrNrencoder_attention_maskrrrHr)rMrNs"r?rORemBertLayer.__annotate__#s))||)))D0) %0047 ) !& 1 1D 8 )  ) $;) u|| )rAc VPVVVVR7pV^,p VR,p VP'dTVeP\VR4'g\RV R24hVP V VVVVR7p V ^,p WR,,p \ VP VPVPV 4p V 3V ,p V #))rrrrrz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r) rrrrrrfeed_forward_chunkrr) r<rtrrrrrrself_attention_outputsrrcross_attention_outputs layer_outputs &&&&&&&, r?r\RemBertLayer.forward#s"& )/+ "0"  2!4(, ???4@4!122 =dVDDD '+&9&9 5&; /"3 ':' # 7q9  ;;G0  # #T%A%A4CSCSUe  /G+rAcJVPV4pVPW!4pV#rk)rr)r<rintermediate_outputrs&& r?rRemBertLayer.feed_forward_chunkOs)"//0@A{{#6I rA)rrrrrrrrrk)NNNNF) r_r`rarbr'r\rrdrerfrgs@@r?rrs ,))XrArcHaa]tRtRtoV3RltRV3RlRlltRtVtV;t#)RemBertEncoderiUc 8<\SV`4Wn\P!VP VP 4Vn\P!\VP4Uu.uFp\WR7NK up4Vn RVn R#uupi)rFN)r&r'r=rrlr*rmembedding_hidden_mapping_in ModuleListrangenum_hidden_layersrlayergradient_checkpointing)r<r=ir>s&& r?r'RemBertEncoder.__init__Vss  +-99V5P5PRXRdRd+e(]]uU[UmUmOn#oOn!L$EOn#op &+#$ps-Bc<V^8dQhRS[PRS[PR,RS[PR,RS[PR,RS[R,RS[R,RS[R S[R S[R S[S[,/ #) rCrtrNrrr use_cacheroutput_hidden_states return_dictrH)r7rLrJrrrr)rMrNs"r?rORemBertEncoder.__annotate__^sD D ||D ))D0D  %0047 D !& 1 1D 8 D  D $;D  D #D D  : :D rAc VP'd2VP'd V'd\PR4RpV'd9Vf5\ \ VP R7\ VP R74pVPV4pV'dRMRp V'dRMRp V'dVP P'dRMRp \VP4FvwrV'd W3,p V!VVVVVV4pV^,pV'gK6V V^,3,p VP P'gKeV V^,3,p Kx V'd W3,p V 'g7\;QJd.RVVV V V 34FNK 5#!RVVV V V 344#\VVV V V R7#)zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FN)r=c32"TF pVfK VxK R#5irk).0vs& r? )RemBertEncoder.forward..s" As )last_hidden_staterrt attentionscross_attentionsr) rtraininglogger warning_oncer r r=r r enumeraterrr)r<rtrrrrrrrrrall_hidden_statesall_self_attentionsall_cross_attentionsr layer_module layer_outputss&&&&&&&&&&, r?r\RemBertEncoder.forward^s  & & &4===##p" 01,dkk2RT`hlhshsTtuO88G "6BD$5b4%64;;;Z;Z;Zr`d(4OA#$58H$H!(%&! M*!,M  &9]1=MS>STrAcN<V^8dQhRS[PRS[P/#rsru)rMrNs"r?rO+RemBertPredictionHeadTransform.__annotate__s#U\\ellrAclVPV4pVPV4pVPV4pV#rk)rnr1r1rs&&r?r\&RemBertPredictionHeadTransform.forwards4 =1 --m< }5 rA)r1rnr1r{rgs@@r?r/r/sUrAr/cDaa]tRtRtoV3RltV3RlRltRtVtV;t#)RemBertLMPredictionHeadic~<\SV`4\P!VPVP 4Vn\P!VP VP4Vn\VP,Vn \P!VP VPR7Vn R#r)r&r'rrlrmoutput_embedding_sizernr)decoderrrrpr1r2r;s&&r?r' RemBertLMPredictionHead.__init__sz YYv1163O3OP yy!=!=v?P?PQ  !2!23f&B&BH]H]^rAcN<V^8dQhRS[PRS[P/#rsru)rMrNs"r?rO$RemBertLMPredictionHead.__annotate__s#U\\ellrAcVPV4pVPV4pVPV4pVPV4pV#rk)rnrpr1r;rs&&r?r\RemBertLMPredictionHead.forwards@ =1  6 }5  ]3 rA)r1rpr;rnr{rgs@@r?r8r8s_rAr8cDaa]tRtRtoV3RltV3RlRltRtVtV;t#)RemBertOnlyMLMHeadicD<\SV`4\V4VnR#rk)r&r'r8 predictionsr;s&&r?r'RemBertOnlyMLMHead.__init__s 26:rAcN<V^8dQhRS[PRS[P/#)rCsequence_outputrHru)rMrNs"r?rORemBertOnlyMLMHead.__annotate__s#!!u||! !rAc(VPV4pV#rkrD)r<rGprediction_scoress&& r?r\RemBertOnlyMLMHead.forwards ,,_=  rArJr{rgs@@r?rBrBs;!!!rArBcFaa]tRtRtoRtRtV3RltV3RltRtVt V;t #)RemBertPreTrainedModelirembertTc<\SV`V4\V\4'db\P !VP \P!VP PR,4PR44R#R#)r$Nr%r#) r& _init_weightsrrinitcopy_r:r7r8rr9)r<moduler>s&&r?rQ$RemBertPreTrainedModel._init_weightss^ f% f/ 0 0 JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h 1rAc&<V^8dQh/S[;R&#)rCr=r)rMrNs"r?rO#RemBertPreTrainedModel.__annotate__s rAr) r_r`rarbbase_model_prefixsupports_gradient_checkpointingrQ__annotate_func__rdrerfrgs@@r?rNrNs "&*#i rArNa  The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. ) custom_introcbaa]tRtRtoRV3RlltRtRt]R V3RlRll4tRt Vt V;t #) RemBertModelic <\SV`V4Wn\V4Vn\ V4VnV'd \V4MRVnVP4R#)z^ add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer N) r&r'r=rr[r encoderripooler post_init)r<r=add_pooling_layerr>s&&&r?r'RemBertModel.__init__sK   +F3%f- /@mF+d  rAc.VPP#rkr[r,r<s&r?get_input_embeddings!RemBertModel.get_input_embeddingss...rAc&WPnR#rkre)r<rs&&r?set_input_embeddings!RemBertModel.set_input_embeddingss */'rAc<V^8dQhRS[PR,RS[PR,RS[PR,RS[PR,RS[PR,RS[PR,RS[PR,R S[R,R S[R,R S[R,R S[R,R S[R,RS[S[,/ #)rCrDNrrEr:rFrrrrrrrrH)r7rIrJrrrr)rMrNs"r?rORemBertModel.__annotate__s] ] ##d*] ((4/] ((4/ ] &&- ] ((4/ ]  %0047] !& 1 1D 8] ] $;]  $;] #Tk] D[]  = =] rAc V eT MVPPp V eT MVPPp V eT MVPPp VPP'dV eT MVPP p MRp VeVe \ R4hVe#VPW4VP4pM#VeVP4RR pM \ R4hVwppVe VPM VPpVf^MVP4pVf"\P!VVV,3VR7pVf(\P!V\PVR7pVPVVVVVR7p\!VPVVR7pVe\!VPVVVR7pVP#VVVVVV V V V R 7 pV^,pVP$eVP%V4MRpV 'gVV3VR ,,#\'VVVP(VP*VP,VP.R 7#) NFzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embeds)rUrS)rDr:rErFrG)r=rFr)r=rFrr)rrrrrrrrr)r! pooler_outputrrtr"r#r%)r=rrrrrr%warn_if_padding_and_no_attention_maskrVrUget_seq_lengthr7onesrWrXr[r r_r`rrrtr"r#)r<rDrrEr:rFrrrrrrrrrY batch_sizerZrUrGembedding_outputencoder_outputsrGrys&&&&&&&&&&&&&, r?r\RemBertModel.forwardsU"2C1N-TXT_T_TqTq$8$D $++JjJj &1%cd d  "  6 6y Q#..*K  &',,.s3KTU U!, J%.%:!! @T@T&5&=?CaCaCc  !"ZZ*jCY6Y)ZdjkN  !"[[EJJvVN??%)'#9 + 3;;*)  " -%>{{.5&; & ",, )"7#9+/!5#'  *!,8< 8O O4UY #]3ob6II I;-'+;;)77&11,==   rA)r=r[r_r`)T) NNNNNNNNNNNN) r_r`rarbr'rgrjrr\rdrerfrgs@@r?r]r]s6 /0] ] ] ] rAr]c^aa]tRtRtoV3RltRtRt]RV3RlRll4tRt Vt V;t #) RemBertForMaskedLMibc<\SV`V4VP'd\P R4\ VRR7Vn\V4VnVP4R#)znIf you want to use `RemBertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.FrbN r&r'rr%warningr]rOrBclsrar;s&&r?r'RemBertForMaskedLM.__init__dsU      NN1  $FeD %f- rAcBVPPP#rkr}rDr;rfs&r?get_output_embeddings(RemBertForMaskedLM.get_output_embeddingssxx##+++rAc:WPPnR#rkrr<new_embeddingss&&r?set_output_embeddings(RemBertForMaskedLM.set_output_embeddingsv'5$rAc<V^8dQhRS[PR,RS[PR,RS[PR,RS[PR,RS[PR,RS[PR,RS[PR,R S[PR,R S[R,R S[R,R S[R,R S[S[,/ #)rCrDNrrEr:rFrrlabelsrrrrH)r7rIrJrrr)rMrNs"r?rORemBertForMaskedLM.__annotate__zs5 5 ##d*5 ((4/5 ((4/ 5 &&- 5 ((4/ 5  %00475 !& 1 1D 85   4'5  $;5 #Tk5 D[5   5 rAc V eT MVPPp VPVVVVVVVV V V R7 p V ^,pVPV4pRpVeG\ 4pV!VP RVPP 4VP R44pV 'g"V3V R,,pVe V3V,#T#\VVV PV PR7#)a{ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. N) rrEr:rFrrrrrrCNNlosslogitsrtr"r%) r=rrOr}rrr)rrtr")r<rDrrEr:rFrrrrrrrrrGrKmasked_lm_lossloss_fctrs&&&&&&&&&&&&, r?r\RemBertForMaskedLM.forwardys,&1%>> from transformers import AutoTokenizer, RemBertForCausalLM, RemBertConfig >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google/rembert") >>> config = RemBertConfig.from_pretrained("google/rembert") >>> config.is_decoder = True >>> model = RemBertForCausalLM.from_pretrained("google/rembert", config=config) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits ```N) rrEr:rFrrrrrrrrRrrr)r)rrrrtr"r#r)r=rrOrrKslicer} loss_functionr)rrrtr"r#)r<rDrrEr:rFrrrrrrrrrrrrt slice_indicesrrrs&&&&&&&&&&&&&&&, r?r\RemBertForCausalLM.forwards.R&1%SV8W8W~ot4]k -=!(;<=  %%pVpFpt{{OeOepiopDY,F)-)9TGf$ Ev E0#33!//))$55   rAr)NNNNNNNNNNNNNr^rrgs@@r?rrs6 ,6M M M M rArz RemBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. cRaa]tRtRtoV3Rlt]RV3RlRll4tRtVtV;t #) RemBertForSequenceClassificationic2<\SV`V4VPVn\V4Vn\ P !VP4Vn\ P!VPVP4Vn VP4R#rk r&r' num_labelsr]rOrr3classifier_dropout_probr5rlrm classifierrar;s&&r?r')RemBertForSequenceClassification.__init__"si   ++#F+ zz&"@"@A ))F$6$68I8IJ rAc\<V^8dQhRS[PR,RS[PR,RS[PR,RS[PR,RS[PR,RS[PR,RS[R,R S[R,R S[R,R S[S[,/ # rCrDNrrEr:rFrrrrrH)r7rJrIrrr)rMrNs"r?rO-RemBertForSequenceClassification.__annotate__-sD D $$t+D ))D0D ((4/ D ''$. D ((4/ D   4'D  $;D #TkD D[D  ) )D rAc V eT MVPPp VPVVVVVVVV R7p V ^,p VPV 4p VP V 4p RpVEeVPP fVP ^8XdRVPnMrVP ^8dQVP\P8Xg VP\P8XdRVPnMRVPnVPP R8XdK\4pVP ^8Xd'V!V P4VP44pMV!W4pMVPP R8Xd>\4pV!V PRVP 4VPR44pM-VPP R8Xd\4pV!W4pV 'g"V 3V R,,pVe V3V,#T#\!VV V P"V P$R7#) ae labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). NrrEr:rFrrr regressionsingle_label_classificationmulti_label_classificationrrr%)r=rrOr5r problem_typerrTr7rXrKrsqueezerrrrrtr")r<rDrrEr:rFrrrrrrryrrrrs&&&&&&&&&&, r?r\(RemBertForSequenceClassification.forward,s(&1%G>SINN2y~~b'9:Y] M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei (   r=#5#5b#9=;M;Mb;Q R  ,, ))%'/!5#     ]3 / ++b+6  ')HO4D%''"+5F)-)9TGf$ Ev E("!//))   rA)rr5rOrrrgs@@r?rrts,W W W W rArcRaa]tRtRtoV3Rlt]RV3RlRll4tRtVtV;t #)RemBertForTokenClassificationic6<\SV`V4VPVn\VRR7Vn\ P !VP4Vn\ P!VPVP4Vn VP4R#FrzNrr;s&&r?r'&RemBertForTokenClassification.__init__sk   ++#FeD zz&"@"@A ))F$6$68I8IJ rAc\<V^8dQhRS[PR,RS[PR,RS[PR,RS[PR,RS[PR,RS[PR,RS[R,R S[R,R S[R,R S[S[,/ #r)r7rJrIrrr)rMrNs"r?rO*RemBertForTokenClassification.__annotate__s1 1 $$t+1 ))D01 ((4/ 1 ''$. 1 ((4/ 1   4'1  $;1 #Tk1 D[1  & &1 rAc V eT MVPPp VPVVVVVVVV R7p V ^,p VPV 4p VP V 4p RpVe=\ 4pV!V P RVP4VP R44pV 'g"V 3V R,,pVe V3V,#T#\VV V PV PR7#)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Nrrrr%) r=rrOr5rrrrrrtr")r<rDrrEr:rFrrrrrrrGrrrrs&&&&&&&&&&, r?r\%RemBertForTokenClassification.forwards$&1%rsD AA&!CC)69   .6,0   H %/ /f BII V.299V.t  ryy8 "))  BII ?-?DM RYYM bRYY" bii "!!i_ii  u )u  u pL /L L ^ a /a  a H P '=P  P fc 5c c L> $:> > BJ "8J J Z rA