+ j 0Rt^RIt^RIt^RIHt^RIHt^RIHt^RIH t H t H t ^RI H t ^RIHtHt^RIHt^R IHtHt^R IHt^R IHtHt^R IHt]P:!]4t!R R]P@4t!!RR]P@4t"!RR]PF4t$!RR]PF4t%!RR]4t&]!RR]44t'!RR]'4t(]!RR7!RR]'44t)]!RR7!R R!]'] 44t*R!R.t+R#)"z/PyTorch TrOCR decoder model (based on RoBERTa).N)nn)CrossEntropyLoss)ACT2FN)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer))BaseModelOutputWithPastAndCrossAttentions!CausalLMOutputWithCrossAttentions)PreTrainedModel)auto_docstringlogging TrOCRConfigc^aa]tRt^%toRtV3RlV3RlltRV3RlV3RllltRtVtV;t #)TrOCRLearnedPositionalEmbeddingzF This module learns positional embeddings up to a fixed maximum size. c&<V^8dQhRS[RS[/#)num_embeddings embedding_dimint)format __classdict__s"i/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/trocr/modeling_trocr.py __annotate__,TrOCRLearnedPositionalEmbedding.__annotate__*sFFsF3FcV<^Vn\SV` WP,V4R#rN)offsetsuper__init__)selfrr __class__s&&&rr%(TrOCRLearnedPositionalEmbedding.__init__*s"  ++5}Er cb<V^8dQhRS[PRS[RS[PR,/#)r input_idspast_key_values_length position_idsNtorchTensorr)rrs"rrr0s5 ; ; ;?B ;V[VbVbeiVi ;r c <<VfiVPR,wrE\P!W"V,\PVPP R7P VR4pMVP^4p\SV`%W0P,4#)z3`input_ids' shape is expected to be [bsz x seqlen].:NrN)dtypedevice) shaper.arangelongweightr2expand unsqueezer$forwardr#)r&r*r+r,bszseq_lenr's&&&& rr:'TrOCRLearnedPositionalEmbedding.forward0s  $??2.LC <<&(HPUPZPZcgcncncucufS"o (11!4Lw|kk9::r )r#)N __name__ __module__ __qualname____firstlineno____doc__r%r:__static_attributes____classdictcell__ __classcell__r'rs@@rrr%s%FF ; ; ;r rc^aa]tRt^AtoRtRV3RlV3RllltV3RlV3RlltRtVtV;t #)TrOCRScaledWordEmbeddingzT This module overrides nn.Embeddings' forward by multiplying with embeddings scale. c @<V^8dQhRS[RS[RS[RS[R,/#)rrr padding_idx embed_scaleN)rfloat)rrs"rr%TrOCRScaledWordEmbedding.__annotate__Fs0''s'3'S'_dgk_k'r c4<\SV`WV4W@nR#N)r$r%rM)r&rrrLrMr's&&&&&rr%!TrOCRScaledWordEmbedding.__init__Fs  D&r c4<V^8dQhRS[P/#)rr*)r.r/)rrs"rrrOJs===r cD<\SV`V4VP,#rQ)r$r:rM)r&r*r's&&rr: TrOCRScaledWordEmbedding.forwardJswy)D,<,<<=CCEOOPQRUXUbUbcdUeeii338a@EEnVXY 1  !))S%++na"@AqIC  ""#CQ vve--/00r c:<V^8dQhRS[PRS[/#)rr*r+r-)rrs"rrr\lssr c VP4wr4VPWPV4PVP4pVP^,V,pVP e W`P P^48d,VP W`PVP4VnVP P^VPR44PW4R4P4pV#)r3) size"create_position_ids_from_input_idsrLror2r_r^r index_selectrmdetach)r&r*r+r;r<r,max_posxs&&& rr:*TrOCRSinusoidalPositionalEmbedding.forwardks ~~' >>yJZJZ\rsvv    ""Q&0 << 7\\->->q-A#A--g7I7I4K[K[\DL LL % %a):):2)> ? D DSSU V ] ] _r cN<V^8dQhRS[PRS[RS[R,/#)rr*rLr+Nr-)rrs"rrr\}s/ 8 8 847 8QTW[Q[ 8r c VPV4P4p\P!V^R7P V4V,V,pVP 4V,#)z Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. rc)nerr.cumsumtype_asr6)r&r*rLr+maskincremental_indicess&&&& rrxETrOCRSinusoidalPositionalEmbedding.create_position_ids_from_input_ids}sW||K(,,.$||Da8@@FI__cgg"'')K77r )rr#rLr_rQ)r>)r@rArBrCrDr% staticmethodr^r.no_gradr:rxrErFrGrHs@@rrYrYNs[NUU111$ ]]_" 8 8 8r rYc\aa]tRt^toRtRV3RlV3RllltRV3RlRlltRtVtV;t #) TrOCRAttentionz>Multi-headed attention from 'Attention Is All You Need' paper.c<V^8dQhRS[RS[RS[R,RS[R,RS[R,RS[R,RS[R,R S[R,R S[R,/ #) r embed_dim num_headskdimNvdimdropout is_decoderbiasis_cross_attention layer_idx)rrNbool)rrs"rrTrOCRAttention.__annotate__s!C!C!C !C Dj !C Dj !C!C4K!CTk!C!4K!C$;!Cr c j<\S V`4W nVeTMTVnVeTMTVnW0nW`nW#,VnVPV,VP8Xg\RVP RV R24hVPR,Vn Wpn Wn \P!VPW(R7Vn\P!VPW(R7Vn\P!W"VR7Vn\P!W"VR7VnR#)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).rg)r$r%rrrrrhead_dim ValueErrorscalingrrrLineark_projv_projq_projout_proj) r&configrrrrrrrrrr's &&&&&&&&&&&rr%TrOCRAttention.__init__s " ,D)  ,D) " !.  )T^^;MdnnM]^;b" }}d* $"ii 9@ ii 9@ ii 4@  )TB r c4<V^8dQhRS[PRS[PR,RS[R,RS[PR,RS[R,RS[S[PS[PR,S[S[P,R,3,/#)r hidden_stateskey_value_statesNpast_key_valuesattention_maskoutput_attentionsreturn)r.r/rrtuple)rrs"rrrsd2d2||d2 ,,-d2 d2  t+ d2  $; d2 u||U\\D0% 2E2LL Md2r c . VRJpVP4wrp VPV4VP,p Rp Ve`\V\4'dHVP P VP4p V'dVPp MVPp MTp V'dTMTpV'd\VeXV 'dPX PVP,PpV PVP,PpMVPV4pVPV4pVPVR VP VP"4P%^^4pVPVR VP VP"4P%^^4pVeXX P'VVVP4wppV'd0\V\4'dRVP VP&WP ,R VP"3pV PWVP VP"4P%^^4p V P(!V!p VP(!V!pVP(!V!pVP^4p\*P,!WP%^^44pVP4WP ,V V38wd3\/RWP ,V V3 RVP4 24hVeVP4V^V V38wd$\/RV^V V3 RVP4 24hVPWP V V4V,pVPWP ,V V4p\0P2P5VR R7pV'dBVPWP V V4pVPWP ,V V4pMRp\0P2P7VVP6VP8R7p\*P,!VV4pVP4WP ,WP"38wd6\/R WP WP"3 RVP4 24hVPWP WP"4pVP%^^4pVP)WV 4pVP;V4pVV3#) z#Input shape: Batch x Time x ChannelNFTz$Attention weights should be of size z , but is z!Attention mask should be of size rcptrainingz `attn_output` should be of size r3)rwrr isinstancer is_updatedgetrcross_attention_cacheself_attention_cachelayerskeysvaluesrrrmrr transposeupdatereshaper.bmmrr functionalsoftmaxrrr)r&rrrrrkwargsrr;tgt_lenr query_statesrcurr_past_key_valuescurrent_states key_states value_states proj_shapesrc_len attn_weightsattn_weights_reshaped attn_probs attn_outputs&&&&&&, rr:TrOCRAttention.forwardsg.T9"/"4"4"6i{{=1DLL@   &/+>??,77;;DNNK %+:+P+P(+:+O+O('6$-?)] /"=*-44T^^DIIJ/66t~~FMML^4J;;~6L#b$..$--PZZ[\^_`J',,S"dnndmmT^^_`bcdL*+?+F+FzS_aeaoao+p( L%*_FY*Z*ZAEO..t~~>NN*B > #((t~~t}}U__`acde #++Z8 ''4 #++Z8 //!$yy/C/CAq/IJ    3#7'"J J6nn8LgW^7_6`a %%'(*   %""$a'(BB 7a'8R7SS\]k]p]p]r\st(,,S..'7SVddL',,S>>-A7GTL}},,\r,B  %1$5$5c>>7T[$\ !055cNN6JGU\]L$( !]]**<4<$>?"(";";$&LL$@!     ... 88777700#'# !D ,.<<+GD (99T^^V-C-CD99V33T^^D " T^^ <r c<V^8dQhRS[PRS[PR,RS[PR,RS[PR,RS[R,RS[R,RS[R,/#) rrrNencoder_hidden_statesencoder_attention_maskrr use_cache)r.r/rr)rrs"rrr<sGG||G t+G %||d2 G !& t 3 G  G $;G$;Gr c FTp VPVVVVR7wr\PPWPVPR7pW,pVP V4pRp VeiTp VP VVVVVR7wr\PPWPVPR7pW,pVPV4pTp VPVPV44p\PPWPVPR7pVPV4p\PPWPVPR7pW,pVPV4pV3p V'd WV 3, p V #)a^ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape `(batch, seq_len, embed_dim)` encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. past_key_values (`Cache`): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. )rrrrrN)rrrrr) rrrrrrrrrrrrr) r&rrrrrrrrresidualself_attn_weightscross_attn_weightsoutputss &&&&&&&&, rr:TrOCRDecoderLayer.forward<s2!,0>>'+)/ ,:, (  --m||VZVcVc-d  0 11-@ " ,$H040A0A+!65 /"3 1B1 -MMM11-< >Gr ) rrrrrrrrrrrrQ)NNNNFT r@rArBrCr%r:rErFrGrHs@@rrrs""="=HGGGr rc:a]tRtRtoRtRtR.tV3RltRtVt R#)TrOCRPreTrainedModelimodelTrc&<V^8dQh/S[;R&#rr)rrs"rr!TrOCRPreTrainedModel.__annotate__s r N) r@rArBrCbase_model_prefixsupports_gradient_checkpointing_no_split_modules__annotate_func__rErF)rs@rrrs# &*#,- r rcLaa]tRtRtoRtV3RlV3RlltRRltRtVtV;t #) TrOCRDecoderiz Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`TrOCRDecoderLayer`] Args: config: TrOCRConfig c <V^8dQhRS[/#rr)rrs"rrTrOCRDecoder.__annotate__s{r c <\SV`V4VPVnVPVnVP VnVP'd!\P!VP4MRp\VPVPVP VR7Vn VP'd'\VP VP4VnMH\%VP VP ,^,VPVP 4VnVP&'d'\(P*!VP4VnMRVn\(P,!\/VP04Uu.uFp\3WR7NK up4VnRVnVP94R#uupi)rWrVN)rF)r$r%rdecoder_layerdrop layerdrop pad_token_idrLscale_embeddingrfsqrtrrJ vocab_size embed_tokensuse_learned_position_embeddingsrmax_position_embeddingsembed_positionsrYlayernorm_embeddingrr ModuleListrangedecoder_layersrrgradient_checkpointing post_init)r&rrMir's&& rr%TrOCRDecoder.__init__s]  ~~ 11!..7=7M7M7Mdii 2 23SV 4   v1143C3CQ\   1 1 1#B6CaCacicucu#vD #E..1A1AAAE""  $D  % % %')||F4F4F'GD $'+D $mmUZ[a[p[pUq$rUqPQ%6v%KUq$rs &+#  %ssG c VeTMVPPpV eT MVPPp VeTMVPPpV eT MVPPp VeVe \ R4hVe'Tp VP RV PR,4pMVe VR,p M \ R4hVP'd2VP'd V'd\PR4RpV'doVfkVfVPP'd5\\VPR7\VPR74M\VPR7pVeVP4M^p VfVP!V4pVPP"'dVP%WR7pMVP%WR7pVP'VP(4pWn,pVP*eVP+V4p\,P.P1WP0VPR7p\3VPVVVR 7pVeVe\5VPVVVR 7pV 'dRMRpV'dRMRpV'dVeRMRp\7VP84FwppV 'd VV3, pVP'd*\:P<!.4pVVP>8dKSV!VVVVVVVR 7pV^,pV'gKuVV^,3, pVfKVV^,3, pK V 'd VV3, pV 'g5\@;QJd.R WVVV34FNK 5#!R WVVV344#\CVVVVVR 7#)a Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timezEYou have to specify either decoder_input_ids or decoder_inputs_embedsz^`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache = False`...F)r)r+r)r inputs_embedsrr)rrrr)rrrrc32"TF pVfK VxK R#5irQr).0vs& r 'TrOCRDecoder.forward..[srArs )last_hidden_staterr attentionscross_attentionsr3)rerer3r)"rroutput_hidden_statesr return_dictrrmr4r rlogger warning_onceis_encoder_decoderrrget_seq_lengthrrrror2rrrrr r enumeraterr.randrrr )r&r*rrrrrrrrrrinputr+ embed_posrall_hidden_statesall_self_attnsall_cross_attentionsidx decoder_layerdropout_probability layer_outputss&&&&&&&&&&&, rr:TrOCRDecoder.forwards@2C1N-TXT_T_TqTq$8$D $++JjJj "+!6IDKKst t  "E!r5;;r?;I  &!(+Ede e  & & &4===##t" 0)4 8V8V8V$L $DlZ^ZeZeFfg!5  FUE`!?!?!Afg   --i8M ;; 6 6 6,,U,bI,,Y,fILL!5!56 %1  # # / 44]CM --m||VZVcVc-d +;;')+   ! ,1G1S%>{{+5&; & "#7BD0d&7q q r ra The TrOCR Model with a language modeling head. Can be used for summarization. This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is used in combination with the [`EncoderDecoderModel`] framework. ) custom_introc8aa]tRtRtoV3RltRtRtVtV;t#)TrOCRDecoderWrapperiicf<\SV`V4\V4VnVP 4R#rQ)r$r%rdecoderr r&rr's&&rr%TrOCRDecoderWrapper.__init__qs&  #F+  r c&VP!V/VB#rQr-)r&argsrs&*,rr:TrOCRDecoderWrapper.forwardvs||T,V,,r r1rrHs@@rr+r+is --r r+zy The TrOCR Decoder with a language modeling head. Can be used as the decoder part of [`EncoderDecoderModel`] and craa]tRtRtoRR/tV3RltRtRtRtRt ] R V3R lR ll4t R t Vt V;t#) TrOCRForCausalLMizzoutput_projection.weightz!model.decoder.embed_tokens.weightc<RVnRVn\SV` V4\ V4Vn\ P!VPVPRR7Vn VP4R#)TFrN) rrr$r%r+rrrrroutput_projectionr r.s&&rr%TrOCRForCausalLM.__init__sZ $)!  (0 !#6+=+=v?P?PW\!] r cBVPPP#rQrr-rr&s&rget_input_embeddings%TrOCRForCausalLM.get_input_embeddingsszz!!...r c:WPPnR#rQr:)r&values&&rset_input_embeddings%TrOCRForCausalLM.set_input_embeddingss*/ 'r cVP#rQr7r;s&rget_output_embeddings&TrOCRForCausalLM.get_output_embeddingss%%%r cWnR#rQrC)r&new_embeddingss&&rset_output_embeddings&TrOCRForCausalLM.set_output_embeddingss!/r c<V^8dQhRS[PR,RS[PR,RS[PR,RS[PR,RS[R,RS[PR,RS[PR,R S[R,R S[R,R S[R,R S[R,R S[S[,/ #)rr*Nrrrrrlabelsrrrrr)r. LongTensorr/ FloatTensorrrrr )rrs"rrTrOCRForCausalLM.__annotate__sm m ##d*m  t+m  %0047 m !& 0 04 7 m  m ((4/m   4'm $;m  $;m #Tkm D[m  2 2m r c V eT MVPPp V eT MVPPp V eT MVPPp VPP VVVVVVVV V V R7 p VP V ^,4pRpVeG\4pV!VPRVPP4VPR44pV 'g"V3V R,,pVe V3V,#T#\VVV PV PV PV PR7#)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import ( ... TrOCRConfig, ... TrOCRProcessor, ... TrOCRForCausalLM, ... ViTConfig, ... ViTModel, ... VisionEncoderDecoderModel, ... ) >>> import httpx >>> from io import BytesIO >>> from PIL import Image >>> # TrOCR is a decoder model and should be used within a VisionEncoderDecoderModel >>> # init vision2text model with random weights >>> encoder = ViTModel(ViTConfig()) >>> decoder = TrOCRForCausalLM(TrOCRConfig()) >>> model = VisionEncoderDecoderModel(encoder=encoder, decoder=decoder) >>> # If you want to start from the pretrained model, load the checkpoint with `VisionEncoderDecoderModel` >>> processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-handwritten") >>> model = VisionEncoderDecoderModel.from_pretrained("microsoft/trocr-base-handwritten") >>> # load image from the IAM dataset >>> url = "https://fki.tic.heia-fr.ch/static/img/a01-122-02.jpg" >>> with httpx.stream("GET", url) as response: ... image = Image.open(BytesIO(response.read())).convert("RGB") >>> pixel_values = processor(image, return_tensors="pt").pixel_values >>> text = "industry, ' Mr. Brown commented icily. ' Let us have a" >>> # training >>> model.config.decoder_start_token_id = processor.tokenizer.eos_token_id >>> model.config.pad_token_id = processor.tokenizer.pad_token_id >>> model.config.vocab_size = model.config.decoder.vocab_size >>> labels = processor.tokenizer(text, return_tensors="pt").input_ids >>> outputs = model(pixel_values, labels=labels) >>> loss = outputs.loss >>> round(loss.item(), 2) 5.30 >>> # inference >>> generated_ids = model.generate(pixel_values) >>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] >>> generated_text 'industry, " Mr. Brown commented icily. " Let us have a' ```N) r*rrrrrrrrr:rvNN)losslogitsrrrrr3)rrrrrr-r7rrmrr rrrr)r&r*rrrrrrKrrrrrrrQrPloss_fctoutputs&&&&&&&&&&&&, rr:TrOCRForCausalLM.forwards@P2C1N-TXT_T_TqTq$8$D $++JjJj &1%rds$6 %!CC)J9l-,,   H %;bll;8 =r|| =9898xJ2RYYJ2Zl2l^.?..X 'X v-.-- H +_H  H V 5 6r