+ j^&Rt^RIHt^RIt^RIt^RIHt^RIHtH t H t ^RI H t ^RIHtHt^RIHt^RIHt^R IHtHtHt^R IHtHt^R IHt^R IHtH t H!t!H"t"^R I#H$t$^RI%H&t&^RI'H(t(]"PR!]*4t+Rt,Rt-R$RRllt.!RR]P^4t0Rt1!RR]P^4t2] !RR]44t3] !RR]344t4] !RR7!RR ]3]44t5] !R!R7!R"R#]344t6.R%Ot7R#)&zPyTorch CTRL model.)CallableN)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)Cache DynamicCache)GenerationMixin)create_causal_mask)BaseModelOutputWithPastCausalLMOutputWithPastSequenceClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)merge_with_config_defaults)capture_outputs CTRLConfigcx^\P!R^V^,,V, 4, pW,#)i')torchpow)posi d_model_size angle_ratess&&& g/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/ctrl/modeling_ctrl.py angle_defnr#-s-eiiQ!V 'DEEK  c\\P!V\PR7P V4P ^4\P!V\PR7P V4P ^4V4p\P !VR,4p\P!VR,4p\P!WE.RR7pV#))dtypedim)NNN:N)r):rNr+) r#rarangeint64to unsqueezesincoscat)positionr r& angle_radssinescosines pos_encodings&&& r"positional_encodingr92s XU[[144U;EEaH \588?II!LJ IIj) *Eii 7+,G99e-26L r$cV^8dQhR\PR\PR\PR\PR\PR,R\R,R\R \ \ ,/#) r+modulequerykeyvalueattention_maskNscalingdropoutkwargs)rModulerTensorfloatrr)formats"r" __annotate__rGBs{%% II% <<% % << % LL4' % T\ %%'(%r$c VfVPR4R,p\P!WP^^44V,pVe W,p\P P VRR7p\P PWVPR7p\P!W4p V P^^4P4p W3#)Nr')ptrainingr,) sizermatmul transposer functionalsoftmaxrArJ contiguous) r;r<r=r>r?r@rArB attn_weights attn_outputs &&&&&&&, r"eager_attention_forwardrTBs**R.D(<<}}Q':;gEL!#4 ==((2(>L==((6??([L,,|3K''1-88:K  $$r$cLaa]tRt^^toRV3RlltRV3RlRlltRtVtV;t#)MultiHeadAttentionc<\SV`4WnVPVnVP VnW nRVn\VP VP, 4Vn VPR,Vn \P!VP VP 4Vn\P!VP VP 4Vn\P!VP VP 4Vn\P!VP VP 4VnR#)TNrK)super__init__confign_head num_headsn_embdr layer_idx is_causalinthead_dimr@rLinearWqWkWvdenseselfrZr^ __class__s&&&r"rYMultiHeadAttention.__init___s  "MM"D-->? }}d* ))D--t/@/@A))D--t/@/@A))D--t/@/@AYYt00$2C2CD r$c0<V^8dQhRS[S[,/#r+rBrr)rF __classdict__s"r"rGMultiHeadAttention.__annotate__ps#)#)+,#)r$c VPRRp.VORNVPN5pVPV4PV4P ^^4p VP V4PV4P ^^4p VP V4PV4P ^^4p VeVPWVP4wr\P!VPP\4p V !VV V V V3RRRVP/VBwrV P!.VORN5!P!4p VP#V 4p W3#)NrAr@r,)shaperarcviewrNrdreupdater^r get_interfacerZ_attn_implementationrTr@reshaperQrf)rhvkq layer_pastr?rB input_shape hidden_shape query_states key_states value_statesattention_interfacerSrRs&&&&&&, r"forwardMultiHeadAttention.forwardpsQggcrl 88b8$--8 wwqz|4>>q!D WWQZ__\2<>q!D  !'1'8'8SWSaSa'b $J(?(M(M KK , ,.E) %8      %  % LL %  % ! "));;;;FFH jj- ((r$) rdrcrerZr rfrar_r^r\r@NNN __name__ __module__ __qualname____firstlineno__rYr__static_attributes____classdictcell__ __classcell__rirns@@r"rVrV^sE"#)#)#)r$rVc\P!\P!W4\P!4\P!W44#r)r SequentialrbReLU)r dffs&&r"point_wise_feed_forward_networkrs- ==<5rwwy"))CB^ __r$cLaa]tRt^toRV3RlltRV3RlRlltRtVtV;t#) EncoderLayerc<\SV`4\WR7Vn\ VP VP 4Vn\P!VP RR7Vn \P!VP RR7Vn \P!VP4Vn\P!VP4VnR#)r^gư>epsN)rXrYrVmulti_head_attentionrr]rffnr LayerNorm layernorm1 layernorm2Dropout resid_pdropdropout1dropout2rgs&&&r"rYEncoderLayer.__init__s $6v$S!26==&**M,,v}}$?,,v}}$? 6#5#56  6#5#56 r$c0<V^8dQhRS[S[,/#rlrm)rFrns"r"rGEncoderLayer.__annotate__s +, r$c VPV4pVP!VVV3RVRV/VBwrgVPV4pW,pVPV4p VP V 4p VP V 4p W,p V #)r{r?)rrrrrr) rhxr{r?rBnormedrS_out1out2 ffn_outputs &&&&, r"rEncoderLayer.forwards#22    "  *     mmK0 t$XXd^ ]]:.   r$)rrrrrrrrrrs@@r"rrs 7r$rc^aa]tRt^toRtRtRtRtRtR] R] /t V3Rlt V3Rlt RtVtV;t#)CTRLPreTrainedModel transformerT hidden_states attentionsc<\SV`V4\V\4'd[\P !VP \VPPVP\P44R#R#r) rX _init_weights isinstance CTRLModelinitcopy_r8r9rZ n_positionsr rrE)rhr;ris&&r"r!CTRLPreTrainedModel._init_weightss[ f% fi ( ( JJ##%89R9RTZTgTginitit%u  )r$c&<V^8dQh/S[;R&#)r+rZr)rFrns"r"rG CTRLPreTrainedModel.__annotate__s r$)rrrrbase_model_prefix_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrrV_can_record_outputsr__annotate_func__rrrrs@@r"rrsC&N"&( r$rcraa]tRt^toV3RltRtRt]]] RV3RlRll444t Rt Vt V;t #)rc <\SV`V4VPVnVPVn\ P!VPVP4Vn \ P!VP4Vn \ P!\VP4Uu.uFp\WR7NK up4Vn\ P"!VPVP$R7Vn\ P(!\+VP,VP\.P04RR7VnVP54R#uupi)rrF) persistentN)rXrYr]r n_layer num_layersr Embedding vocab_sizewr embd_pdroprA ModuleListrangerhrlayer_norm_epsilon layernormBufferr9rrrEr8 post_init)rhrZrris&& r"rYCTRLModel.__init__s  "MM ..f//?zz&"3"34 5QWQ_Q_K`aK`a V AK`abfmm9R9RSII  2 2D4E4Eu{{ S`e   bs/EcVP#rr)rhs&r"get_input_embeddingsCTRLModel.get_input_embeddingss vv r$cWnR#rr)rhnew_embeddingss&&r"set_input_embeddingsCTRLModel.set_input_embeddingssr$c&<V^8dQhRS[PR,RS[R,RS[PR,RS[PR,RS[PR,RS[PR,RS[R,R S[S[,R S[/ #) r+ input_idsNpast_key_valuesr?token_type_ids position_ids inputs_embeds use_cacherBreturn)r LongTensorr FloatTensorboolrrr )rFrns"r"rGCTRLModel.__annotate__s\ \ ##d*\ \ ))D0 \ ((4/ \ &&- \ ((4/\ $;\ +,\  !\ r$c VeTMVPPpVeVe \R4hVeVPV4pMVf \R4hVPRR p VP^,p VP p V'dVf\ VPR7pVeVP4M^p VfG\P!WR ,V ,\PV R7pVP^4pVe:VPV4p V \P!VP4,p M^p VeD\V\P 4'd$VP"^8dVP%V R 4p\'VPVVVVR7pV\P!VP4,pVP(P+WP,R7VnVP(VR3,pWo,V ,pVP/V4pVP0FpV!V3R VR V/VBpK VP3V4p\5TV'dVR 7#RR 7#) a1 Example: ```python >>> from transformers import AutoTokenizer, CTRLModel >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLModel.from_pretrained("Salesforce/ctrl") >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Opinion My dog is cute", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> outputs = model(**inputs) >>> last_hidden_states = outputs.last_hidden_state >>> list(last_hidden_states.shape) [1, 5, 1280] ```NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embeds)rZ)r&device)rZrr?rrrr&r)r{r?)last_hidden_staterr,)rZr ValueErrorrrrrr get_seq_lengthrr-longr0npsqrtr rrDndimrsr r8r/r&rArrr )rhrrr?rrrrrBr| batch_sizer past_lengthtoken_type_embeds causal_mask pos_embedsrrs&&&&&&&&, r"rCTRLModel.forwardsKD"+!6IDKKcd d   FF9-M  "TU U#))#2. "((+ %% 0*$++>O:I:Uo446[\   << _{5RZ_ZdZdmstL'11!4L  % $~ 6  ):):!; ;  !   %*^U\\*R*RWeWjWjmnWn+00R@N(;;')+%   !2!233 !--00FYFY0Z&&|Q7 %25FF  ]3 A* + M}5 &+/8O  >B  r$)r rArrrr8r)NNNNNNN)rrrrrYrrrrrrrrrrs@@r"rrsF&  \ \  \ \ r$rz The CTRL Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). ) custom_introctaa]tRtRtoRR/tV3Rlt]]R V3RlRll44tR V3Rllt Rt Vt V;t #) CTRLLMHeadModeliTzlm_head.weightztransformer.w.weightc<\SV`V4\V4Vn\P !VP VPRR7VnVP4R#)TbiasN) rXrYrrrrbr]rlm_headrrhrZris&&r"rYCTRLLMHeadModel.__init__]sG  $V,yy0A0AM  r$cx<V^8dQhRS[PR,RS[R,RS[PR,RS[PR,RS[PR,RS[PR,RS[PR,R S[R,R S[S[P ,R S[S[,R S[ / #) r+rNrr?rrrlabelsrlogits_to_keeprBr) rrrrrr`rDrrr )rFrns"r"rGCTRLLMHeadModel.__annotate__gsK K ##d*K K ))D0 K ((4/ K &&- K ((4/K   4'K $;K ell*K +,K  K r$c  VP!V3RVRVRVRVRVRV/V Bp V ^,p \V \4'd\V )R4MT p VP V RV R3,4pRpVe+VP !VV3R VP P/V Bp\VVV PV PV PR 7#) a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` Example: ```python >>> import torch >>> from transformers import AutoTokenizer, CTRLLMHeadModel >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLLMHeadModel.from_pretrained("Salesforce/ctrl") >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Wikipedia The llama is", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> sequence_ids = model.generate(inputs["input_ids"]) >>> sequences = tokenizer.batch_decode(sequence_ids) >>> sequences ['Wikipedia The llama is a member of the family Bovidae. It is native to the Andes of Peru,'] >>> outputs = model(**inputs, labels=inputs["input_ids"]) >>> round(outputs.loss.item(), 2) 9.21 >>> list(outputs.logits.shape) [1, 5, 246534] ```rr?rrrrNr)r)losslogitsrrr) rrr`slicer loss_functionrZrr rrr)rhrrr?rrrrrrrBtransformer_outputsr slice_indicesr rs&&&&&&&&&&, r"rCTRLLMHeadModel.forwardes\#..   +  *  *  &  (       ,A. 8B>SV8W8W~ot4]k mA}a,?@A  %% ;;11 D&/??-;;*55   r$c Z<\SV`!V3RVRVRV/VBpVPRR4V#)rris_first_iterationrN)rXprepare_inputs_for_generationpop)rhrrrrrB model_inputsris&&&&&, r"r-CTRLLMHeadModel.prepare_inputs_for_generationsV w<  +    2     )40r$)rr) NNNNNNNNr*)NNF) rrrr_tied_weights_keysrYrrrrrrrrs@@r"rrTsG+,BCK K K Zr$ra The CTRL Model transformer with a sequence classification head on top (linear layer). [`CTRLForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). c\aa]tRtRtoV3Rlt]]RV3RlRll44tRtVt V;t #)CTRLForSequenceClassificationic<\SV`V4VPVn\V4Vn\ P !VPVPRR7VnVP4R#)FrN) rXrY num_labelsrrrrbr] classifierrrs&&r"rY&CTRLForSequenceClassification.__init__sR   ++$V,))FMM4??O r$cN<V^8dQhRS[PR,RS[R,RS[PR,RS[PR,RS[PR,RS[PR,RS[PR,R S[R,R S[S[,R S[/ #) r+rNrr?rrrrrrBr)rrrrrrrr)rFrns"r"rG*CTRLForSequenceClassification.__annotate__sY Y ##d*Y Y ))D0 Y ((4/ Y &&- Y ((4/Y   4'Y $;Y +,Y  "Y r$c  VP!V3RVRVRVRVRVRV/V Bp V ^,p VPV 4p VeVPR,wrMVPR,wrVPPfV ^8wd \ R 4hVPPfRpMVeWPP8gP V P\P4p\P!VPR,V P\PR 7pVV,PR4pM.Rp\PVPP R 24V \P!WPR 7V3,pRpVEeVPP fVP"^8XdR VPnMrVP"^8dQVP$\P&8Xg VP$\P(8XdRVPnMRVPnVPP R 8XdL\+4pVP"^8Xd'V!VP-4VP-44pMV!VV4pMVPP R8Xd>\/4pV!VP1RVP"4VP1R44pM.VPP R8Xd\34pV!VV4p\5VVV P6V P8R7#)a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Example of single-label classification: ```python >>> import torch >>> from transformers import AutoTokenizer, CTRLForSequenceClassification >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLForSequenceClassification.from_pretrained("Salesforce/ctrl") >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Opinion My dog is cute", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> with torch.no_grad(): ... logits = model(**inputs).logits >>> predicted_class_id = logits.argmax().item() >>> model.config.id2label[predicted_class_id] 'LABEL_0' ``` ```python >>> import torch >>> torch.manual_seed(42) # doctest: +IGNORE_RESULT >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)` >>> num_labels = len(model.config.id2label) >>> model = CTRLForSequenceClassification.from_pretrained("Salesforce/ctrl", num_labels=num_labels) >>> labels = torch.tensor(1) >>> loss = model(**inputs, labels=labels).loss >>> round(loss.item(), 2) 0.93 ``` Example of multi-label classification: ```python >>> import torch >>> from transformers import AutoTokenizer, CTRLForSequenceClassification >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLForSequenceClassification.from_pretrained( ... "Salesforce/ctrl", problem_type="multi_label_classification" ... ) >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Opinion My dog is cute", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> with torch.no_grad(): ... logits = model(**inputs).logits >>> predicted_class_id = logits.argmax().item() >>> model.config.id2label[predicted_class_id] 'LABEL_0' ``` ```python >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)` >>> num_labels = len(model.config.id2label) >>> model = CTRLForSequenceClassification.from_pretrained("Salesforce/ctrl", num_labels=num_labels) >>> num_labels = len(model.config.id2label) >>> labels = torch.nn.functional.one_hot(torch.tensor([predicted_class_id]), num_classes=num_labels).to( ... torch.float ... ) >>> loss = model(**inputs, labels=labels).loss >>> loss.backward() # doctest: +IGNORE_RESULT ```rr?rrrrN:Nr+Nz=Cannot handle batch sizes > 1 if no padding token is defined.rz will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`)r regressionsingle_label_classificationmulti_label_classification)rr rrr,)rrrrrZ pad_token_idrr/rrint32r-argmaxlogger warning_oncerir problem_typerr&rr`rsqueezerrsrrrr)rhrrr?rrrrrrBr rr rsequence_lengthlast_non_pad_token non_pad_mask token_indices pooled_logitsrloss_fcts&&&&&&&&&, r"r%CTRLForSequenceClassification.forwardsv#..   +  *  *  &  (       ,A. /  *3//"*= 'J*7*=*=b*A 'J ;; # # + a\] ] ;; # # +!#   "%)A)AAEEfmmUZU`U`aL!LL)!F!Fr!J !#    >>**+,ZZ  u||J}}MOaab   {{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#M$9$9$;V^^=MND#M6:D))-JJ+- 2 22t GUWY))-II,. v6' -;;*55   r$)rrr)NNNNNNNN) rrrrrYrrrrrrrs@@r"rrs4Y Y Y Y r$r)Nrq)rrrr)8__doc__collections.abcrnumpyrrrtorch.nnrrrrr cache_utilsrr generationr masking_utilsr modeling_outputsr r rmodeling_utilsrrprocessing_utilsrutilsrrrr utils.genericrutils.output_capturingrconfiguration_ctrlr get_loggerrr%r#r9rTrCrVrrrrrr__all__rr$r"rAs+$ AA&.)/iiF& 85*   H %  %85)5)p`%299%P/(y #y y x j)?j  jZ e $7e  e P cr$