+ jp^RIHt^RIt^RIt^RIHt^RIHuHt^RI H t ^RI H t ^RIHtHtHt^RIHt^RIHt^RIHt^R IHt^R IHtHt^R IHtHt^R I H!t!H"t"^R I#H$t$^RI%H&t&H't'H(t(^RI)H*t*^RI+H,t,H-t-^RI.H/t/H0t0^RI1H2t2H3t3H4t4H5t5H6t6!RR]Pn4t8!RR]Pn4t9!RR]Pn4t:!RR]4t;RRltRAR lt?!R!R"]Pn4t@!R#R$]Pn4tA]'!R%R&]"44tB!R'R(]B4tC!R)R*]B4tD!R+R,]B4tER-R.ltF!R/R0]B4tGRBR1R2lltHRCR3R4lltIR5R6ltJR7^]P3R8R9lltL!R:R;]B4tM]'!RR?]B]44tN.RDOtOR#)E)CallableN)initialization)ACT2FN)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)create_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)deprecate_kwarg)maybe_autocastmerge_with_config_defaults)OutputRecordercapture_outputs) BltConfigBltGlobalTransformerConfigBltLocalDecoderConfigBltLocalEncoderConfigBltPatcherConfigc8aa]tRt^4toV3RltRtRtVtV;t#)BltMLPc<\SV`4WnVPVnVPVn\ P !VPVPRR7Vn\ P !VPVPRR7Vn\ P !VPVPRR7Vn \VP,Vn R#FbiasN) super__init__config hidden_sizeintermediate_sizennLinear gate_projup_proj down_projr hidden_actact_fnselfr) __class__s&&e/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/blt/modeling_blt.pyr(BltMLP.__init__5s  !--!'!9!94#3#3T5K5KRWXyy!1!143I3IPUV 4#9#94;K;KRWXV../ cVPVPVPV44VPV4,4pV#N)r0r2r.r/)r4xr0s&& r6forwardBltMLP.forward@s6NN4;;t~~a/@#ADLLQRO#ST r8)r2r)r0r.r*r+r/ __name__ __module__ __qualname____firstlineno__r(r<__static_attributes____classdictcell__ __classcell__r5 __classdict__s@@r6r"r"4s 0r8r"cZaa]tRt^EtoRV3RlV3RllltV3RlRltRtRtVtV;t #) BltRMSNormc$<V^8dQhRS[RR/#)epsreturnN)float)formatrGs"r6 __annotate__BltRMSNorm.__annotate__Fs$$$$$r8c <\SV`4\P!\P !V44VnW nR#)z) BltRMSNorm is equivalent to T5LayerNorm N)r'r(r, Parametertorchonesweightvariance_epsilon)r4r*rLr5s&&&r6r(BltRMSNorm.__init__Fs/ ll5::k#:; #r8cN<V^8dQhRS[PRS[P/#)rK hidden_statesrMrTTensor)rOrGs"r6rPrQNs#;;U\\;ell;r8c:VPpVP\P4pVP ^4P RRR7pV\P !W0P,4,pVPVPV4,#)rKT)keepdim) dtypetorTfloat32powmeanrsqrtrWrV)r4rZ input_dtypevariances&& r6r<BltRMSNorm.forwardNsw#)) %((7  $$Q',,R,>% H?T?T4T(UU {{]--k:::r8c^\VPP4 RVP 2#)z, eps=)tuplerVshaperWr4s&r6 extra_reprBltRMSNorm.extra_reprUs* ))*+6$2G2G1HIIr8)rWrV)gư>) r?r@rArBr(r<rmrCrDrErFs@@r6rIrIEs%$$;;JJr8rIcaa]tRt^Yto]!RRR7R V3RlV3Rlll4t]]!RRR7R V3RlRll44t]P!4] R44t R t Vt V;t#) BltRotaryEmbeddingdevicez5.18)versionc <V^8dQhRS[/#rKr)r)rOrGs"r6rPBltRotaryEmbedding.__annotate__[sOOyOr8c<\SV`4VPVnVPVnWnVP P R,VnVPpVPR8wd\VP,pV!VP V4wq@n \P!VRR7Vn \P!VP4RR7VnR#) rope_typedefaultF) persistentN)r'r(max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr)rope_parametersrxcompute_default_rope_parametersrattention_scalingr,Bufferinv_freqcloneoriginal_inv_freq)r4r)rq rope_init_fnrr5s&&& r6r(BltRotaryEmbedding.__init__Zs "("@"@$*$B$B! 44[A!%!E!E >>Y &.t~~>L+7 V+L(( (u= !#8>>+;!Nr8cP<V^8dQhRS[RS[S[PS[3,/#)rKr)rM)rrjrTr\rN)rOrGs"r6rPrvms.55 5UZ[`[g[gin[nUo5r8c 0VPR,p\VRR4;'gVPVP,pRpRV\P !^V^\P R7V, ,, pVPV4V3#)a~ Computes the inverse frequencies according to the original RoPE implementation Args: config ([`~transformers.PreTrainedConfig`]): The model configuration. Returns: Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). rope_thetahead_dimN?r`)r~getattrr*num_attention_headsrTarangerNra)r)rqkwargsbasedimattention_factorrs&&, r6r2BltRotaryEmbedding.compute_default_rope_parametersks~%%l3fj$/cc63E3EIcIc3c$5<<3#MPS#STU{{6"$444r8cbVPR,P4PVP^,R^4pVR ,P4p\ VP P \4'd2VP P R8wdVP P MRp\VRR7;_uu_4VP4VP4,P^^4p\P!V^RR7pVP4VP,pVP4VP,p RRR4XPVP R7X PVP R73# +'giLH;i) NmpscpuF) device_typeenabledrr)NNNNNr_)rNr)rrNexpandrk isinstancerqtypestrr transposerTrepeat_interleavecosrsinrar`) r4r; position_idsinv_freq_expandedposition_ids_expandedrfreqsembrrs &&& r6r<BltRotaryEmbedding.forwards5!MM-8>>@GG HZHZ[\H]_acde ,Z 8 > > @'1!((--'E'E!((--[`J`ahhmmfk  U C C&,,.1F1L1L1NNYYZ[]^_E))%;C'')d444C'')d444C D vvAGGv$cff177f&;;; D Cs BF F. )rr)rr|rr}rxr:)r?r@rArBrr( staticmethodrrTno_gradrr<rCrDrErFs@@r6rprpYsuXv.OO/O Xv.55/5$ ]]_ < < )&*<*<&BUBUV(263E3E6K^K^(_%"r8c2<V^8dQhRS[PRS[PR,RS[PR,RS[PR,RS[S[PS[P3,R,RS[PR,RS[R,R S[R,R S[S[PS[P3,R,R S[S[,R S[S[PS[S[PS[P3,R,3,/ #) rKrZcross_attention_statesNcross_attention_maskattention_maskfull_text_row_masked_out_maskrpast_key_values use_cacheposition_embeddingsrrM) rTr\rj LongTensorrboolrr FloatTensor)rOrGs"r6rPrs 55||5!& t 35$llT1 5  t+ 5 (-U\\5<<-G'H4'O 5&&-55$;5#5<<#=>E5-.5 u  %(9(95;L;L(L"MPT"TT U5r8c  Tp VPV4pVP!RRVRVRVRVRVRV /V BwrW,pTp VPV4pVPV4pW,pV#)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1, query_sequence_length, key_sequence_length)` if default attention is used. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). past_key_values (`Cache`, *optional*): cached past key and value projection states position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, with `head_dim` being the embedding dimension of each attention head. kwargs (`dict`, *optional*): Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code into the model rZrrrrr)rrrr) r4rZrrrrrrrrrresidualself_attn_weightss &&&&&&&&&&, r6r<BltTransformerLayer.forwards>!,,]; ,0>>, ', ), &, , ,  , !4 , , ( !0 !55mD /  0 r8)r*rrrrr)NNNNNNFNr>rFs@@r6rrs # #555r8rcdV^8dQhR\PR\R\P/#)rKrZn_reprMrTr\r)rOs"r6rPrPs. U UU\\ U# U%,, Ur8cVPwr#rEV^8XdV#VR,PW#WV4pVPW#V,WE4#)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) )rrNrr)rkrreshape)rZrbatchnum_key_value_headsslenrs&& r6 repeat_kvrsU 2?1D1D.E z!"23::5W\dlmM  e(CT TTr8cV^8dQhR\PR\PR\PR\PR\PR,R\R\R \ \ ,/#) rKmodulequerykeyvaluerNscalingdropoutr)r,ModulerTr\rNrr)rOs"r6rPrPsw%% II% <<% % << % LL4' %  %%'(%r8c "\W P4p\W0P4p \P!WP ^^44V,p Ve W,p \ P PV R\PR7PVP4p \ P PWVPR7p \P!W4p V P ^^4P4p W3#)rK)rr`ptrainingr_)rnum_key_value_groupsrTmatmulrr, functionalsoftmaxrbrar`rr contiguous) rrrrrrrr key_states value_states attn_weights attn_outputs &&&&&&&, r6eager_attention_forwardrs3 ; ;C/0Gw;q>C/0G  r8cTaa]tRtRtoV3RlV3RlltRV3RlRlltRtVtV;t#)ric&<V^8dQhRS[RS[/#)rKr)rrr)rOrGs"r6rPBltSelfAttention.__annotate__s^^y^S^r8c<\SV`4WnVPVnVP VnVP VnVPVnVP VP,VnVPVP,Vn VPR,Vn W n RVn \P!VP VPVP,RR7Vn\P!VP VPVP,RR7Vn\P!VP VPVP,RR7Vn\P!VPVP,VP RR7VnR#)?TFr%N)r'r(r)r num_headsrr*rrrrr is_causalr,r-q_projk_projv_projo_projrs&&&r6r(BltSelfAttention.__init__s4  33~~ !--#)#=#= **dnn< $(NNd6N6N$N!}}d* "ii 0 0$..4==2PW\] ii 0 0$2J2JT]]2Zafg ii 0 0$2J2JT]]2Zafg ii >@P@PW\] r8ch<V^8dQhRS[PRS[PRS[P/#)rKrZrrr[)rOrGs"r6rPr.s5*)*)||*) *)#\\ *)r8c VP4wrgpVPV4p VPV4p VPV4p V P WgVP VP 4P^^4p V P WgVPVP 4P^^4p V P WgVPVP 4P^^4p Vwr\WW4wrVeVPWVP4wr\P!VPP\ 4pT!VV V V V3RVP"'gRM VP$RVP&/VBwppVP)WgR4P+4pVP-V4pVV3#)rrrr_)sizerrrviewrrrrrupdaterr get_interfacer)_attn_implementationrrrrrrr )r4rZrrrrbszq_len_ query_statesrrrrattention_interfacerrs&&&&&, r6r<BltSelfAttention.forward.s&**, A{{=1 [[/ {{=1 #((T^^T]]S]]^_abc __S1I1I4==Yccdeghi #((T5M5Mt}}]gghiklm &#7 RU#[  &'6'='=jX\XfXf'g $J(?(M(M KK , ,.E) %8      % $}}}C$,, % LL %  % ! \"))#b9DDF kk+. L((r8)r)rrr*rrrrrrr rrrr:r>rFs@@r6rrs ^^&*)*)*)r8rc\aa]tRtRtoRtRV3RlV3RllltR V3RlRlltRtVtV;t #) BltCrossAttentioni[z@P@PW\]  !1!1v7J7JK  !1!1v7J7JK r8c"<V^8dQhRS[PRS[PR,RS[PR,RS[S[,RS[S[PS[PR,S[S[P,R,3,/#)rKrZrNrrrM)rTr\rrrj)rOrGs"r6rPrrs$)$)||$)!& t 3$) t+ $) +, $) u||U\\D0% 2E2LL M $)r8c lVP4wrVpVPV4pVPV4pVPWVVPVP 4P ^^4pVPV4pVPV4p VPV4p V PVRVPVP 4P ^^4p V PVRVPVP 4P ^^4p \P!VPP\4p T !VVV V V3RVP 'gRM VP"RVP$/VBwrV P'WVR4P)4p VP+V 4p W,p W3#)z#Input shape: Batch x Time x Channelrr rr_)rrrrrrrrrrrrrr)rrrrrrrr )r4rZrrrrrrrrrrrrs&&&&, r6r<BltCrossAttention.forwardrs&**, A{{=1 {{<0 #((T^^T]]S]]^_abc !%-C!D[[!78 {{#9: __S"d.F.F V``abdef #((b$2J2JDMMZddefhij (?(M(M KK , ,.E) %8      % $}}}C$,, % LL %  % ! "))#b9DDF kk+. !1 ((r8)r)rrr*rrrrrrrr rrrrr:NN) r?r@rArB__doc__r(r<rCrDrErFs@@r6rr[s!F($)$)$)r8rcaa]tRtRtoRtR tRtRR.tRtRt Rt Rt Rt R] !]^R7R ] !]^R7/t]P$!4V3R l4tV3R ltR tVtV;t#)BltPreTrainedModelimodelTrrFrZ)index attentionsc  <\SV`V4VPPp\ V\ P 4'd\VPRR4pVf>\VPR4'd"\VPPRR4pVf VPpVR,p\P!VPRVRV,^V,R7VPe2\P !VPVP,4R#\ V\"\$34'g VR9Ed\VPRR4pVf\VR4'd VP&pVfMRFFp\WR4pVfK\VR4'gK)VPP(R,pM VfR#VR,pRFp\WR4pVfK\VR4'gK)\P!VPRVRV,^V,R7\VR R4fKm\P !VP*4K \VR\VRR44p V eu\V R4'dc\P!V PRVRV,^V,R7\V R R4e!\P !V P*4R#\ V\,4'g VR 8XEd<\VPRR4pVf>\VPR 4'd"\VPP.RR4pVf>\VPR4'd"\VPPRR4pRp Ve VR,p \VR \VR R44p \VRR4p \VR\VRR44p W3FpVfK \VR4'gKT ;'g%VPP(^,R,p\P!VPRVRV,^V,R7\VR R4fK\P !VP*4K V e\V R4'dV PP(^,pVR,p\P!V PRVRV,^V,R7\V R R4e!\P !V P*4R#\ V\ P04'dyVP2pVR,p\P!VPRVRV,^V,R7VP*e!\P !VP*4R#R#)z Initialize BLT weights following the original ByteLatentTransformer: - Most weights are drawn from a truncated normal. - Scale is ~ 1 / sqrt(model_dim) (or 1 / sqrt(hidden_dim) for FFN outputs). - Norm layers are set to weight = 1, bias = 0. r*Nencoder_configr )rdstdabr denserVr& MllamaTextMLPdecoder_configr.fc1r/r0fc2r)MllamaTextSelfAttentionMllamaTextCrossAttention)rrrr r0r_)rrr)r' _init_weightsr5r?rr, Embeddingrr)hasattrr, embedding_diminit trunc_normal_rV padding_idxzeros_rrr*rkr&r"r2r- in_features)r4r class_namer*r-rnameproj proj_namer in_stdr.r/r0 hidden_dimout_stdfan_inr5s&& r6r8 BltPreTrainedModel._init_weightss f%%%..  fbll + +!$++}dCK"wt{{  f/1BC D D W I $++}d;C{wv}==(({MD"66D'GD(,C,C"kk//3 N {t)C< v$7#h(?(?&&  s(c' tVT2> DII.<VXwvw/MNF!gfh&?&?""MM3h#g 6640<KK ,  ff % %)F!$++}dCK"wt{{ DII.-$H)E)E&--33A6 $d*""$$7l'k 9fd3?KK /  fbii ( (''F$,C    s(c'  {{& FKK(  )r8c&<V^8dQh/S[;R&#rtru)rOrGs"r6rPBltPreTrainedModel.__annotate__s r8r)imagetext)r?r@rArBbase_model_prefixinput_modalitiessupports_gradient_checkpointing_no_split_modules_can_compile_fullgraph_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backendrrr_can_record_outputsrTrr8__annotate_func__rCrDrErFs@@r6r'r's (&*#.0CD"N "'(;1En%5Q?  ]]_JJ%r8r'caa]tRtRtoR]!]^RR7/tV3RlV3RlltR V3RlRlltR t V3R lt R t Vt V;t #) BltLocalEncoderi8encoder_attentions local_encoderr) layer_namec <V^8dQhRS[/#rtr)rOrGs"r6rPBltLocalEncoder.__annotate__>s4r8c <\SV`V4RVnWn\P !\ VP4Uu.uFp\W4NK up4Vn \VR7Vn \P!VPVPVP,RR7Vn\P !VP"VP4Vn\P !4VnVP('d VPM^p\ V4F3pVP&P+\-WVPR74K5 VP/4R#uupi)Fr)r@ out_featuresr&r)rr*N)r'r(gradient_checkpointingr)r, ModuleListrangenum_hidden_layersrlayersrp rotary_embr-r* cross_attn_kpatch_embedding_projectionr9 vocab_size embed_tokenscross_attn_layerscross_attn_all_layersappendr post_initr4r)r layers_to_addr5s&& r6r(BltLocalEncoder.__init__>s  &+# mmEJ6KcKcEd eEd  3Ed e  -F;*,))**++f.A.AA+ ' LL):):F>|L +33 2 21 5 8P8P PRVR]R]RiRi  $(;;#D#D#DC! ,0,B,B9,M-".-+8-$:- -)& ,.D.G.G H[H[.\\ +1, ,222r8c ^VP^,pVPR,pVPR4PRRVPR,4p\P!WBV3VP VP R7pVPV^VRRR7pVRRV1R3,pV#)a> Reduce variable length patches to single embedding per patch Note: this works with variable number of patches for different sequences in the batch It handles variable length patches by assuming that patch_lengths will be 0 for any extra patches on the *right*. Since there can be a variable number of patches this function also return the number of patches for each sequence in the batch. Any embeddings on the right that are not allocated to a patch (i.e. if the sum(patch_lengths[i]) < seq_len for any i) will be sent to a dummy patch, which is trimmed before returning. r`rqamaxF)srcrr)reduce include_selfrNr_)rkrrrTzerosr`rqscatter_reduce)r4rZmax_num_patchesr~rr;reduced_embeddingss&&&& r6rBltLocalEncoder.patch_reduces#((+ %++B/ ''+222r=;N;Nr;RS "[[ - 8 @S@S\i\p\p 0>> ? 03CO3CQ0FG!!r8c&<V^8dQh/S[;R&#rtr`)rOrGs"r6rPra8 !!r8)r)rqrprgrkrnrl NNNNNNNNN)r?r@rArBrrrWr(r<rrXrCrDrErFs@@r6rZrZ8sA n-=QSbc.2323h"cr8rZc`aa]tRtRtoV3RlV3RlltRV3RlRlltV3RltRtVtV;t #) BltLocalDecoderic <V^8dQhRS[/#rtr)rOrGs"r6rPBltLocalDecoder.__annotate__s4r8c <\SV`V4RVnWnRVn\ P !\VP4Uu.uFp\W4NK up4Vn \VR7Vn \ P!VPVPVP ,RR7Vn\%VPVP&R7Vn\ P !4VnVP,'d VPM^p\V4F3pVP*P/\1WVPR74K5 VP34R#uupi)FTrcrdrrfN)r'r(rgr)cross_attn_decoderr,rhrirjrrkrprlr-hidden_size_globalr*rmrnrIrnormrqrrrsrrtrus&& r6r(BltLocalDecoder.__init__s"  &+# "&mmEJ6KcKcEd eEd  3Ed e  -F;*,))11++f.A.AA+ ' v11v7J7JK !#4:4P4P4P00VW }-I  " " ) )!RXRdRde . ! fs E4c4<V^8dQhRS[PR,RS[PR,RS[PR,RS[PR,RS[PR,RS[R,RS[PR,R S[S[,/#) rKryNrzr{rrrr|r)rTrr\rrr)rOrGs"r6rPrs..##d*.||d*.llT) .  t+ . &&- ..!& t 3.+,.r8c VP^,p Tp VPV4pVPWP^,VPP,VPP 4pVeVP 'g W,p VfS\P!VP^,VPR7P^4PV R4pVPW4p \P!WPPVPR7p \!VP"4FywrV ^8XgVPP$'dCVP&V ,!R RV RVRV/VBwrWP)V P4,p V !V 3RV RVRV/VBp K{ VP+V 4pV#) rrrrZrrrrr_r)rkrnrr)rmr*rrTrrqrrrlrrrrrkrrrqrar)r4ryrzr{rrrr|rrrZrirrrlogitss&&&&&&&&, r6r<BltLocalDecoder.forwards#((+ % 66|D #++ **1- 0H0HH$++JaJa   #D,C,C,C)8M   ]003M+>+>r+BHMH$$Z4MaG,,}2215m>R>RSVdVhVh WiWimK&&}5O##Kw1GH.$$$+//16689G 1hwh; ''r8ctaa]tRtRtoV3RlV3RlltR V3RlRllt]R Rl4tV3RltRt Vt V;t #) BltPatcheriWc <V^8dQhRS[/#rtr )rOrGs"r6rPBltPatcher.__annotate__Zs/r8c<\SV`V4\VPR7Vn\ P !4Vn\VPP4F2pVPP\VPV44K4 \ P!VPPVPP4Vn\!VPPVPP"R7Vn\ P&!VPPVPPRR7VnVP+4R#)rcrFr%N)r'r(rpr)rlr,rhrkrirjrsrr9ror*rprIrrr-lm_headrtrs&& r6r(BltPatcher.__init__Zs  ,DKK@mmo t{{<<=I KK  24;; J K>LL)?)?AXAXYt{{66DKKO  CRC^==?de  << (;(;A(>}G[G[\_ooL'11!4L(;;')+%  & "oomJ[[E!-ituM!dii 67$22>>f>MUUW&3&9&9"&=# O  ! ==. /%# >M"JJ_-]5H5HQ^QeQeM.m=MN #]F::r8c VP^,p\P!^^.\PVPR7P ^4P V^4pVP^,pVR,pW8pVP^,p\P!WPR7P ^4PVR4p \P!W4p \P!W.^R7p \P!Ww(.^R7p W,PWH4p VP^R7P4pV RRV13,p \P!W]V,3^R7p\P!VR,V^, 4p\P!VR,^, V3^R7pVV, ^,pV#) z Computes patch lengths from token entropies. Depending on whether a threshold is provided, the function uses either: - Thresholding the entropy values (when `threshold` is set). rrrrNr:rNNr_)r:NrN)rkrTtensorlongrqrrepeatrrrcatrrr)rrrrr init_tokensoffset patch_maskr token_indicessentinelpadded_indices padded_mask patch_startsmax_valid_patchespatch_start_ids last_token patch_endsrs&&&& r6r'BltPatcher.patch_lengths_from_entropiess__Q'  LL!Quzz):J:J K U UVW X _ _`jlm n ""1%e$ * ""1% W5E5EFPPQRSZZ[egij ??=:M#>> tokens = torch.tensor([[1, 2, 3], [4, 5, 6]]) >>> hashes = rolling_polynomial_hash(tokens, prime=31) >>> # hash[0] = 1*31^0 + 2*31^1 + 3*31^2 >>> # hash[1] = 4*31^0 + 5*31^1 + 6*31^2 rrrr_)rTrint64rqrrkr) token_tensorr prime_tensorpowers prime_powerss&& r6rolling_polynomial_hashr s].<<U[[ATATUL \\,,,R09L9L MF'L 99\0b 99r8c\V^8dQhR\PR\R\R\/#)rK token_ids group_sizermax_hashr)rOs"r6rPrPs0||),9<UXr8c \P!4;_uu_4VPwrE\P!WA^, \PVP R7p\P !W`.^R7pVP^V^4p\W4p W,p RRR4V # +'giX #;i)z1Hash token groups and map to range [0, max_hash].rrN) rTrrkrrrqrunfoldr ) r rrrrrpadding padded_tokenswindowshashes hash_valuess &&&& r6byte_group_hash_functionrs 'oo ++jq. T]TdTde 7"6A>  &&q*a8(8'     s BB.. B? c V^8dQhR\PR\PR\R\ R\R\P/#)rKlocal_encoder_tokensencoder_hash_tok_embedding$encoder_hash_byte_group_nb_functionsencoder_hash_byte_group_sizeencoder_hash_byte_group_vocabrM)rTr\r,r9rlist)rOs"r6rPrP sR##,,#!# #+. # #' # $' # \\#r8c0.ROpVPV4p^p\V4Fop Wi\V4,,p VFNp \W W4p WV,,p Wr!V 4P VP 4, pV^, pKP Kq V#)z=Compute token embeddings enhanced with hash-based embeddings.) ʚ;l21Ai oYlvtl.l}glAul 0lTlAK l|)rprirrrarq)rr\rrrrprimes embeddings embedding_idxfunc_nbrrhash_idsoffset_hash_idss&&&&&& r6compute_hash_embeddingsr' s F++,@AJM=>V,-6J/0DRWwH&9V)VVO 4_EHHIZIZ[ [J Q M 7? r8FcV^8dQhR\PR\R\R\R\R\PR\ \P\P3,/#)rKr~r}rpatches_as_queriesrmr`rM)rTr\rrr`rj)rOs"r6rPrP1sqK K ||K K K  K  K ;; K  5<< %&K r8cVPwrgVPpV'drW,p Tp \P!WR7P ^4P R4P WaV4p VP ^4P WaV4p MpTp W,p VP R4P WgV4p \P!WR7P ^4P ^4P WgV4p W8Hp V'd^MRpV P WNR7p WiV 3pV PV8wd\RV P RV 24hV P ^4p RV PV4, pVPVP\P4\P!V4P4p V #)a Prepare cross-attention mask for patch-based attention, following mllama's robust approach. This function creates masks that control which patches can attend to which other patches, with support for query/key role swapping and cross-attention multipliers. Args: patch_ids (torch.Tensor): Tensor of shape [batch_size, seq_len] containing patch ids. num_patches (int): Total number of patches. sequence_length (int): Length of the sequence. patches_as_queries (bool): If True, patches are used as queries, otherwise as keys. cross_attn_k (int): Cross-attention multiplier for repeating patches. dtype (torch.dtype): Data type for the output mask. Returns: Tuple[torch.Tensor, torch.Tensor]: - cross_attention_mask: 4D tensor [batch_size, 1, q_len, kv_len] rrzCross attention mask shape z doesn't match expected rr_) rkrqrTrrrrrra masked_fillrfinfomin)r~r}rr)rmr`rrrqrkv_len q_patch_ids kv_patch_idsr repeat_dimexpected_shapeinverted_cross_attn_masks&&&&&& r6#_prepare_patch_cross_attention_maskr41s4$//J   F*  LL 4 Yq\ Yr] VJW 5  !**1-44ZgV +))"-44Z+V LL 4 > >q A K KA N U UV`kv w  '6)bJ/AA,A_!0N!!^3)*>*D*D)EE]^l]m n  099!< #%9%<%rpatching_thresholdrrHrqr`rrTrU_patch_ids_from_lengthsrrrr self_attention_cacher4rmrrr;cross_attention_cacher )r4ryrrrrrzrrencoder_embedsrrrrqr`r~rrcross_attn_mask_encencoder_hidden_statesrglobal_position_idsglobal_causal_maskglobal_hidden_statesdecoder_patch_idscross_attn_mask_decoutputs&&&&&&&&, r6r<BltModel.forwards  -t"; < <YZ Z &"5  4l$++6V# 1DEE#6o|[_[f[fGg"h  $*N-:-@-@ *J*3// 'J4""// @@ 88 99 N  {{((I5$,,:R$$%WXX&*ll#{{55"kk<<%)[[%A%A(, (G(G$++ '3'#!.7-B)) H\H\+4+@ mFYFY 5JJ a,?@]KK00! 00P  CRC^==?de  <<(<(/;;.    $66 . - -    !88u9M_A'%++A.+$11 &&  ##    /  .  '  &  FUE`OAAfj  $7    '$+  r8c.VPP#r:r\rprls&r6get_input_embeddingsBltModel.get_input_embeddingss!!...r8c&WPnR#r:rZ)r4rs&&r6set_input_embeddingsBltModel.set_input_embeddingss*/'r8cT<V^8dQhRS[PRS[RS[P/#)rKrrrMr)rOrGs"r6rPr8!s2 i iU\\ iC iTYT`T` ir8c VP^,p\P!\P!V^VPVP R7VP RR7RRR13,.RR7p\P!W!P R7pVP^4VP^4PR48*PRR7^, #)rrrrNrr_) rkrTrrr`rqcumsumrrr)r4rrrrtoken_positionss&&& r6rL BltModel._patch_ids_from_lengths!s"((+ yy J1D1D]MaMab$$$,QV4    ,,w7K7KL&&q)_-F-Fq-I-S-STV-WW\\ac\dghhhr8)r)rrrgr;r\r>r)r?r@rArBr(rrr<r[r^rLrCrDrErFs@@r6r6r6sP( C C  C J/0 i i ir8r6zB The Blt Text Model with a language modeling head on top. ) custom_introcaa]tRtRtoRtRtRR/tV3RlV3Rllt]] R V3RlR ll44t V3R lt R t Vt V;t#) BltForCausalLMi.Fr(z'model.local_encoder.embed_tokens.weightzlm_head.weightc <V^8dQhRS[/#rtru)rOrGs"r6rPBltForCausalLM.__annotate__9syr8c*<\SV`V4VP4VnVPVn\ V4Vn\P!VPPVPRR7Vn VP4R#r$) r'r(get_text_config text_configror6r(r,r-r2r*rrtr3s&&r6r(BltForCausalLM.__init__9sk  !113 ++f% yy!6!6!B!BFDUDU\ab  r8c<V^8dQhRS[PR,RS[PR,RS[PR,RS[PR,RS[PR,RS[S[PS[P3,R,RS[R,R S[P R,R S[PR,R S[R,R S[S[P,R S[S[ ,RS[S[ ,/ #)rKryNrrrrrrrzlabelsrlogits_to_keeprrM) rTrr\rjrrrrrrr)rOrGs"r6rPriDsV V ##d*V  t+V &&- V !& 0 04 7 V $..5 V (-U\\5<<-G'H4'OV V ((4/V   4'V $;V ell*V +,V  ' 'V r8c  VP!R RVRVRVRVRVRVRVRV /V Bp V Pp\V \4'd\ V )R 4MT pVP VR VR 3,4P 4pR pV eVP!VWP3/V Bp\VVV PV PV PR 7#) a cross_attention_states (`torch.FloatTensor`, *optional*): Output of the vision model, used for cross-attention. This tensor contains the processed image features that the language model will attend to. cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*): Cross-attention mask to control the interaction between text tokens and image tiles. This 4D tensor defines which image tiles each text token should attend to. For each text token (in seq_length): - 1 indicates the token **should attend** to the corresponding image tile - 0 indicates the token **should not attend** to the corresponding image tile full_text_row_masked_out_mask (`tuple[torch.Tensor, torch.Tensor]`, *optional*): A tuple containing two tensors that mask out rows in the cross-attention mechanism: - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1. A value of 0 indicates that the corresponding text token's entire row in the cross-attention matrix should be masked out (all image tokens ignored). - The second tensor has the same shape and is used internally to apply the masking during the forward pass of cross-attention layers. This mask is derived from the cross_attention_mask and is used to handle cases where a text token should not attend to any image token. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, BltForCausalLM >>> model = BltForCausalLM.from_pretrained("itazap/blt-1b-hf") >>> tokenizer = AutoTokenizer.from_pretrained("itazap/blt-1b-hf") >>> prompt = "If I had to write a haiku, it would be:" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=40, do_sample=True, temperature=0.6) >>> result = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] >>> print(result) If I had to write a haiku, it would be: "Snowflakes gently fall" - simple, yet peaceful. I love the idea of snowflakes gently falling, each one ``` ryrrrrrrzrNr)lossrrrZr*r) r(rIrrslicerrN loss_functionrorrrZr*)r4ryrrrrrrrzrorrproutputsrZ slice_indicesrrrs&&&&&&&&&&&&, r6r<BltForCausalLM.forwardBs |**    )  &  "6  +H  ,  (        11 8B>SV8W8W~ot4]k mA}a,?@AGGI  %%ffooPPD%#33!//))   r8c&<V^8dQh/S[;R&#rtru)rOrGs"r6rPri.s  r8)rr(rlro) NNNNNNNNNNr)r?r@rArBrRrN_tied_weights_keysr(rrr<rXrCrDrErFs@@r6rgrg.sT#CEUVV V V -r8rg)r )r)r )rKr i0u)r'r6rrg)Pcollections.abcrrTtorch.distributionstorch.nnr,torch.nn.functionalrrrr< activationsr cache_utilsrrr generationr masking_utilsr modeling_flash_attention_utilsr modeling_layersr modeling_outputsr rmodeling_rope_utilsrrmodeling_utilsrrprocessing_utilsrutilsrrrutils.deprecationr utils.genericrrutils.output_capturingrrconfiguration_bltrrrrr rr"rIrprrrrrrrr'rZrrrrr rr'rbr4r6rg__all__rr8r6rs*% &!CC)/B9OKF&II0GERYY"JJ(3<3)ryy>)B;) ;)|[[[|n"(n"bI(IX0-0f,(^B#BJ::$#T % K \li!li^ g 'g  g T Mr8