+ jRt^RIt^RIHt^RIt^RIHuHt^RIHt^RI H t H t H t ^RI Ht^RIHt^RIHtHt^RIHt^R IHtHt^R IHt^R IHtHt^R IH t ^R I!H"t"H#t#H$t$^RI%H&t&H't'^RI(H)t)^RI*H+t+H,t,H-t-H.t.^RI/H0t0^RI1H2t2H3t3H4t4^RI5H6t6^RI7H8t8].Pr!]:4t;!RR]Px4t=RRlt>R@RRllt?!RR]Px4t@RtA]!RR4RAR R!ll4tB]!R"R4RAR#R$ll4tC]!R%R&4RBR'R(ll4tD]!R)R&4RCR*R+ll4tE]!]E]D]C]B.4!R,R-]Px44tF!R.R/]Px4tG!R0R1]Px4tH!R2R3] 4tI!R4R5] 4tJ],!R6R7]'44tK],!R8R9]K44tL!R:R;]K]4tM],!RR?]K44tN.RDOtOR#)EzPyTorch Zamba model.N)Callable)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)ACT2FN)Cache DynamicCache)GenerationMixin)&use_kernel_func_from_hub_with_fallbackuse_kernelized_func)force_accelerate_hooks)create_causal_maskcreate_recurrent_attention_mask)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast SequenceClassifierOutputWithPast)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)merge_with_config_defaults)is_mambapy_availableis_torch_greater_or_equal is_tracing)capture_outputs ZambaConfigcZaa]tRt^7toRV3RlV3RllltV3RlRltRtRtVtV;t #) ZambaRMSNormc$<V^8dQhRS[RR/#)epsreturnN)float)format __classdict__s"i/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/zamba/modeling_zamba.py __annotate__ZambaRMSNorm.__annotate__8s$$$$$c <\SV`4\P!\P !V44VnW nR#)z+ ZambaRMSNorm is equivalent to T5LayerNorm N)super__init__r Parametertorchonesweightvariance_epsilon)self hidden_sizer' __class__s&&&r,r2ZambaRMSNorm.__init__8s/ ll5::k#:; #r/cN<V^8dQhRS[PRS[P/#)r& hidden_statesr()r4Tensor)r*r+s"r,r-r.@s#;;U\\;ell;r/c:VPpVP\P4pVP ^4P RRR7pV\P !W0P,4,pVPVPV4,#)r&T)keepdim) dtypetor4float32powmeanrsqrtr7r6)r8r= input_dtypevariances&& r,forwardZambaRMSNorm.forward@sw#)) %((7  $$Q',,R,>% H?T?T4T(UU {{]--k:::r/c^\VPP4 RVP 2#)z, eps=)tupler6shaper7)r8s&r, extra_reprZambaRMSNorm.extra_reprGs* ))*+6$2G2G1HIIr/)r7r6)gư>) __name__ __module__ __qualname____firstlineno__r2rJrO__static_attributes____classdictcell__ __classcell__r:r+s@@r,r$r$7s%$$;;JJr/r$cdV^8dQhR\PR\R\P/#)r&r=n_repr()r4r>int)r*s"r,r-r-Ls. U UU\\ U# U%,, Ur/cVPwr#rEV^8XdV#VR,PW#WV4pVPW#V,WE4#)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) )NNNr]Nr]r])rNexpandreshape)r=rZbatchnum_key_value_headsslenhead_dims&& r, repeat_kvrdLsU 2?1D1D.E z!"23::5W\dlmM  e(CT TTr/cV^8dQhR\PR\PR\PR\PR\PR,R\R\/#) r&modulequerykeyvalueattention_maskNscalingdropout)rModuler4r>r))r*s"r,r-r-Xsg%% II% <<% % << % LL4' %  %%r/c "\W P4p\W0P4p \P!WP ^^44V,p Ve W,p \ P PV R\PR7PVP4p \ P PWVPR7p \P!W4p V P ^^4P4p W3#)r&)dimrB)ptrainingrA)rdnum_key_value_groupsr4matmul transposer functionalsoftmaxrDrCrBrlrq contiguous) rfrgrhrirjrkrlkwargs key_states value_states attn_weights attn_outputs &&&&&&&, r,eager_attention_forwardr}Xs3 ; ;$>&B\B\$\!'-'E'E$ )d2 !'!9!9ii < X>X[_[h[h>hotu ii < X>X[_[h[h>hotu ii < X>X[_[h[h>hotu ii : :T]] JFL^L^ejk r/c<V^8dQhRS[PRS[RS[PR,RS[R,RS[S[,RS[S[PS[PR,S[S[P,R,3,/#)r&r=rrjNpast_key_valuesrxr()r4r>r[r rrrM)r*r+s"r,r-rs#)#)||#)#) t+ #)  #) +, #) u||U\\D0% 2E2LL M#)r/c VPRRp.VORNVPN5pVPV4PV4P ^^4pVP V4PV4P ^^4p VP V4PV4P ^^4p VeVPWV4wr\P!VPP\4p T !VVV V V3RVP'gRM VPRVP/VBwrV P !.VORN5!P#4p VP%V 4p W3#)NrlrkrA)rNrcrviewrtrrupdater get_interfacer_attn_implementationr}rqrrkr_rwr)r8r=rrjrrx input_shape hidden_shape query_statesryrzattention_interfacer|r{s&&&&&, r,rJZambaAttention.forwardsj$))#2. 88b8$--8 {{=166|DNNqRST [[/44\BLLQPQR {{=166|DNNqRST  &'6'='=jXa'b $J(?(M(M KK , ,.E) %8      % $}}}C$2H2H % LL %  % ! "));;;;FFH kk+. ((r/) rrrrcrrrrrrrrrkrN) rQrRrSrT__doc__r2rJrUrVrWrXs@@r,rrqs% ll$#)#)#)r/rcbVe+VPpWR,,PV4pV#)ze Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66 )r]r]N)rBrC)r=rjrBs&& r,apply_mask_to_padding_statesrs3 !##& )CCGGN r/causal_conv1d_update causal_conv1dc V^8dQhR\PR\PR\PR\PR,R\R,/#)r&r= conv_stater6rN activationr4r>rr3str)r*s"r,r-r-sS''<<' ' LL' ,,  ' d 'r/cVPwrVpVPR,p\P!W.RR7PVP4p VP V RRV)R13,4\ P!WP^4V^VR7p V RRV)R13,p Ve\V,!V 4p V PVP4#)ror]N)paddinggroupsrA) rNr4catrCrBcopy_Fconv1d unsqueezer) r=rr6rr_r9seq_len state_lenhidden_states_newouts &&&&& r,rrs,11AG  $I :"=2FII&,,W&q!iZ['89: (($&6&6q&94S^ _C aWHIo CZ % 66-%% &&r/causal_conv1d_fnc V^8dQhR\PR\PR\PR,R\R,/#)r&r=r6rNrr)r*s"r,r-r-sE''<<' LL' ,, 'd 'r/c VVPwrVpVPR,^, p\P!VPVP4VP ^4VVVR7RRRV13,p Ve\ V,!V 4p V PVP4#)r)r6rrrr]NrA)rNrrrCrBrr) r=r6rrrxrr9rrrs &&&&, r,rrs,11AGll2"G ((&"    HWHn  CZ % 66-%% &&r/selective_state_update mamba_ssmcnV^8dQhR\PR\PR\PR\PR\PR\PR\PR,R \PR,R \R \PR,/ #) r&stater=dtABCDNdt_bias dt_softpluszr4r>bool)r*s"r,r-r-s** <<*<<*  * || *  || *  || * ||d*\\D ** ||d*r/c  VPp Ve"W'PVP4,pV'd\P!V4p\P !VP 4R,VP 4,4PVPR7p VP 4R,VP 4R,,p WP 4R,,pVP 4V ,V,pVPVPVP44\P!VPVP4VPR44PR4pVeVW,,pV eV\P!V 4,pVPV 4#)Ndevice.N)r]Nr]rA) rBrCrsoftplusr4expr)rrrsrsqueezesilu)rr=rrrrrrrrrxrHdAdBdBx ssm_staters&&&&&&&&&&, r,mamba_selective_state_updatersE" %%K **RXX& & ZZ^ 288:i(17794 5 8 8 8 MB I :!6 6B ""$Y/ /C "S(I KK U[[)* ,,y||AGG,akk"o > F Fr JC }M%%}AFF1Io 66+ r/selective_scan_fncrV^8dQhR\PR\PR\PR\PR\PR\PR,R\PR,R \PR,R \R \R \R \/ #)r&r=rrrrrNr delta_biasdelta_softplusreturn_last_state use_mambapyuse_associative_scanr)r*s"r,r-r-#scc<<c c ||c || c  || c  ||d c ||dc t#cccccr/c  \R4'd^RIHp MRp \4'd^RIHpMRpVP wpppVPpVe)WPVP4R ,,pV'd\P!V4pVP^^4pVP^^4p\P!VR ,VR ,,4pVR ,VR,P4,pVVR ,P4,pV 'dpVelV!VP^^4VP^^44pVVPR4,P!^4P^^4pVR,pEMV 'dV e\#V4'dRpVP$P&R9dRMRpV !VVV3^VR7wpp\P(!VP^^4PV4VPR44pVP!R4P^^4pVR,pM\P*!VVVP R,VVP$R 7p.p\-V4F~pVRRV3,V,VRRV3,,p\P(!VPV4VRVR3,PR44pVP/VR,4K \P0!VRR 7pVeVWR,,,pVeV\P2!V4,pV 'dVV3#V#)z2.9.0)associative_scanN)pscanr]c@Vwr#VwrEW$,WC,V,3#r)leftrighta_leftb_lefta_rightb_rights&& r, combine_fn(mamba_selective_scan..combine_fnVs'!NF$ G#W%5%?? ?r/ pointwisegeneric)ro combine_mode)rBrrr)Nr]Nr])r]r]r]N)r]Nr]r]rA)r]rA)cudaxpu)r]r]rA)r]r])Nr]N)r(torch._higher_order_ops.associative_scanrr mambapy.pscanrrNrBrCrrrtr4rr)rrrrtyperszerosrangeappendstackr)r=rrrrrrrrrrrrxrr batch_sizeintermediate_sizerrH discrete_A discrete_BdeltaB_u all_states scan_outputrrrr scan_outputsindexs&&&&&&&&&&&&, r,mamba_selective_scanrs(!))M'-:-@-@*J!7%%K --))4 4 ZZ^ AqA AqA1-.M1BBCJM"Q}%5%;%;%==JM-8>>@@Hu(://15x7I7I!Q7OP !AKKO3<:mC\C\ @ '1&7&7&<&<&O{U^ (   "%  :ll:#7#71#=#@#@#Mq{{[]_ !))"-771= x( KK   GGBK ''   7^E"1a;/);hq!U{>SSI ,,y||K'@!AuaK.BZBZ[]B^_K    G 4 5 $kk,B7 }!Mm4D$DD }!AFF1I- I%% r/cnaa]tRtRtoRtV3RlV3Rllt]!R4R V3RlRll4tRtVt V;t #) ZambaMambaMixeriu! Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`. A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective) ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4, and is why Mamba is called **selective** state spaces) This module differs from `transformers.models.mamba.modeling_mamba.MambaMixer` in two ways: - Added multi-head: the output of `self.in_proj` is split into `self.n_mamba_heads` heads, and each head undergoes an independent forward pass, identical to the original `MambaMixer`, up until the pre-activations of `self.out_proj`. The pre-activations, coming from different mamba heads, are then concatenated and fed into `self.out_proj`. c <V^8dQhRS[/#r&rr!)r*r+s"r,r-ZambaMambaMixer.__annotate__s6868{68r/c f<\SV`4WnW nVPVnVP VnVPVnVPVP,Vn VPVn VPVn VPVP,VnVPVnVP"Vn\&P(!VPVPVP VPVPVP^, R7VnVP,Vn\0VP,,VnVP4Vn\&P8!VPVP^,VP$R7Vn\&P<!\>P@!VPVPVP ^,,VP44Vn!\&P<!\>P@!VPVPVP4R, ^,VPR,, 4Vn"\&P<!\>P@!VPVP44Vn#\>PH!^VP ^,\>PJR7R,pVPMVPR4PO4p\&P<!\>PP!V4PSVPVPR44Vn*\&P<!\>PV!VPVP44Vn,\&P8!VPVPVP$R7Vn-VP\V,Vn/R#)r) in_channels out_channelsr kernel_sizerrrg?rBNNr]rA)0r1r2rrr9 mamba_d_statessm_state_size mamba_d_convconv_kernel_size mamba_expandr mamba_dt_ranktime_step_rank n_mamba_headsmamba_head_dimmamba_conv_bias use_conv_biasmamba_proj_biasuse_biasrConv1drhidden_mamba_actrractuse_mamba_kernelsuse_fast_kernelsrin_projr3r4r x_proj_weightdt_proj_weight dt_proj_biasarangerDr^rwlogr_A_logr5rout_proj layer_types layer_type)r8rrrr:s&&& r,r2ZambaMambaMixer.__init__s  "!--$22 & 3 3!'!4!4v7I7I!I$22#11"448J8JJ#33.. ii..//##--))))A-   !11&112 & 8 8yy!1!143I3IA3MTXTaTab  \\ KK""##d&9&9A&==##  !ll [[++T-@-@$BUBU VY\ \ !!3& '  LLT5G5GI\I\)]^ LLD//!35== I' R HHT++R 0 ; ; =\\%))A,"6"6t7I7I4K^K^`b"cd ejj););T=P=PQR $"8"8$:J:JQUQ^Q^_  ,,Y7r/rcp<V^8dQhRS[PRS[R,RS[PR,/#)r&r= cache_paramsNrj)r4r>r LongTensor)r*r+s"r,r-r sAI%I%||I%dlI%((4/ I%r/c F VPwrVpVPpVRJ;'dVPVP4p \ W4pVP V4P ^^4p V PVR ^V4P^^R7wrV P^4P4p V PWPPVPV4P ^^4p V 'd]VPVP,P^,p VPVP,P ^,pV 'dV^8XdVPVP,P"'gS\%V X VP&P(P^4VP&P*VP,R7p MVe)VP/V VPVP0R7p \3V VP&P(P^4VP&P*VP,VP5R4R7p VeV RRV)R13,p \ V P ^^4V4p V P ^^4p V PVVPVPV4P ^^4p VP6R ,V ,P R R4p\8P:!VVP<VP>VP>.R R7wpppVP@R ,VP R R4,pVPBeVPBPE4MRp\8PF!VPHPE44)pV 'dV^8Xd.p\KVP4Fp\MXRV3,V VR^3,VVR^3,VV,VVR^3,VVR^3,VPNV,V VR^3,Ve VV,MRRR 7 PQR 4pVPSV4K \8PT!V^R7pEMGVRJp.p.p\KVP4Fp\WV V,VV,VV,VV,P ^^4VV,P ^^4VPNV,PE4V V,Ve VV,MRRVR R R 7 pV'd*VwppVPSV4VPSV4KVPSV4K \8PT!V^R7pV'd5\8PX!V^R7pVP[VVP4VP]VP ^^4P_V44pV#)Nr)r)rseq_idx)rr3r].T)rrrF)rrrrrrrrA)r]N)0rNrBhas_previous_staterrr$rtrchunkrrwr_rrlayers conv_statesrecurrent_states record_pastrrr6rrupdate_conv_staterrgetr%r4splitrrr&r'r)rr*rrrrrrrrupdate_recurrent_stater+rC)r8r=r0rjrxrrrrBuse_precomputed_statesprojected_stateshidden_states_B_Cgaterrecurrent_statessm_parameters time_steprrtime_proj_biasrrhead_idxroutput_final_state final_states scan_result head_output final_statecontextualized_statess&&&&, r,rJZambaMambaMixer.forwardss"/!4!4 Q##!-T!9!m!ml>]>]^b^l^l>m5]S << 6@@AF#3"7"7 B7"S"Y"YZ[ab"Y"c-55a8CCE||J(:(:D 33KP!% k.C.CAq.I.L.LU.S T$$r/)r*rr!rrrrr'r&r9r$rrr-rrr+rrrrr#r%NN) rQrRrSrTrr2rrJrUrVrWrXs@@r,rrs< 6868pH%I%I%&I%I%r/rc8aa]tRtRtoV3RltRtRtVtV;t#)ZambaMLPi]c<\SV`4WnVPVnVPVn\ P !VPVPRR7Vn\ P !VPVPRR7Vn\ P !VPVPRR7Vn \VP,Vn R#FrN) r1r2rr9rrr gate_projup_proj down_projr hidden_actact_fnr8rr:s&&r,r2ZambaMLP.__init__^s  !--!'!9!94#3#3T5K5KRWXyy!1!143I3IPUV 4#9#94;K;KRWXV../ r/cVPVPVPV44VPV4,4pV#r)rVrXrTrU)r8xrVs&& r,rJZambaMLP.forwardhs6NN4;;t~~a/@#ADLLQRO#ST r/)rXrrVrTr9rrU rQrRrSrTr2rJrUrVrWrXs@@r,rQrQ]s0r/rQcXaa]tRtRtoRV3RlV3RllltRV3RlRlltRtVtV;t#) ZambaAttentionDecoderLayerimc4<V^8dQhRS[RS[R,/#)r&rrNr)r*r+s"r,r-'ZambaAttentionDecoderLayer.__annotate__ns#ZZ{ZsTzZr/c<\SV`4\W4Vn\ V4Vn\ VPVPR7Vn \ VPVPR7Vn R#)r'N) r1r2r self_attnrQ feed_forwardr$r rms_norm_epsinput_layernormr9pre_ff_layernormrs&&&r,r2#ZambaAttentionDecoderLayer.__init__ns] ':$V,+F,H,HfNaNab ,V-?-?VEXEX Yr/c6<V^8dQhRS[PRS[PRS[RS[PR,RS[R,RS[R,RS[S[,R S[S[PS[S[PS[P3,R,3,/#) r&r=original_hidden_statesrrjNr use_cacherxr( r4r>r[r rrrrM FloatTensor)r*r+s"r,r-rbvs''||'!& ' '  t+ '  '$;'+,' u  %(9(95;L;L(L"MPT"TT U'r/c \P!W.RR7pVPV4pVP!RRVRVRVRVRV/VBwrVP V4pVP V4pV#) a Args: hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)` original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`. This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The concatenated tensor is then used as input of the pre-attention RMSNorm (see fig. 2 in https://huggingface.co/papers/2405.16712). layer_idx (`int`): layer_idx in the forward pass. Used to distinguish Zamba's tied transformer layers. attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch, sequence_length)` where padding elements are indicated by 0. past_key_values (`Cache`, *optional*): cached past key and value projection states use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). rr=rrjrrmrAr)r4 concatenaterhrerirf) r8r=rlrrjrrmrxrs &&&&&&&, r,rJ"ZambaAttentionDecoderLayer.forwardvs2))=*QWYZ ,,]; >> '  * ,       --m< ))-8 r/)rfrhrirer)NNFr^rXs@@r,r`r`ms ZZ'''r/r`cTaa]tRtRtoV3RlV3RlltRV3RlRlltRtVtV;t#)ZambaMambaDecoderLayeric&<V^8dQhRS[RS[/#rr)r*r+s"r,r-#ZambaMambaDecoderLayer.__annotate__s##{#s#r/c<\SV`4\WR7Vn\ VP VP R7VnW nR#))rrrdN) r1r2rmambar$r9rgrhrrs&&&r,r2ZambaMambaDecoderLayer.__init__s: $FH +F,>,>FDWDWX"r/ch<V^8dQhRS[PRS[PR,RS[R,RS[PR,RS[PR,RS[R,RS[R,R S[P R,R S[PR,R S[S[,R S[P/ #) r&r=rlNrrj causal_maskrrm position_idstransformer_hidden_statesrxr()r4r>r[r rr1rr)r*r+s"r,r-rvs**||*!& t 3*: *  t+ * \\D( **$;*&&-*$)<<$#6*+,* *r/c Tp V e W,MTpVPV4pVP!RRVRVRV/V BpW,pV#)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch, sequence_length)` where padding elements are indicated by 0. past_key_values (`Cache`, *optional*): cached past key and value projection states use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). r=r0rjr)rhrx) r8r=rlrrjr{rrmr|r}rxresiduals &&&&&&&&&&, r,rJZambaMambaDecoderLayer.forwardss0! :S9^M 5dq ,,];   ' ( *   !0 r/)rhrrx)NNNNNFNNr^rXs@@r,rtrts## ***r/rtcTaa]tRtRtoV3RlV3RlltRV3RlRlltRtVtV;t#)ZambaHybridLayeric@<V^8dQhRS[RS[PRS[/#)r& shared_transflinearrx)r`rrrt)r*r+s"r,r-ZambaHybridLayer.__annotate__s(##&@#"))#\r#r/cH<\SV`4WnW nW0nR#r)r1r2rr mamba_decoder)r8rrrxr:s&&&&r,r2ZambaHybridLayer.__init__s * "r/cz<V^8dQhRS[PRS[PR,RS[R,RS[PR,RS[PR,RS[R,RS[R,R S[S[,R S[S[PS[S[PS[P3,R,3,/ #) r&r=rlNrrjr{rrmrxr(rn)r*r+s"r,r-rs--||-!& t 3-: -  t+ - \\D( --$;-+,- u  %(9(95;L;L(L"MPT"TT U-r/c VP!V3RVRVRVRVRV/VBp VPV 4p VP!V3RV RVRVRV/VBpV#)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with hidden activations to form the input of the shared transformer layer. layer_idx (`int`): layer number. attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch, sequence_length)` where padding elements are indicated by 0. past_key_values (`Cache`, *optional*): cached past key and value projection states use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). rlrrjrrmr})rrr) r8r=rlrrjr{rrmrxr}s &&&&&&&&, r,rJZambaHybridLayer.forwards2%)$6$6 % #9%  % ' % , %  % % !%)KK0I$J!**  &? * ,      r/)rrr)NNNNNFr^rXs@@r,rrs## ---r/rcaa]tRtRtoRtRtRR.tR.tRtRt Rt R] R] /t ]P!4V3R l4tV3R ltR tVtV;t#) ZambaPreTrainedModeli modelTrrtrr= attentionsc0<VPPp\SV` V4\ V\ 4'Ed\ P!VPRVR7VPPR,p\ P!VPV)V4VPPVPP,VPP,p\P !\P"!VPPV4\$P&!VPP(4\$P&!VPP*4, ,\$P&!VPP*4,4P-VPP.R7pV\P&!\P0!V)4)4,p\ P2!VP4V4\P6!^VP8^,\P:R7R,pVP=VP>R4PA4p\ P2!VPB\P&!V4PEVPVPFR44\ PH!VPJ4R#R#)r)rFstd)minrNrrrA)&rinitializer_ranger1 _init_weights isinstancerinitnormal_r%runiform_r&rr9rr4rrandmathr) time_step_max time_step_minclamptime_step_floorexpm1rr'r(rrDr^rrwr*r_rones_r) r8rfr dt_init_stdrrinv_dtrr:s && r,r"ZambaPreTrainedModel._init_weightsskk++ f% fo . . LL--CS A++33T9K MM&//+{ K![[55 8O8OOSWS^S^SlSllN 4;;44nE88DKK556$++B[B[9\\^((4;;4456e 33e4  %))U[["%5$566F JJv**F 3 Q 5 5 9OPWXA1126AACA JJv||UYYq\%9%9&:N:NPVPePegi%j k JJvxx % /r/c&<V^8dQh/S[;R&#r r!)r*r+s"r,r-!ZambaPreTrainedModel.__annotate__ s r/r)rQrRrSrTbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa _is_statefulrtr_can_record_outputsr4no_gradr__annotate_func__rUrVrWrXs@@r,rr sl &*#+-EF#4"5NL/n  ]]_!!!r/rcvaa]tRtRtoRtV3RlV3Rllt]]]RV3RlRll444t Rt Vt V;t #) ZambaModeli3z Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`ZambaDecoderLayer`] Args: config: ZambaConfig c <V^8dQhRS[/#r r!)r*r+s"r,r-ZambaModel.__annotate__<s{r/cX<\SV`V4VPVnVPVn\ P !VPVPVP4VnVPVn .pRVn \VP4Fwr4\WR7pVR8Xd\ P!VPPVPPRR7pVP\!\#V4We44VPfRV R2RV R2/Vn KKVPV4K \ P$!V4Vn\)VPVP*R 7VnRVnVP14R#) N)rhybridFrz layers.(?!z\.)\d+.shared_transfzlayers.z.shared_transfrd)r1r2 pad_token_id padding_idx vocab_sizer Embeddingr9 embed_tokenslayers_block_type_tied_weights_keys enumeratertrrrrr` ModuleListr7r$rgfinal_layernormgradient_checkpointing post_init)r8rr7layer_idr-rxrr:s&& r,r2ZambaModel.__init__<s]  !.. ++LL):):F,>FDWDWX&+# r/c<V^8dQhRS[PR,RS[PR,RS[PR,RS[R,RS[PR,RS[R,RS[S[,R S[S[ ,/#) r& input_idsNrjr|r inputs_embedsrmrxr() r4r1r>r rorrrrMr)r*r+s"r,r-r[s= = ##d*=  t+= &&- =  = ((4/ = $;= +,=  ( (= r/c VRJVRJ, 'd \R4hVfVPV4pTp\P!V4p V'dVf\ VP R7pVfbVeVP 4M^p \P!VP^,VPR7V ,pVP^4p\T;p \4'g/RVP RVRVRVRV/p R \R /V BR \R /V B/p \VP 4F$wrV!VV V V R ,V R ,3RVR V/VBpK& VP#V4p\%TV'dVR 7#RR 7#)NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either one)rrrrrjrr|full_attentionlinear_attentionrm)last_hidden_staterr) ValueErrorrr4cloner rget_seq_lengthr(rNrrrdictrrrr7rr)r8rrjr|rrrmrxr=rlpast_seen_tokenscausal_mask_mapping mask_kwargsrlayers&&&&&&&, r,rJZambaModel.forwardXs  -t"; < <s    --i8M% !&]!; 0*$++>O  CRC^==?de  << (;(;A(>}G[G[\_ooL'11!4L?-FF$++ .!? K!"4"C{"C"$C$Rk$R#  !*$++ 6 I!&#$67#$45  !0 $  M!7,,]; &+/8O  >B  r/)rrrrr7rrr)NNNNNN) rQrRrSrTrr2rr rrJrUrVrWrXs@@r,rr3sB8 = =  = = r/rc|aa]tRtRtoRR/tV3RlV3Rllt]]R V3RlRll44tV3Rlt R t Vt V;t #) ZambaForCausalLMizlm_head.weightzmodel.embed_tokens.weightc <V^8dQhRS[/#r r!)r*r+s"r,r-ZambaForCausalLM.__annotate__s{r/c<\SV`V4\V4VnVPVn\ P !VPVPRR7VnVP4R#rS) r1r2rrrrrr9lm_headrrYs&&r,r2ZambaForCausalLM.__init__sU  '  ++yy!3!3V5F5FUS  r/c`<V^8dQhRS[PR,RS[PR,RS[PR,RS[R,RS[PR,RS[PR,RS[R,R S[S[P,R S[S[,R S[ S[ ,/ #) r&rNrjr|rrlabelsrmlogits_to_keeprxr() r4r1r>r rorr[rrrMr)r*r+s"r,r-rs@ @ ##d*@  t+@ &&- @  @ ((4/ @   4'@ $;@ ell*@ +,@  ' '@ r/c  |VP!R RVRVRVRVRVRV/V Bp V Pp \V\4'd\ V)R4MTp VP V RV R3,4p RpVe VP !V VVP3/V Bp\VV V PV PV PR 7#) a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, ZambaForCausalLM >>> model = ZambaForCausalLM.from_pretrained("Zyphra/Zamba-7B-v1") >>> tokenizer = AutoTokenizer.from_pretrained("Zyphra/Zamba-7B-v1") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```rrjr|rrrmNr]losslogitsrr=rr) rrrr[slicer loss_functionrrrr=r)r8rrjr|rrrrmrrxoutputsr= slice_indicesrrs&&&&&&&&&, r,rJZambaForCausalLM.forwardsH,0::, , ), &, , , ( ,  , ,  11 8B>SV8W8W~ot4]k mA}a,?@A  %% D&#33!//))   r/c \<VPPVR&\SV` !V3/VBpV#)r)rnum_logits_to_keepr1prepare_inputs_for_generation)r8rrx model_inputsr:s&&, r,r.ZambaForCausalLM.prepare_inputs_for_generations2#';;#A#A wr rorrrrMr)r*r+s"r,r-+ZambaForSequenceClassification.__annotate__ sR R ##d*R  t+R &&- R  R ((4/ R   4'R $;R +,R  1 1R r/c VP!V3RVRVRVRVRV/VBp V Pp VPV 4p VeVP^,p MVP^,p VPP fV ^8wd \ R4hVPP fRp MVeWPP 8gPV P\P4p\P!VPR,V P\PR7pW,PR4p M.Rp \PVPP R 24V \P!WPR 7V 3,pRpVEeVPV P4pVPP"fVP$^8XdR VPnMrVP$^8dQVP&\P(8Xg VP&\P*8XdR VPnMR VPnVPP"R 8XdL\-4pVP$^8Xd'V!VP/4VP/44pMV!VV4pMVPP"R 8Xd>\14pV!VP3RVP$4VP3R44pM.VPP"R 8Xd\54pV!VV4p\7VVV P8V P:V P<R7#)ae labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). rjr|rrrmNz=Cannot handle batch sizes > 1 if no padding token is defined.)rrBz will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r regressionsingle_label_classificationmulti_label_classificationrrA)rrrrNrrrrCrr4int32r(argmaxlogger warning_oncer:rQ problem_typerrBlongr[rrrrrrrr=r)r8rrjr|rrrrmrxtransformer_outputsr=rrlast_non_pad_token non_pad_mask token_indices pooled_logitsrloss_fcts&&&&&&&&, r,rJ&ZambaForSequenceClassification.forward s&8!F!Fr!J !#    >>**+,ZZ  u||J}}MOaab   YYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#M$9$9$;V^^=MND#M6:D))-JJ+- 2 22t GUWY))-II,. v6/ /??-;;*55   r/)rrr)NNNNNNN) rQrRrSrTr2rrrJrUrVrWrXs@@r,rrs4R R R R r/r)rrO)NNFN)NNNFFFF)rrrr)Prrcollections.abcrr4torch.nn.functionalrrurtorch.nnrrrrr activationsr cache_utilsr r generationr integrationsr r integrations.accelerater masking_utilsrrmodeling_layersrmodeling_outputsrrrmodeling_utilsrrprocessing_utilsrutilsrrrr utils.genericrutils.import_utilsrrrutils.output_capturingr configuration_zambar" get_loggerrQrrmr$rdr}rrrrrrrrQr`rtrrrrr__all__rr/r,rs& $ AA&!.)W=P9qqF&RR7 6,   H %J299J* U%2C)RYYC)L ((>P'Q'&((:OL'M',(* *Z(c cL*,HJZ\pqrO%biiO%sO%f ryy  00f171h414n$!?$!$!Nd %d d PS+_Sl  ^ %9^  ^ B gr/