+
    œ°³jÃ  ã                   ó  € R t ^ RIHt ^ RIt^ RIt^ RIHt ^ RIHu Ht	 ^RI
Ht ^RIHtHtHt ^RIHt ^RIHt ^RIHtHt ^RIHt ^R	IHtHt ^R
IHt ^RIH t H!t!H"t"H#t# ^RI$H%t%H&t& ^RI'H(t(H)t) ^RI*H+t+ ^RI,H-t- ^RI.H/t/H0t0H1t1H2t2H3t3H4t4H5t5 ^RI6H7t7H8t8H9t9H:t:H;t; ]#Px                  ! ]=4      t>R9R R llt?R:R R llt@R R ltAR^]P„                  3R R lltCR R ltD ! R R]24      tE ! R R ]34      tF ! R! R"]-4      tG ! R# R$]04      tH ! R% R&]44      tI ! R' R(]14      tJ]! ! R) R*]/4      4       tK ! R+ R,]K4      tL ! R- R.]K4      tM ! R/ R0]K4      tN ! R1 R2]K4      tO ! R3 R4]K4      tP]!! R5R67       ! R7 R8]K]4      4       tQ. R;OtRR# )<z<Blt modular model, inheriting from Mllama where appropriate.)ÚCallableN)Úinitialization)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_causal_mask)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)Údynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputs)Úrotate_half)ÚLlamaRotaryEmbedding)ÚMllamaPreTrainedModelÚMllamaSelfAttentionDecoderLayerÚMllamaTextCrossAttentionÚMllamaTextMLPÚMllamaTextRMSNormÚMllamaTextSelfAttentionÚeager_attention_forward)Ú	BltConfigÚBltGlobalTransformerConfigÚBltLocalDecoderConfigÚBltLocalEncoderConfigÚBltPatcherConfigc                ó$   € V ^8„  d   QhR\         /# )é   Úprime©Úint)Úformats   "Úd/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/blt/modular_blt.pyÚ__annotate__r,   9   s   € ÷ :ñ :´ñ :ó    c                ó  € \         P                  ! V\         P                  V P                  R7      p\         P                  ! V P
                  R,          V P                  R7      pW#,          p\         P                  ! W,          RR7      # )aX  
A polynomial rolling hash algorithm that converts sequences
of tokens into hash values. The hash is computed as:
    hash = (token_0 * prime^0 + token_1 * prime^1 + ... + token_n * prime^n)

The rolling hash allows the model to efficiently
identify and encode recurring byte-level patterns in the input text.

Args:
    token_tensor (torch.Tensor): [batch_size, seq_len, group_size] containing token IDs to hash
    prime (int): Prime number used as the base for the polynomial hash.

Returns:
    torch.Tensor: Hash values of shape [batch_size, seq_len] where each value
                 represents the hash of the corresponding token group

Example:
    >>> tokens = torch.tensor([[1, 2, 3], [4, 5, 6]])
    >>> hashes = rolling_polynomial_hash(tokens, prime=31)
    >>> # hash[0] = 1*31^0 + 2*31^1 + 3*31^2
    >>> # hash[1] = 4*31^0 + 5*31^1 + 6*31^2
©ÚdtypeÚdevice©r1   ©Údiméÿÿÿÿ)ÚtorchÚtensorÚint64r1   ÚarangeÚshapeÚsum)Útoken_tensorr'   Úprime_tensorÚpowersÚprime_powerss   &&   r+   Úrolling_polynomial_hashr@   9   s]   € ô. —<’< ¬U¯[©[À×ATÑATÔU€LÜ�\Š\˜,×,Ñ,¨RÕ0¸×9LÑ9LÔM€FØÕ'€LÜ�9Š9�\Õ0°bÔ9Ð9r-   c                ó\   € V ^8„  d   QhR\         P                  R\        R\        R\        /# )r&   Ú	token_idsÚ
group_sizer'   Úmax_hash©r6   ÚTensorr)   )r*   s   "r+   r,   r,   V   s0   € ÷ ñ Ü�|‰|ðÜ),ðÜ9<ðÜUXñr-   c           	     ó„  € \         P                  ! 4       ;_uu_ 4        V P                  w  rE\         P                  ! WA^,
          \         P                  V P
                  R7      p\         P                  ! W`.^R7      pVP                  ^V^4      p\        W‚4      p	W“,          p
RRR4       V
#   + '       g   i     X
# ; i)z1Hash token groups and map to range [0, max_hash].r/   r3   N)	r6   Úno_gradr:   Úzerosr8   r1   ÚcatÚunfoldr@   )rB   rC   r'   rD   Ú
batch_sizeÚseq_lenÚpaddingÚpadded_tokensÚwindowsÚhashesÚhash_valuess   &&&&       r+   Úbyte_group_hash_functionrS   V   s”   € ô 
�Š��Ø'Ÿo™oÑˆ
ä—+’+˜j°q­.ÄÇÁÐT]×TdÑTdÔeˆÜŸ	š	 7Ð"6¸AÔ>ˆð  ×&Ñ& q¨*°aÓ8ˆÜ(¨Ó8ˆØÕ'ˆ÷ 
ð Ð÷ 
Žð Ðús   žBB.Â.B?	c                óœ   € V ^8„  d   QhR\         P                  R\        P                  R\        R\
        R\        R\         P                  /# )r&   Úlocal_encoder_tokensÚencoder_hash_tok_embeddingÚ$encoder_hash_byte_group_nb_functionsÚencoder_hash_byte_group_sizeÚencoder_hash_byte_group_vocabÚreturn)r6   rF   ÚnnÚ	Embeddingr)   Úlist)r*   s   "r+   r,   r,   h   sR   € ÷ #ñ #ÜŸ,™,ð#ô !#§¡ð#ô +.ð	#ô
 #'ð#ô $'ð#ô ‡\�\ñ#r-   c                ó0  € . ROpVP                  V 4      p^ p\        V4       Fo  p	Wi\        V4      ,          ,          p
V FN  p\        WW¥4      pWÈV,          ,           pWr! V4      P	                  VP
                  4      ,          pV^,          pKP  	  Kq  	  V# )z=Compute token embeddings enhanced with hash-based embeddings.)éÊš;l   Í21A ißoYl   Ívt l   ß. l   }îg l   �A§u l   í†0 l   ©ÿT l   AK l   ™| )Úembed_tokensÚrangeÚlenrS   Útor1   )rU   Úlocal_encoderrV   rW   rX   rY   ÚprimesÚ
embeddingsÚembedding_idxÚfunc_nbr'   rC   Úhash_idsÚoffset_hash_idss   &&&&&&        r+   Úcompute_hash_embeddingsrk   h   s›   € ò€Fð ×+Ñ+Ð,@ÓA€JØ€MÜÐ=Ö>ˆØ¤ V£Õ,Õ-ˆÛ6ˆJÜ/Ð0DÐRWÓwˆHà&Ð9VÕ)VÕVˆOØÐ4°_ÓE×HÑHÈ×IZÑIZÓ[Õ[ˆJØ˜QÕŠMó 7ñ ?ð Ðr-   Fc                óÞ   € V ^8„  d   QhR\         P                  R\        R\        R\        R\        R\         P                  R\
        \         P                  \         P                  3,          /# )r&   Ú	patch_idsÚnum_patchesÚsequence_lengthÚpatches_as_queriesÚcross_attn_kr0   rZ   )r6   rF   r)   Úboolr0   Útuple)r*   s   "r+   r,   r,   Ž   sq   € ÷ K ñ K Ü�|‰|ðK äðK ô ðK ô ð	K ô
 ðK ô �;‰;ðK ô Œ5�<‰<œŸ™Ð%Õ&ñK r-   c                óœ  € V P                   w  rgV P                  pV'       dr   W,          p	Tp
\        P                  ! WR7      P	                  ^ 4      P	                  R4      P                  WaV4      pV P	                  ^4      P                  WaV4      pMpTp	W,          p
V P	                  R4      P                  WgV4      p\        P                  ! WR7      P	                  ^ 4      P	                  ^ 4      P                  WgV4      pW¼8H  pV'       d   ^MRpVP                  WNR7      pWiV
3pVP                   V8w  d   \        RVP                    RV 24      hVP	                  ^4      pRVP                  V4      ,
          pVP                  VP                  \        P                  4      \        P                  ! V4      P                  4      pV# )a  
Prepare cross-attention mask for patch-based attention, following mllama's robust approach.

This function creates masks that control which patches can attend to which other patches,
with support for query/key role swapping and cross-attention multipliers.

Args:
    patch_ids (torch.Tensor): Tensor of shape [batch_size, seq_len] containing patch ids.
    num_patches (int): Total number of patches.
    sequence_length (int): Length of the sequence.
    patches_as_queries (bool): If True, patches are used as queries, otherwise as keys.
    cross_attn_k (int): Cross-attention multiplier for repeating patches.
    dtype (torch.dtype): Data type for the output mask.

Returns:
    Tuple[torch.Tensor, torch.Tensor]:
        - cross_attention_mask: 4D tensor [batch_size, 1, q_len, kv_len]
r2   r3   zCross attention mask shape z doesn't match expected g      ð?r5   )r:   r1   r6   r9   Ú	unsqueezeÚexpandÚrepeat_interleaveÚ
ValueErrorrc   Úmasked_fillrr   ÚfinfoÚmin)rm   rn   ro   rp   rq   r0   rL   rM   r1   Úq_lenÚkv_lenÚq_patch_idsÚkv_patch_idsÚcross_attention_maskÚ
repeat_dimÚexpected_shapeÚinverted_cross_attn_masks   &&&&&&           r+   Ú#_prepare_patch_cross_attention_maskr„   Ž   s±  € ð4 $Ÿ/™/Ñ€JØ×Ñ€F÷ ØÕ*ˆØ ˆô �LŠL˜Ô4ß‰Y�q‹\ß‰Y�r‹]ß‰V�J¨WÓ5ð	 	ð !×*Ñ*¨1Ó-×4Ñ4°ZÈgÓV‰àˆØÕ+ˆà×)Ñ)¨"Ó-×4Ñ4°ZÈ+ÓVˆä�LŠL˜Ô4×>Ñ>¸qÓA×KÑKÈAÓN×UÑUÐV`ÐkvÓwð 	ð 'Ñ6Ð÷ )‘¨b€JØ/×AÑAÀ,ÐAÓ_Ðð !¨Ð0€NØ×!Ñ! ^Ô3ÜØ)Ð*>×*DÑ*DÐ)EÐE]Ð^lÐ]mÐnó
ð 	
ð
 0×9Ñ9¸!Ó<Ðð  #Ð%9×%<Ñ%<¸UÓ%CÕCÐØ3×?Ñ?Ø ×#Ñ#¤E§J¡JÓ/´·²¸UÓ1C×1GÑ1GóÐð  Ðr-   c                ór   € V ^8„  d   QhR\         P                  R\        R,          R\         P                  /# )r&   Úpatch_lengthsÚmax_patch_lengthNrZ   rE   )r*   s   "r+   r,   r,   Ü   s0   € ÷ ,(ñ ,(¬¯©ð ,(ÌÈtÍð ,(ÔX]×XdÑXdñ ,(r-   c                ó  € Vf   V # V P                  ^ 4      pV ^ 8„  p\        P                  ! W0V,          ^ 4      p\        P                  ! W0V,          ^ 4      pWE^ 8„  P                  VP                  4      ,           p\        VP                  4       P                  4       4      pV^ 8X  d0   \        P                  ! V^ 3V P                  V P                  R7      # \        P                  ! WpP                  V P                  R7      pW„P                  R4      8  p	W„P                  R4      8H  VP                  R4      ^ 8„  ,          p
\        P                  ! V	\        P                  ! W�4      \        P                  ! V4      4      p\        P                  ! W¥P                  R4      V4      pVP                  VR4      pV^ 8„  p\        P                  ! VP                  ^,          V P                  R7      VP!                  ^R7      P                  R4      8  p\        P                  ! V4      pVP#                  WìV,          4       \        VP!                  ^R7      P                  4       P                  4       4      pVRRV13,          # )aƒ  
Splits patch lengths into smaller segments if they exceed `max_patch_length`.
Pads the result to uniform length across the batch.

Args:
    patch_lengths (torch.Tensor): [batch_size, num_patches] tensor of patch lengths.
    max_patch_length (int, optional): Maximum allowed length per patch.

Returns:
    torch.Tensor: [batch_size, max_len] tensor of split and padded patch lengths.
Nr/   r2   r3   ºNNNr5   )Úsizer6   Úwhererc   r0   r)   ÚmaxÚitemrI   r1   r9   ru   Ú	full_likeÚ
zeros_likeÚreshaper:   r;   Úmasked_scatter_)r†   r‡   rL   ÚpositiveÚfull_chunksÚ	remainderÚsegment_countsÚmax_segments_per_patchÚsegment_indexÚ	full_maskÚremainder_maskÚsegmentsÚflat_segmentsÚnonzeroÚpacked_maskÚpacked_segmentsÚmax_lens   &&               r+   Úprocess_patch_lengthsr    Ü   s$  € ð ÒØÐà×#Ñ# AÓ&€JØ˜qÑ €HÜ—+’+˜hÐ9IÕ(IÈ1ÓM€KÜ—’˜HÐ6FÕ&FÈÓJ€IØ °¡M×#5Ñ#5°k×6GÑ6GÓ#HÕH€Nä  ×!3Ñ!3Ó!5×!:Ñ!:Ó!<Ó=ÐØ Ô"Ü�{Š{˜J¨˜?°-×2EÑ2EÈm×NbÑNbÔcÐcä—L’LÐ!7×?RÑ?RÐ[h×[oÑ[oÔp€MØ× 5Ñ 5°bÓ 9Ñ9€IØ#×'<Ñ'<¸RÓ'@Ñ@ÀY×EXÑEXÐY[ÓE\Ð_`ÑE`Õa€NÜ�{Š{ØÜ�Š˜Ó8Ü×Ò˜Ó'ó€Hô
 �{Š{˜>×+>Ñ+>¸rÓ+BÀHÓM€Hà×$Ñ$ Z°Ó4€MØ˜aÑ€GÜ—,’,˜}×2Ñ2°1Õ5¸m×>RÑ>RÔSÐVd×VhÑVhØð Wió Wç�i�ƒmñ€Kô ×&Ò& }Ó5€OØ×#Ñ# K¸wÕ1GÔHä�.×$Ñ$¨Ð$Ó+×/Ñ/Ó1×6Ñ6Ó8Ó9€GØ˜1˜h˜w˜h˜;Õ'Ð'r-   c                   ó   € ] tR tRtRtR# )ÚBltMLPi  © N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__static_attributes__r£   r-   r+   r¢   r¢     ó   † Ûr-   r¢   c                   ó   € ] tR tRtRtR# )Ú
BltRMSNormi  r£   Nr¤   r£   r-   r+   r¬   r¬     rª   r-   r¬   c                   óX   a € ] tR tRt o ]P
                  ! 4       ]R 4       4       tRtV t	R# )ÚBltRotaryEmbeddingi  c                ób  € V P                   R,          P                  4       P                  VP                  ^ ,          R^4      pVR	,          P                  4       p\	        VP
                  P                  \        4      '       d2   VP
                  P                  R8w  d   VP
                  P                  MRp\        VRR7      ;_uu_ 4        VP                  4       VP                  4       ,          P                  ^^4      p\        P                  ! V^RR7      pVP                  4       V P                  ,          pVP                  4       V P                  ,          p	R R R 4       XP                  VP                   R7      X	P                  VP                   R7      3#   + '       g   i     LH; i)
NÚmpsÚcpuF)Údevice_typeÚenabledr3   )r0   )Nr‰   Nr5   )r‰   Nr‰   )Úinv_freqÚfloatrv   r:   Ú
isinstancer1   ÚtypeÚstrr   Ú	transposer6   rw   ÚcosÚattention_scalingÚsinrc   r0   )
ÚselfÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr²   ÚfreqsÚembrº   r¼   s
   &&&       r+   ÚforwardÚBltRotaryEmbedding.forward  s5  € ð !ŸM™M¨-Õ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÕH]Ð_aÐcdÓeÐØ ,¨ZÕ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÒ'EÈ!Ï(É(Ï-É-Ð[`ÔJ`�a—h‘h—m’mÐfkˆÜ¨¸U×CÖCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÕN×YÑYÐZ[Ð]^Ó_ˆEÜ×)Ò)¨%°¸Ô;ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆCØ—'‘'“)˜d×4Ñ4Õ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷ D×Cús   ÃBFÆF.	r£   N)
r¥   r¦   r§   r¨   r6   rH   r   rÄ   r©   Ú__classdictcell__©Ú__classdict__s   @r+   r®   r®     s%   ø‡ € Ø
‡]‚]ƒ_Øñ<ó ó ö<r-   r®   c                   ó>   a a€ ] tR tRt oV3R lV 3R lltRtVtV ;t# )ÚBltTransformerLayeri$  c                ó    <€ V ^8„  d   QhRS[ /# )r&   Ú	layer_idxr(   )r*   rÈ   s   "€r+   r,   Ú BltTransformerLayer.__annotate__%  s   ø€ ÷ `ñ `©#ñ `r-   c                óþ   <€ \         SV `  4        \        WR 7      V n        \	        V4      V n        \        VP                  VP                  R7      V n	        \        VP                  VP                  R7      V n
        R# ))ÚconfigrÌ   ©ÚepsN)ÚsuperÚ__init__ÚBltSelfAttentionÚ	self_attnr¢   Úmlpr¬   Úhidden_sizeÚrms_norm_epsÚinput_layernormÚpost_attention_layernorm©r½   rÏ   rÌ   Ú	__class__s   &&&€r+   rÓ   ÚBltTransformerLayer.__init__%  s[   ø€ Ü‰ÑÔä)°ÔMˆŒÜ˜&“>ˆŒÜ)¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(2°6×3EÑ3EÈ6×K^ÑK^Ô(_ˆÖ%r-   )rÙ   rÖ   rÚ   rÕ   ©r¥   r¦   r§   r¨   rÓ   r©   rÆ   Ú__classcell__©rÜ   rÈ   s   @@r+   rÊ   rÊ   $  s   ù‡ € ÷`÷ `ó `r-   rÊ   c                   ó>   a a€ ] tR tRt oV3R lV 3R lltRtVtV ;t# )rÔ   i.  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r&   rÏ   rÌ   ©r    r)   )r*   rÈ   s   "€r+   r,   ÚBltSelfAttention.__annotate__/  s   ø€ ÷ ,ñ ,™yð ,±Sñ ,r-   c                ó&   <€ \         SV `  W4       R # ©N)rÒ   rÓ   rÛ   s   &&&€r+   rÓ   ÚBltSelfAttention.__init__/  s   ø€ Ü‰Ñ˜Ö+r-   r£   rÞ   rà   s   @@r+   rÔ   rÔ   .  s   ù‡ € ÷,÷ ,ó ,r-   rÔ   c                   ó\   a a€ ] tR tRt oRtRV3R lV 3R llltR	V3R lR lltRtVtV ;t	# )
ÚBltCrossAttentioni3  z<Cross-attention module for Blt, following transformers stylec                ó:   <€ V ^8„  d   QhRS[ RS[RS[R,          /# )r&   rÏ   rÌ   r×   Nrã   )r*   rÈ   s   "€r+   r,   ÚBltCrossAttention.__annotate__6  s+   ø€ ÷ Lñ L™yð L±Sð LÁsÈTÅzñ Lr-   c                óÊ   <€ \         SV `  4        R V n        \        V P                  VP
                  R7      V n        \        V P                  VP
                  R7      V n        R# )FrÐ   N)rÒ   rÓ   Ú	is_causalr¬   r×   rØ   Úq_normÚk_norm)r½   rÏ   rÌ   r×   rÜ   s   &&&&€r+   rÓ   ÚBltCrossAttention.__init__6  sI   ø€ Ü‰ÑÔØˆŒÜ  ×!1Ñ!1°v×7JÑ7JÔKˆŒÜ  ×!1Ñ!1°v×7JÑ7JÔKˆŽr-   c          	      óš   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[S[,          /# )r&   Úhidden_statesÚcross_attention_statesNÚattention_maskÚkwargs)r6   rF   r   r   )r*   rÈ   s   "€r+   r,   rë   <  sO   ø€ ÷ #)ñ #)á—|‘|ð#)ñ !&§¡¨tÕ 3ð#)ñ Ÿ™ tÕ+ð	#)ñ
 Ñ+Õ,ñ#)r-   c                ól  € VP                  4       w  rVpV P                  V4      pV P                  V4      pVP                  WVV P                  V P
                  4      P                  ^^4      pV P                  V4      pV P                  V4      p	V P                  V4      p
V	P                  VRV P                  V P
                  4      P                  ^^4      p	V
P                  VRV P                  V P
                  4      P                  ^^4      p
\        P                  ! V P                  P                  \        4      pT! V VV	V
V3RV P                   '       g   RMV P"                  RV P$                  /VB w  rÍVP'                  WVR4      P)                  4       pV P+                  V4      pWÁ,           pWÍ3# )é   Údropoutç        Úscalingr5   )rŠ   rî   Úq_projÚviewÚ	num_headsÚhead_dimr¹   rï   Úk_projÚv_projÚnum_key_value_headsr   Úget_interfacerÏ   Ú_attn_implementationr   Útrainingrø   rú   r�   Ú
contiguousÚo_proj)r½   rò   ró   rô   rõ   Úbszr|   Ú_Úquery_statesÚ
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss   &&&&,         r+   rÄ   ÚBltCrossAttention.forward<  s’  € ð &×*Ñ*Ó,‰ˆ�AØ—{‘{ =Ó1ˆØ—{‘{ <Ó0ˆØ#×(Ñ(¨°T·^±^ÀTÇ]Á]ÓS×]Ñ]Ð^_ÐabÓcˆà!%§¡Ð-CÓ!DÐØ—[‘[Ð!7Ó8ˆ
Ø—{‘{Ð#9Ó:ˆØ—_‘_ S¨"¨d×.FÑ.FÈÏÉÓV×`Ñ`ÐabÐdeÓfˆ
Ø#×(Ñ(¨¨b°$×2JÑ2JÈDÏMÉMÓZ×dÑdÐefÐhiÓjˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ	%
ð  $Ÿ}Ÿ}˜}‘C°$·,±,ð	%
ð —L‘Lð	%
ð ñ	%
Ñ!ˆð "×)Ñ)¨#°bÓ9×DÑDÓFˆØ—k‘k +Ó.ˆØ!Õ1ˆØÐ(Ð(r-   )rí   rï   rî   ræ   ©NN)
r¥   r¦   r§   r¨   Ú__doc__rÓ   rÄ   r©   rÆ   rß   rà   s   @@r+   ré   ré   3  s#   ù‡ € ÙF÷Lõ L÷#)÷ #)ò #)r-   ré   c                   óš   a € ] tR tRt o RtRtRtRR.tR]! ]	^ R7      R]! ]
^R7      /t]P                  ! 4       R 4       tV 3R	 ltR
tV tR# )ÚBltPreTrainedModelib  FrÊ   ré   rò   )ÚindexÚ
attentionsc           	     ó  € \         P                  ! W4       VP                  P                  p\	        V\
        P                  4      '       dæ   \        V P                  RR4      pVf>   \        V P                  R4      '       d"   \        V P                  P                  RR4      pVf   VP                  pVR,          p\        P                  ! VP                  RVRV,          ^V,          R7       VP                  e2   \        P                   ! VP                  VP                  ,          4       R# \	        V\"        \$        34      '       g	   VR9   Ed¾   \        V P                  RR4      pVf   \        VR4      '       d   VP&                  pVfM   R FF  p\        WR4      pVf   K  \        VR4      '       g   K)  VP                  P(                  R,          p M	  Vf   R# VR,          pR F�  p\        WR4      pVf   K  \        VR4      '       g   K)  \        P                  ! VP                  RVRV,          ^V,          R7       \        VR	R4      f   Km  \        P                   ! VP*                  4       K�  	  \        VR\        VRR4      4      p	V	eu   \        V	R4      '       dc   \        P                  ! V	P                  RVRV,          ^V,          R7       \        V	R	R4      e!   \        P                   ! V	P*                  4       R# \	        V\,        4      '       g	   VR
8X  Ed<   \        V P                  RR4      pVf>   \        V P                  R4      '       d"   \        V P                  P.                  RR4      pVf>   \        V P                  R4      '       d"   \        V P                  P                  RR4      pRp
Ve
   VR,          p
\        VR\        VRR4      4      p\        VRR4      p\        VR\        VRR4      4      pW¼3 F¯  pVf   K	  \        VR4      '       g   K  T
;'       g%    VP                  P(                  ^,          R,          p\        P                  ! VP                  RVRV,          ^V,          R7       \        VR	R4      f   K�  \        P                   ! VP*                  4       K±  	  Ve›   \        VR4      '       d‰   VP                  P(                  ^,          pVR,          p\        P                  ! VP                  RVRV,          ^V,          R7       \        VR	R4      e!   \        P                   ! VP*                  4       R# \	        V\
        P0                  4      '       dy   VP2                  pVR,          p\        P                  ! VP                  RVRV,          ^V,          R7       VP*                  e!   \        P                   ! VP*                  4       R# R# )zô
Initialize BLT weights following the original ByteLatentTransformer:

- Most weights are drawn from a truncated normal.
- Scale is ~ 1 / sqrt(model_dim) (or 1 / sqrt(hidden_dim) for FFN outputs).
- Norm layers are set to weight = 1, bias = 0.
r×   NÚencoder_configrù   )ÚmeanÚstdÚaÚbr  ÚdenseÚweightÚbiasr   Údecoder_configÚ	gate_projÚfc1Úup_projÚ	down_projÚfc2g      à¿éýÿÿÿ)r   r   )rû   rÿ   r   r  r  r5   )rû   rÿ   r   )r   Ú_init_weightsrÜ   r¥   r¶   r[   r\   ÚgetattrrÏ   Úhasattrr  Úembedding_dimÚinitÚtrunc_normal_r  Úpadding_idxÚzeros_rÔ   ré   r×   r:   r  r¢   r  ÚLinearÚin_features)r½   ÚmoduleÚ
class_namer×   r  r4   ÚnameÚprojÚ	proj_namer  Úin_stdr   r"  r#  Ú
hidden_dimÚout_stdÚfan_ins   &&               r+   r&  Ú BltPreTrainedModel._init_weightsv  sÞ  € ô 	×%Ò% dÔ3à×%Ñ%×.Ñ.ˆ
ô �fœbŸl™l×+Ò+Ü! $§+¡+¨}¸dÓCˆKØÒ"¤w¨t¯{©{Ð<L×'MÒ'MÜ% d§k¡k×&@Ñ&@À-ÐQUÓV�ØÒ"Ø$×2Ñ2�à˜tÕ#ˆCÜ×ÒØ—‘ØØØ�s•(Ø�c•'õð ×!Ñ!Ò-Ü—’˜FŸM™M¨&×*<Ñ*<Õ=Ô>Ùô �fÔ/Ô1BÐC×DÒDÈ
ð W
õ I
ô ˜$Ÿ+™+ }°dÓ;ˆCØŠ{œw v¨}×=Ò=Ø×(Ñ(�ØŠ{ÛM�DÜ" 6°Ó6�DØÔ'¬G°D¸(×,CÔ,CØ"Ÿk™k×/Ñ/°Õ3˜Ùñ	 Nð
 Š{Ùà�t•)ˆCó <�	Ü˜v°$Ó7�ØÔ#¬°°h×(?Ô(?Ü×&Ò&ØŸ™Ø ØØ˜s�(Ø˜c�'õô ˜t V¨TÓ2Ô>ÜŸš D§I¡IÖ.ñ <ô ˜V X¬w°v¸wÈÓ/MÓNˆFØÒ!¤g¨f°h×&?Ò&?Ü×"Ò"Ø—M‘MØØØ˜3•hØ˜#•gõô ˜6 6¨4Ó0Ò<Ü—K’K §¡Ô,Ùô �fœf×%Ò%¨°Õ)FÜ! $§+¡+¨}¸dÓCˆKØÒ"¤w¨t¯{©{Ð<L×'MÒ'MÜ% d§k¡k×&@Ñ&@À-ÐQUÓV�ØÒ"¤w¨t¯{©{Ð<L×'MÒ'MÜ% d§k¡k×&@Ñ&@À-ÐQUÓV�ð ˆFØÒ&Ø$ dÕ*�ä ¨´W¸VÀUÈDÓ5QÓRˆIÜ˜f i°Ó6ˆGÜ ¨´W¸VÀUÈDÓ5QÓRˆIð #Ó,�ØÔ#¬°°h×(?Ô(?Ø ×BÐB T§[¡[×%6Ñ%6°qÕ%9¸TÕ%A�CÜ×&Ò&ØŸ™Ø ØØ˜s�(Ø˜c�'õô ˜t V¨TÓ2Ô>ÜŸš D§I¡IÖ.ñ -ð Ò$¬°¸H×)EÒ)EØ&×-Ñ-×3Ñ3°AÕ6�
Ø$ dÕ*�Ü×"Ò"Ø×$Ñ$ØØØ˜7•lØ˜'•kõô ˜9 f¨dÓ3Ò?Ü—K’K 	§¡Ô/Ùô �fœbŸi™i×(Ò(Ø×'Ñ'ˆFØ˜$•,ˆCÜ×ÒØ—‘ØØØ�s•(Ø�c•'õð �{‰{Ò&Ü—’˜FŸK™KÔ(Ùñ )r-   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # ©r&   rÏ   ©r    )r*   rÈ   s   "€r+   r,   ÚBltPreTrainedModel.__annotate__b  s   ø‡ ‚ áÑò r-   r£   N)r¥   r¦   r§   r¨   Ú_supports_attention_backendÚ_supports_flash_attnÚ_supports_flex_attnÚ_no_split_modulesr   rÊ   rÔ   Ú_can_record_outputsr6   rH   r&  Ú__annotate_func__r©   rÆ   rÇ   s   @r+   r  r  b  sj   ø‡ € ð #(ÐØ ÐØÐØ.Ð0CÐDÐà™Ð(;À1ÔEØ‘nÐ%5¸QÔ?ðÐð ‡]‚]ƒ_ñJó ðJ÷+ ƒ r-   r  c                   ó€   a a€ ] tR tRt oR]! ]^RR7      /tV3R lV 3R lltRV3R lR lltR	 t	V3R
 lt
RtVtV ;t# )ÚBltLocalEncoderi  Úencoder_attentionsrd   ©r  Ú
layer_namec                ó    <€ V ^8„  d   QhRS[ /# r;  ©r#   )r*   rÈ   s   "€r+   r,   ÚBltLocalEncoder.__annotate__
  s   ø€ ÷ ñ Ñ4ñ r-   c           	     óø  <€ \         SV `  V4       R V n        Wn        \        P
                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n	        \        VR7      V n        \        P                  ! VP                  VP                  VP                  ,          R R7      V n        \        P                   ! VP"                  VP                  4      V n        \        P
                  ! 4       V n        VP(                  '       d   VP                  M^p\        V4       F3  pV P&                  P+                  \-        WVP                  R7      4       K5  	  V P/                  4        R# u upi )F©rÏ   ©r/  Úout_featuresr  ©rÏ   rÌ   r×   N)rÒ   rÓ   Úgradient_checkpointingrÏ   r[   Ú
ModuleListra   Únum_hidden_layersrÊ   Úlayersr®   Ú
rotary_embr.  r×   rq   Úpatch_embedding_projectionr\   Ú
vocab_sizer`   Úcross_attn_layersÚcross_attn_all_layersÚappendré   Ú	post_init©r½   rÏ   rÌ   Úlayers_to_addrÜ   s   &&  €r+   rÓ   ÚBltLocalEncoder.__init__
  s  ø€ Ü‰Ñ˜Ô Ø&+ˆÔ#ØŒÜ—m’mÜEJÈ6×KcÑKcÔEdÓeÑEd¸	Ô  Ö3ÑEdÑeó
ˆŒô -°FÔ;ˆŒÜ*,¯)ª)Ø×*Ñ*Ø×+Ñ+¨f×.AÑ.AÕAØô+
ˆÔ'ô
 ŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔÜ!#§¢£ˆÔØ4:×4P×4PÐ4P˜×0Ò0ÐVWˆÜ˜}Ö-ˆIØ×"Ñ"×)Ñ)Ü!¨ÐRX×RdÑRdÔeöñ .ð
 	�‰Öùò! fs   ÁE7c                óp  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          R	S[R,          R
S[ P                  R,          RS[S[,          /
# )r&   Ú	input_idsNÚinputs_embedsÚpatch_embedsrô   r¿   Úpast_key_valuesÚencoder_attention_maskrn   rm   rõ   )r6   Ú
LongTensorrF   r   r)   r   r   )r*   rÈ   s   "€r+   r,   rK  !  sÀ   ø€ ÷ 23ñ 23á×#Ñ# dÕ*ð23ñ —|‘| dÕ*ð23ñ —l‘l TÕ)ð	23ñ
 Ÿ™ tÕ+ð23ñ ×&Ñ&¨Õ-ð23ñ  �ð23ñ !&§¡¨tÕ 3ð23ñ ˜4•Zð23ñ —<‘< $Õ&ð23ñ Ñ+Õ,ñ23r-   c
           
     ó’  € Vf   V P                  V4      pVP                  ^ ,          p\        P                  ! W P                  P                  V P
                  R7      pVfS   \        P                  ! VP                  ^,          VP                  R7      P                  ^ 4      P                  VR4      pV P                  WÅ4      p\        P                  ! WÀP                  P                  V P
                  R7      p\        V P                  4       EF-  w  rïV! V3RVRVRV/V
B pV\        V P                  4      ^,
          8X  g   V P                  P                  '       g   KT  V P!                  WÈV	4      pV P#                  V4      pVP%                  W³P                  ^,          V P                  P&                  ,          V P                  P(                  4      pV P                  P                  '       d   TM^ pV P*                  V,          ! R	RVRVRV/V
B w  ppVVP-                  VP                  4      ,           pEK0  	  TpVV3# )
N©Úpr  r2   Úposition_embeddingsrô   rc  rò   ró   r5   r£   )r`   r:   ÚFrø   rÏ   r  r6   r9   r1   ru   rv   rU  Ú	enumeraterT  rb   rY  Úpatch_reducerV  r�   rq   r×   rX  rc   )r½   r`  ra  rb  rô   r¿   rc  rd  rn   rm   rõ   rL   rò   ri  ÚidxÚlayerrÌ   Úcross_attention_outputr  Úencoder_cross_statess   &&&&&&&&&&,         r+   rÄ   ÚBltLocalEncoder.forward!  s  € ð Ò Ø ×-Ñ-¨iÓ8ˆMà"×(Ñ(¨Õ+ˆ
ÜŸ	š	 -·;±;×3FÑ3FÐQU×Q^ÑQ^Ô_ˆàÒä—’˜]×0Ñ0°Õ3¸M×<PÑ<PÔQ×[Ñ[Ð\]Ó^×eÑeÐfpÐrtÓuð ð #Ÿo™o¨mÓJÐÜŸ	š	 -·;±;×3FÑ3FÐQU×Q^ÑQ^Ô_ˆä# D§K¡K×0‰JˆCÙ!Øñà$7ðð  .ðð !0ð	ð
 ñˆMð ”c˜$Ÿ+™+Ó&¨Õ*Ô*¨d¯k©k×.O×.OÒ.OØ#×0Ñ0°ÈYÓW�Ø#×>Ñ>¸|ÓL�Ø+×3Ñ3Ø× 2Ñ 2°1Õ 5¸¿¹×8PÑ8PÕ PÐRV×R]ÑR]×RiÑRió �ð $(§;¡;×#D×#DÐ#D™CÈ!�	Ø,0×,BÑ,BÀ9Ö,Mñ -Ø".ð-à+8ð-ð $:ð-ð ñ	-Ñ)Ð&¨ð  ,Ð.D×.GÑ.GÈ×H[ÑH[Ó.\Õ\“ñ+ 1ð,  ,ÐØÐ2Ð2Ð2r-   c                ó^  € VP                   ^ ,          pVP                   R,          pVP                  R4      P                  RRVP                   R,          4      p\        P                  ! WBV3VP
                  VP                  R7      pVP                  V^VRRR7      pVRRV1R3,          pV# )a>  
Reduce variable length patches to single embedding per patch
Note: this works with variable number of patches for different sequences in the batch
It handles variable length patches by assuming that patch_lengths will be 0 for any
extra patches on the *right*. Since there can be a variable number of patches
this function also return the number of patches for each sequence in the batch.
Any embeddings on the right that are not allocated to a patch
(i.e. if the sum(patch_lengths[i]) < seq_len for any i)
will be sent to a dummy patch, which is trimmed before returning.
r/   ÚamaxF)Úsrcr4   r  ÚreduceÚinclude_selfr‰   Nr5   )r:   ru   rv   r6   rI   r0   r1   Úscatter_reduce)r½   rò   Úmax_num_patchesrm   rL   r)  Úreduced_embeddingss   &&&&   r+   rl  ÚBltLocalEncoder.patch_reduceU  s½   € ð #×(Ñ(¨Õ+ˆ
Ø%×+Ñ+¨BÕ/ˆà×'Ñ'¨Ó+×2Ñ2°2°r¸=×;NÑ;NÈrÕ;RÓSˆ	ä"Ÿ[š[Ø¨-Ð8À×@SÑ@SÐ\i×\pÑ\pô
Ðð 0×>Ñ>ØØØØØð ?ó 
Ðð 0°Ð3C°OÐ3CÀQÐ0FÕGÐà!Ð!r-   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r;  rJ  )r*   rÈ   s   "€r+   r,   rK    ó   ø‡ ‚ Ù!Ñ!ò r-   )rÏ   rX  r`   rQ  rT  rV  rU  ©	NNNNNNNNN)r¥   r¦   r§   r¨   r   rÔ   rB  rÓ   rÄ   rl  rC  r©   rÆ   rß   rà   s   @@r+   rE  rE    sA   ù‡ € ð 	™nÐ-=ÀQÐSbÔcðÐ÷ó ÷.23ò 23òh"÷c … r-   rE  c                   ó`   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltV3R ltRtVtV ;t	# )	ÚBltLocalDecoderiu  c                ó    <€ V ^8„  d   QhRS[ /# r;  ©r"   )r*   rÈ   s   "€r+   r,   ÚBltLocalDecoder.__annotate__x  s   ø€ ÷ ñ Ñ4ñ r-   c           	     óò  <€ \         SV `  V4       R V n        Wn        RV n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n
        \        VR7      V n        \
        P                  ! VP                  VP                  VP                   ,          R R7      V n        \%        VP                  VP&                  R7      V n        \
        P                  ! 4       V n        VP,                  '       d   VP                  M^p\        V4       F3  pV P*                  P/                  \1        WVP                  R7      4       K5  	  V P3                  4        R# u upi )FTrM  rN  rÐ   rP  N)rÒ   rÓ   rQ  rÏ   Úcross_attn_decoderr[   rR  ra   rS  rÊ   rT  r®   rU  r.  Úhidden_size_globalr×   rq   rV  r¬   rØ   ÚnormrX  rY  rZ  ré   r[  r\  s   &&  €r+   rÓ   ÚBltLocalDecoder.__init__x  s"  ø€ Ü‰Ñ˜Ô Ø&+ˆÔ#ØŒØ"&ˆÔÜ—m’mÜEJÈ6×KcÑKcÔEdÓeÑEd¸	Ô  Ö3ÑEdÑeó
ˆŒô -°FÔ;ˆŒÜ*,¯)ª)Ø×1Ñ1Ø×+Ñ+¨f×.AÑ.AÕAØô+
ˆÔ'ô
 ˜v×1Ñ1°v×7JÑ7JÔKˆŒ	Ü!#§¢£ˆÔØ4:×4P×4PÐ4P˜×0Ò0ÐVWˆÜ˜}Ö-ˆIØ×"Ñ"×)Ñ)Ü!¨ÐRX×RdÑRdÔeöñ .ð
 	�‰Öùò! fs   ÁE4c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          R	S[S[,          /# )
r&   r`  Nra  rb  rô   r¿   rc  rd  rõ   )r6   re  rF   r   r   r   )r*   rÈ   s   "€r+   r,   r‚  �  sŸ   ø€ ÷ .ñ .á×#Ñ# dÕ*ð.ñ —|‘| dÕ*ð.ñ —l‘l TÕ)ð	.ñ
 Ÿ™ tÕ+ð.ñ ×&Ñ&¨Õ-ð.ñ  �ð.ñ !&§¡¨tÕ 3ð.ñ Ñ+Õ,ñ.r-   c           
     ó²  € VP                   ^ ,          p	Tp
V P                  V4      pVP                  W“P                   ^,          V P                  P                  ,          V P                  P
                  4      pVe   V P                  '       g	   W£,           p
VfS   \        P                  ! VP                   ^,          VP                  R7      P                  ^ 4      P                  V	R4      pV P                  W¥4      p\        P                  ! W P                  P                  V P                  R7      p
\!        V P"                  4       Fy  w  rÍV^ 8X  g   V P                  P$                  '       dC   V P&                  V,          ! R	RV
RVRV/VB w  rïW®P)                  V
P                  4      ,           p
V! V
3RVRVRV/VB p
K{  	  V P+                  V
4      pV# )
é    r2   rg  rò   ró   rô   ri  rc  r5   r£   )r:   rV  r�   rÏ   rq   r×   r„  r6   r9   r1   ru   rv   rU  rj  rø   r  rk  rT  rY  rX  rc   r†  )r½   r`  ra  rb  rô   r¿   rc  rd  rõ   rL   rò   ri  Úirn  ro  r  Úlogitss   &&&&&&&&,        r+   rÄ   ÚBltLocalDecoder.forward�  s¿  € ð #×(Ñ(¨Õ+ˆ
Ø%ˆØ×6Ñ6°|ÓDˆØ#×+Ñ+Ø×*Ñ*¨1Õ-°·±×0HÑ0HÕHÈ$Ï+É+×JaÑJaó
ˆð Ò#¨D×,C×,CÐ,CØ)Õ8ˆMàÒä—’˜]×0Ñ0°Õ3¸M×<PÑ<PÔQ×[Ñ[Ð\]Ó^×eÑeÐfpÐrtÓuð ð #Ÿo™o¨mÓJÐÜŸ	š	 -·;±;×3FÑ3FÐQU×Q^ÑQ^Ô_ˆä! $§+¡+Ö.‰HˆAØ�AŒv˜Ÿ™×:×:Ð:Ø,0×,BÑ,BÀ1Ö,Eñ -Ø"/ð-à+7ð-ð $:ð-ð ñ	-Ñ)Ð&ð !.×0IÑ0IÈ-×J^ÑJ^Ó0_Õ _�Ù!Øñà$7ðð  .ðð !0ð	ð
 ñŠMñ /ð  —‘˜=Ó)ˆØˆr-   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r;  r�  )r*   rÈ   s   "€r+   r,   r‚  u  r|  r-   )rÏ   r„  rX  rQ  rT  r†  rV  rU  ©NNNNNNN)
r¥   r¦   r§   r¨   rÓ   rÄ   rC  r©   rÆ   rß   rà   s   @@r+   r  r  u  s    ù‡ € ÷ó ÷0.ò .÷7 … r-   r  c                   óz   a a€ ] tR tRt oR]! ]^RR7      /tV3R lV 3R lltRV3R lR lltV3R	 lt	R
t
VtV ;t# )ÚBltGlobalTransformeriÁ  Úglobal_attentionsÚglobal_transformerrG  c                ó    <€ V ^8„  d   QhRS[ /# r;  ©r!   )r*   rÈ   s   "€r+   r,   Ú!BltGlobalTransformer.__annotate__Ç  s   ø€ ÷ ñ Ñ9ñ r-   c                óà  <€ \         SV `  V4       Wn        \        P                  ! 4       V n        \        VP                  4       F'  pV P
                  P                  \        W4      4       K)  	  \        VR 7      V n        \        VRR4      e4   \        P                  ! VP                  VP                  RR7      V n        M\        P"                  ! 4       V n        V P%                  4        R# )rM  Úencoder_cross_output_sizeNF©r  )rÒ   rÓ   rÏ   r[   rR  rT  ra   rS  rZ  rÊ   r®   rU  r'  r.  r˜  r×   Útoken_embedding_projectionÚIdentityr[  rÛ   s   && €r+   rÓ   ÚBltGlobalTransformer.__init__Ç  s«   ø€ Ü‰Ñ˜Ô ØŒÜ—m’m“oˆŒÜ˜v×7Ñ7Ö8ˆIØ�K‰K×ÑÔ2°6ÓEÖFñ 9ä,°FÔ;ˆŒô �6Ð6¸Ó=ÒIÜ.0¯iªiØ×0Ñ0°&×2DÑ2DÈ5ô/ˆDÕ+ô /1¯kªk«mˆDÔ+à�‰Ör-   c                ó®   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          /# )r&   ra  rô   Nr¿   rc  rõ   )r6   rF   re  r   r   r   )r*   rÈ   s   "€r+   r,   r–  Ù  s_   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ ×&Ñ&¨Õ-ð	ñ
  �ðñ Ñ+Õ,ñr-   c           
     óÚ  € VP                   w  rgpV P                  V4      p	\        P                  ! W�P                  P                  V P
                  R 7      p	VfS   \        P                  ! VP                   ^,          VP                  R7      P                  ^ 4      P                  VR4      pV P                  W“4      p
\        V P                  4       F  w  r¼V! V	3RV
RVRV/VB p	K  	  V	# )rg  r2   ri  rô   rc  r5   )r:   rš  rj  rø   rÏ   r  r6   r9   r1   ru   rv   rU  rk  rT  )r½   ra  rô   r¿   rc  rõ   rL   rM   r  rò   ri  r‹  rn  s   &&&&&,       r+   rÄ   ÚBltGlobalTransformer.forwardÙ  së   € ð "/×!4Ñ!4Ñˆ
˜QØ×7Ñ7¸ÓFˆÜŸ	š	 -·;±;×3FÑ3FÐQU×Q^ÑQ^Ô_ˆØÒä—’˜]×0Ñ0°Õ3¸M×<PÑ<PÔQ×[Ñ[Ð\]Ó^×eÑeÐfpÐrtÓuð ð #Ÿo™o¨mÓJÐÜ! $§+¡+Ö.‰HˆAÙ!Øñà$7ðð  .ðð !0ð	ð
 ñŠMñ /ð Ðr-   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r;  r•  )r*   rÈ   s   "€r+   r,   r–  Á  s   ø‡ ‚ Ù&Ñ&ò r-   )rÏ   rT  rU  rš  )NNN)r¥   r¦   r§   r¨   r   rÔ   rB  rÓ   rÄ   rC  r©   rÆ   rß   rà   s   @@r+   r‘  r‘  Á  s:   ù‡ € ð 	™^Ð,<ÀAÐRfÔgðÐ÷ó ÷$ò ÷1 … r-   r‘  c                   ót   a a€ ] tR tRt oV3R lV 3R lltR	V3R lR llt]R
R l4       tV3R ltRt	Vt
V ;t# )Ú
BltPatcheriô  c                ó    <€ V ^8„  d   QhRS[ /# r;  ©r$   )r*   rÈ   s   "€r+   r,   ÚBltPatcher.__annotate__÷  s   ø€ ÷ ñ Ñ/ñ r-   c                óâ  <€ \         SV `  V4       \        V P                  R 7      V n        \
        P                  ! 4       V n        \        V P                  P                  4       F2  pV P                  P                  \        V P                  V4      4       K4  	  \
        P                  ! V P                  P                  V P                  P                  4      V n        \!        V P                  P                  V P                  P"                  R7      V n        \
        P&                  ! V P                  P                  V P                  P                  RR7      V n        V P+                  4        R# )rM  rÐ   Fr™  N)rÒ   rÓ   r®   rÏ   rU  r[   rR  rT  ra   rS  rZ  rÊ   r\   rW  r×   r`   r¬   rØ   r†  r.  Úlm_headr[  rÛ   s   && €r+   rÓ   ÚBltPatcher.__init__÷  sæ   ø€ Ü‰Ñ˜Ô Ü,°D·K±KÔ@ˆŒÜ—m’m“oˆŒÜ˜tŸ{™{×<Ñ<Ö=ˆIØ�K‰K×ÑÔ2°4·;±;À	ÓJÖKñ >äŸLšL¨¯©×)?Ñ)?ÀÇÁ×AXÑAXÓYˆÔÜ˜tŸ{™{×6Ñ6¸D¿K¹K×<TÑ<TÔUˆŒ	Ü—y’yØ�K‰K×#Ñ#Ø�K‰K×"Ñ"Øô
ˆŒð 	�‰Ör-   c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[R,          RS[S[	,          /
# )r&   r`  Nrô   r¿   rc  ra  Ú	use_cacheÚ
patch_sizeÚ	thresholdr‡   rõ   )
r6   re  rF   r   ÚFloatTensorrr   r)   rµ   r   r   )r*   rÈ   s   "€r+   r,   r¥    s³   ø€ ÷ 9;ñ 9;á×#Ñ# dÕ*ð9;ñ Ÿ™ tÕ+ð9;ñ ×&Ñ&¨Õ-ð	9;ñ
  �ð9;ñ ×(Ñ(¨4Õ/ð9;ñ ˜$•;ð9;ñ ˜$•Jð9;ñ ˜4•<ð9;ñ  �*ð9;ñ Ñ+Õ,ñ9;r-   c
                ó†  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP	                  4       M^ p\
        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        V P                  VVVVR7      pTpV P                  WÓ4      pV P                   F  pV! WÞVR7      pK  	  V P                  V P                  V4      4      p\
        P                  P!                  VR7      P#                  4       pVP                  R,          w  ppVe   V P%                  VVVVR7      pM/\
        P&                  ! VV3VP(                  VP                  R	7      p\+        VV	4      pVVV3# )
Nú:You must specify exactly one of input_ids or inputs_embedsrM  r2   ©rÏ   ra  rô   rc  r¿   )ri  rô   )rŒ  :Nr&   N)Ú	entropiesro   r«  r¬  r/   )rx   r`   r   rÏ   Úget_seq_lengthr6   r9   r:   r1   ru   r   rU  rT  r§  r†  ÚdistributionsÚCategoricalÚentropyÚpatch_lengths_from_entropiesÚonesr0   r    )r½   r`  rô   r¿   rc  ra  rª  r«  r¬  r‡   rõ   Úpast_seen_tokensÚcausal_maskrò   ri  rn  rŒ  Úprediction_entropiesrL   ro   r†   s   &&&&&&&&&&,          r+   rÄ   ÚBltPatcher.forward  s°  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨mÓJÐà—[”[ˆEÙ! -ÐitÔuŠMñ !ð —‘˜dŸi™i¨Ó6Ó7ˆÜ$×2Ñ2×>Ñ>ÀfÐ>ÓM×UÑUÓWÐà&3×&9Ñ&9¸"Õ&=Ñ#ˆ
�OØÒ!Ø ×=Ñ=Ø.Ø /Ø%Ø#ð	 >ó ‰Mô "ŸJšJØ˜_Ð-°]×5HÑ5HÐQ^×QeÑQeôˆMô .¨mÐ=MÓNˆØ# ]°FÐ:Ð:r-   c                óÊ  € V P                   ^ ,          p\        P                  ! ^ ^.\        P                  V P                  R7      P                  ^ 4      P                  V^4      pVP                   ^,          pV R,          p W8„  pVP                   ^,          p\        P                  ! W€P                  R7      P                  ^ 4      P                  VR4      p	\        P                  ! W˜4      p
\        P                  ! Wš.^R7      p\        P                  ! Ww( .^R7      pW¼,          P                  WH4      pVP                  ^R7      P                  4       pVRRV13,          p\        P                  ! W]V,           3^R7      p\        P                  ! VR,          V^,
          4      p\        P                  ! VR,          ^,
          V3^R7      pVV,
          ^,           pV# )	z´
Computes patch lengths from token entropies.

Depending on whether a threshold is provided, the function uses either:
- Thresholding the entropy values (when `threshold` is set).
r/   r‰   r2   r3   N©r‰   :r÷   NNr5   )r‰   :Nr÷   N)r:   r6   r7   Úlongr1   ru   Úrepeatr9   rv   rŽ   rJ   r�   r;   rŒ   )r±  ro   r«  r¬  rL   Úinit_tokensÚoffsetÚ
patch_maskrM   Útoken_indicesÚsentinelÚpadded_indicesÚpadded_maskÚpatch_startsÚmax_valid_patchesÚpatch_start_idsÚ
last_tokenÚ
patch_endsr†   s   &&&&               r+   r¶  Ú'BltPatcher.patch_lengths_from_entropiesB  s¢  € ð —_‘_ QÕ'ˆ
ô �LŠL˜!˜Q˜¤u§z¡z¸)×:JÑ:JÔK×UÑUÐVWÓX×_Ñ_Ð`jÐlmÓnð 	ð ×"Ñ" 1Õ%ˆð ˜eÕ$ˆ	ð Ñ*ˆ
à×"Ñ" 1Õ%ˆô Ÿš W×5EÑ5EÔF×PÑPÐQRÓS×ZÑZÐ[eÐgiÓjˆÜ—?’? =Ó:ˆÜŸš MÐ#<À!ÔDˆô —i’i ¨[Ð 9¸qÔAˆð &Õ2×:Ñ:¸:ÓOˆØ&ŸN™N¨q˜NÓ1×5Ñ5Ó7ÐØ# AÐ'9Ð(9Ð'9Ð$9Õ:ˆô  Ÿ)š) [ÀÕ2GÐ$HÈaÔPˆô —_’_ _°UÕ%;¸_ÈqÕ=PÓQˆ
Ü—Y’Y °Õ 6¸Õ :¸JÐGÈQÔOˆ
à" _Õ4°qÕ8ˆàÐr-   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r;  r¤  )r*   rÈ   s   "€r+   r,   r¥  ô  s   ø‡ ‚ ÙÑò r-   )r`   rT  r§  r†  rU  r}  r  )r¥   r¦   r§   r¨   rÓ   rÄ   Ústaticmethodr¶  rC  r©   rÆ   rß   rà   s   @@r+   r¢  r¢  ô  s6   ù‡ € ÷ó ÷ 9;ò 9;ðv ó3ó ð3÷_ … r-   r¢  c                   ó†   a a€ ] tR tRt oV3R lV 3R llt]]RV3R lR ll4       4       tR tR t	V3R lR	 lt
R
tVtV ;t# )ÚBltModeliy  c                ó    <€ V ^8„  d   QhRS[ /# r;  r<  )r*   rÈ   s   "€r+   r,   ÚBltModel.__annotate__z  s   ø€ ÷ ñ ™yñ r-   c                óÌ  <€ \         SV `  V4       R V n        Wn        \	        VP
                  4      V n        \        VP                  4      V n	        \        VP                  4      V n        VP                  \        VP                  4      ,          pVP                   V,          p\"        P$                  ! W1P
                  P&                  4      V n        V P                  P*                  '       d^   \-        VP.                  4      V n        V P0                  P3                  4        V P0                  P5                  4        F
  pR Vn        K  	  MRV n        V P9                  4        R# )FN)rÒ   rÓ   rQ  rÏ   rE  r  rd   r‘  Úglobal_configr“  r  r  Úlocal_decoderrW   rb   rX   rY   r[   r\   r×   rV   Úpatch_in_forwardr¢  Úpatcher_configÚpatcherÚevalÚ
parametersÚrequires_gradr[  )r½   rÏ   Únum_embeddingsÚtotal_vocab_sizeÚparamrÜ   s   &&   €r+   rÓ   ÚBltModel.__init__z  s  ø€ Ü‰Ñ˜Ô Ø&+ˆÔ#àŒÜ,¨V×-BÑ-BÓCˆÔÜ"6°v×7KÑ7KÓ"LˆÔÜ,¨V×-BÑ-BÓCˆÔØ×DÑDÄsÈ6×KnÑKnÓGoÕoˆØ!×?Ñ?À.ÕPÐÜ*,¯,ª,Ð7G×I^ÑI^×IjÑIjÓ*kˆÔ'Ø�;‰;×'×'Ð'Ü% f×&;Ñ&;Ó<ˆDŒLØ�L‰L×ÑÔØŸ™×0Ñ0Ö2�Ø&+�Ö#ò 3ð  ˆDŒLØ�‰Ör-   c                ó6  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          R	S[S[,          R
S[S[	,          /	# )r&   r`  Nr†   rô   r¿   rc  ra  rª  rõ   rZ   )
r6   re  rF   r   r­  rr   r   r   rs   r	   )r*   rÈ   s   "€r+   r,   rÒ  �  s·   ø€ ÷ C
ñ C
á×#Ñ# dÕ*ðC
ñ —|‘| dÕ*ðC
ñ Ÿ™ tÕ+ð	C
ñ
 ×&Ñ&¨Õ-ðC
ñ  �ðC
ñ ×(Ñ(¨4Õ/ðC
ñ ˜$•;ðC
ñ Ñ+Õ,ðC
ñ 
Ñ(Õ	(ñC
r-   c                ó0	  € VR J VR J,          '       d   \        R4      hV'       dp   Vf6   \        \        V P                  R7      \        V P                  R7      4      pM6\	        V\        4      '       g!   \        V\        V P                  R7      4      pVe   Tp	VP
                  w  r«pMnVP
                  w  r«\        VV P                  V P                  V P                  P                  V P                  P                  V P                  P                  4      p	VEf$   V P                  P                  R8X  d“   V P                  e…   Vf   \        R4      hV P                  VV P                  P                  V P                  P                  V P                  P                   V P                  P"                  VP$                  R7      w  rÂpMvVe   VP$                  MVP$                  pVe   VP&                  MVP&                  p\)        \*        P,                  ! W«^,           3WíR7      V P                  P                   4      pV P/                  W+4      pVfb   Ve   VP1                  4       M^ p\*        P2                  ! V	P
                  ^,          V	P$                  R7      V,           pVP5                  ^ 4      p\7        V P                  T	TVe   VP8                  MR VR7      p\;        VVP
                  ^,          VR	V P                  P<                  V	P&                  R
7      pV P                  ! RRTRT	RTRTRTRVP
                  ^,          RTRVe   VP8                  MR /VB w  ppVP?                  W¢P
                  ^,          R4      p\*        P2                  ! ^ VP
                  ^,          VP$                  R7      pVP5                  ^ 4      p\7        V P                  VR R R R7      pV P@                  ! RRVRVRV/VB pV P/                  VR,          V4      p\;        VVP
                  ^,          VRV P                  P<                  V	P&                  R
7      pV PB                  ! RRTRTRTRTRTRVe   VPD                  MR RV/VB p\G        VVR7      # )Nr¯  rM  rµ  z0input_ids is required for entropy-based patching)r«  r¬  r‡   Úpatching_batch_sizer1   r/   r2   r°  T)rm   rn   ro   rp   rq   r0   r`  ra  rô   r¿   rd  rn   rm   rc  Frb  )Úlast_hidden_staterc  r£   r5   r½  )$rx   r   r   rÏ   r¶   r:   rk   rd   rV   rW   rX   rY   Úpatching_moderØ  r«  Úpatching_thresholdr‡   râ  r1   r0   r    r6   r·  Ú_patch_ids_from_lengthsr²  r9   ru   r   Úself_attention_cacher„   rq   rü   r“  rÕ  Úcross_attention_cacher	   )r½   r`  r†   rô   r¿   rc  ra  rª  rõ   Úencoder_embedsrL   ro   r  r1   r0   rm   r¸  r¹  Úcross_attn_mask_encÚencoder_hidden_statesrp  Úglobal_position_idsÚglobal_causal_maskÚglobal_hidden_statesÚdecoder_patch_idsÚcross_attn_mask_decÚoutputs   &&&&&&&&,                  r+   rÄ   ÚBltModel.forwardŽ  s«  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZçØÒ&Ü"5Ü ¨¯©Ô4´lÈ$Ï+É+Ô6Vó#‘ô   Ô1D×EÒEô #6°oÄ|Ð[_×[fÑ[fÔGgÓ"h�ð Ò$Ø*ˆNØ-:×-@Ñ-@Ñ*ˆJ©à*3¯/©/Ñ'ˆJÜ4ØØ×"Ñ"Ø×/Ñ/Ø—‘×@Ñ@Ø—‘×8Ñ8Ø—‘×9Ñ9óˆNð Ó Ø�{‰{×(Ñ(¨IÔ5¸$¿,¹,Ò:RØÒ$Ü$Ð%WÓXÐXØ&*§l¡lØØ#Ÿ{™{×5Ñ5Ø"Ÿk™k×<Ñ<Ø%)§[¡[×%AÑ%AØ(,¯©×(GÑ(GØ$×+Ñ+ð '3ó 'Ñ#�¡!ð .7Ò-B˜×)Ò)È×H\ÑH\�Ø+4Ò+@˜	ŸšÀm×FYÑFY�Ü 5Ü—J’J 
¸aÕ,?Ð@ÈÔ]Ø—K‘K×0Ñ0ó!�ð ×0Ñ0°ÓPˆ	àÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(<Ñ(<¸QÕ(?È×H]ÑH]Ô^ÐaqÕqˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø(Ø)ØDSÒD_˜O×@Ò@ÐeiØ%ô
ˆô BØØ%×+Ñ+¨AÕ.Ø+Ø#ØŸ™×1Ñ1Ø ×&Ñ&ô
Ðð 7;×6HÒ6Hñ 
7
Øð
7
à(ð
7
ð 'ð
7
ð &ð	
7
ð
 $7ð
7
ð &×+Ñ+¨AÕ.ð
7
ð  ð
7
ð ETÒD_˜O×@Ò@Ðeið
7
ð ñ
7
Ñ3ÐÐ3ð  4×8Ñ8¸×EXÑEXÐYZÕE[Ð]_Ó`ÐÜ#Ÿlšl¨1Ð.B×.HÑ.HÈÕ.KÐTh×ToÑToÔpÐØ1×;Ñ;¸AÓ>ÐÜ/Ø—;‘;Ø.ØØ Øô
Ðð  $×6Ò6ñ  
Ø.ð 
à-ð 
ð -ð 
ð ñ	 
Ðð !×8Ñ8¸ÀuÕ9MÈÓ_ÐÜAØ'Ø%×+Ñ+¨AÕ.Ø+Ø$ØŸ™×1Ñ1Ø ×&Ñ&ô
Ðð ×#Ò#ñ 	
Øð	
à/ð	
ð .ð	
ð 'ð		
ð
 &ð	
ð FUÒE`˜O×AÒAÐfjð	
ð $7ð	
ð ñ	
ˆô 'Ø$Ø+ô
ð 	
r-   c                ó.   € V P                   P                  # ræ   ©rd   r`   )r½   s   &r+   Úget_input_embeddingsÚBltModel.get_input_embeddings  s   € Ø×!Ñ!×.Ñ.Ð.r-   c                ó&   € WP                   n        R # ræ   rô  )r½   Úvalues   &&r+   Úset_input_embeddingsÚBltModel.set_input_embeddings  s   € Ø*/×ÑÖ'r-   c                óT   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  /# )r&   r†   rM   rZ   rE   )r*   rÈ   s   "€r+   r,   rÒ    s2   ø€ ÷ 
iñ 
i±U·\±\ð 
iÉCð 
iÑTY×T`ÑT`ñ 
ir-   c           	     ó¾  € VP                   ^ ,          p\        P                  ! \        P                  ! V^VP                  VP
                  R7      VP                  RR7      RRR13,          .RR7      p\        P                  ! W!P
                  R7      pVP                  ^4      VP                  ^ 4      P                  R4      8*  P                  RR7      ^,
          # )rŠ  r/   r3   r‰   Nr2   r5   )
r:   r6   rJ   rI   r0   r1   Úcumsumr9   ru   r;   )r½   r†   rM   rL   rÇ  Útoken_positionss   &&&   r+   ræ  Ú BltModel._patch_ids_from_lengths  s¾   € Ø"×(Ñ(¨Õ+ˆ
Ü—y’yä—’˜J¨°×1DÑ1DÈ]×MaÑMaÔbØ×$Ñ$¨Ð$Ó,¨Q°°°¨VÕ4ðð ô
ˆô  Ÿ,š, w×7KÑ7KÔLˆØ×&Ñ& qÓ)¨_×-FÑ-FÀqÓ-I×-SÑ-SÐTVÓ-WÑW×\Ñ\ÐacÐ\ÓdÐghÕhÐhr-   )rÏ   rV   r“  rQ  rÕ  rd   rØ  r�  )r¥   r¦   r§   r¨   rÓ   r   r   rÄ   rõ  rù  ræ  r©   rÆ   rß   rà   s   @@r+   rÐ  rÐ  y  sP   ù‡ € ÷ó ð(  Ø÷C
ñ C
ó ó  ðC
òJ/ò0÷
i÷ 
ið 
ir-   rÐ  zB
    The Blt Text Model with a language modeling head on top.
    )Úcustom_introc                   ó„   a a€ ] tR tRt oRtRtRR/tV3R lV 3R llt]]	RV3R lR	 ll4       4       t
V3R
 ltRtVtV ;t# )ÚBltForCausalLMi(  FÚmodelz'model.local_encoder.embed_tokens.weightzlm_head.weightc                ó    <€ V ^8„  d   QhRS[ /# r;  r<  )r*   rÈ   s   "€r+   r,   ÚBltForCausalLM.__annotate__3  s   ø€ ÷ ñ ™yñ r-   c                ó*  <€ \         SV `  V4       VP                  4       V n        VP                  V n        \        V4      V n        \        P                  ! VP                  P                  VP                  R R7      V n        V P                  4        R# )Fr™  N)rÒ   rÓ   Úget_text_configÚtext_configrW  rÐ  r  r[   r.  r  r×   r§  r[  )r½   rÏ   rÜ   s   &&€r+   rÓ   ÚBltForCausalLM.__init__3  sk   ø€ Ü‰Ñ˜Ô Ø!×1Ñ1Ó3ˆÔØ ×+Ñ+ˆŒÜ˜fÓ%ˆŒ
Ü—y’y ×!6Ñ!6×!BÑ!BÀF×DUÑDUÐ\aÔbˆŒà�‰Ör-   c                ó  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[S[ P                  S[ P                  3,          R,          RS[R,          R	S[ P
                  R,          R
S[ P                  R,          RS[R,          RS[S[ P                  ,          RS[S[	,          RS[S[
,          /# )r&   r`  Nrô   r¿   ró   r€   Úfull_text_row_masked_out_maskrc  ra  Úlabelsrª  Úlogits_to_keeprõ   rZ   )r6   re  rF   rs   r   r­  rr   r)   r   r   r
   )r*   rÈ   s   "€r+   r,   r  >  s  ø€ ÷ V
ñ V
á×#Ñ# dÕ*ðV
ñ Ÿ™ tÕ+ðV
ñ ×&Ñ&¨Õ-ð	V
ñ
 !&× 0Ñ 0°4Õ 7ðV
ñ $×.Ñ.°Õ5ðV
ñ (-©U¯\©\¹5¿<¹<Ð-GÕ'HÈ4Õ'OðV
ñ  �ðV
ñ ×(Ñ(¨4Õ/ðV
ñ × Ñ  4Õ'ðV
ñ ˜$•;ðV
ñ ™eŸl™lÕ*ðV
ñ Ñ+Õ,ðV
ñ 
Ñ'Õ	'ñV
r-   c                óž  € V P                   ! RRVRVRVRVRVRVRVRV
/VB pVP                  p\        V\        4      '       d   \	        V) R	4      MTpV P                  VR
VR
3,          4      P                  4       pR	pV	e   V P                  ! VW�P                  3/ VB p\        VVVP                  VP                  VP                  R7      # )aÕ	  
cross_attention_states (`torch.FloatTensor`, *optional*):
    Output of the vision model, used for cross-attention. This tensor contains the processed image features that
    the language model will attend to.
cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
    Cross-attention mask to control the interaction between text tokens and image tiles.
    This 4D tensor defines which image tiles each text token should attend to.

    For each text token (in seq_length):
    - 1 indicates the token **should attend** to the corresponding image tile
    - 0 indicates the token **should not attend** to the corresponding image tile
full_text_row_masked_out_mask (`tuple[torch.Tensor, torch.Tensor]`, *optional*):
    A tuple containing two tensors that mask out rows in the cross-attention mechanism:
    - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1.
      A value of 0 indicates that the corresponding text token's entire row in the cross-attention
      matrix should be masked out (all image tokens ignored).
    - The second tensor has the same shape and is used internally to apply the masking during
      the forward pass of cross-attention layers.
    This mask is derived from the cross_attention_mask and is used to handle cases where a text token
    should not attend to any image token.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, BltForCausalLM

>>> model = BltForCausalLM.from_pretrained("itazap/blt-1b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("itazap/blt-1b-hf")

>>> prompt = "If I had to write a haiku, it would be:"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=40, do_sample=True, temperature=0.6)
>>> result = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
>>> print(result)
If I had to write a haiku, it would be: "Snowflakes gently fall" - simple, yet peaceful.
I love the idea of snowflakes gently falling, each one
```
r`  rô   r¿   r€   r  rc  ra  rª  Nr‰   )ÚlossrŒ  rc  rò   r  r£   )r  rã  r¶   r)   Úslicer§  rµ   Úloss_functionrW  r
   rc  rò   r  )r½   r`  rô   r¿   ró   r€   r  rc  ra  r  rª  r  rõ   Úoutputsrò   Úslice_indicesrŒ  r  s   &&&&&&&&&&&&,     r+   rÄ   ÚBltForCausalLM.forward<  s  € ð| —*’*ñ 

Øð

à)ð

ð &ð

ð "6ð	

ð
 +Hð

ð ,ð

ð (ð

ð  ð

ð ñ

ˆð  ×1Ñ1ˆÜ8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓA×GÑGÓIˆàˆØÒØ×%Ò% f¨f·o±oÑPÈÑPˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r-   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r;  r<  )r*   rÈ   s   "€r+   r,   r  (  s   ø‡ ‚ ñ Ñò r-   )r§  r  r  rW  )NNNNNNNNNNrŠ  )r¥   r¦   r§   r¨   Ú_can_compile_fullgraphÚbase_model_prefixÚ_tied_weights_keysrÓ   r   r   rÄ   rC  r©   rÆ   rß   rà   s   @@r+   r  r  (  sT   ù‡ € ð #ÐØÐØCÐEUÐVÐ÷ó ð Ø÷V
ñ V
ó ó ðV
÷- … r-   r  )r_   )r&   r_   i0u  )r  rÐ  r¢  r  )Sr  Úcollections.abcr   r6   Útorch.distributionsÚtorch.nnr[   Útorch.nn.functionalÚ
functionalrj  Ú r   r*  Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   Úmodeling_outputsr	   r
   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   r   Úcohere2.modeling_cohere2r   Úllama.modeling_llamar   Úmllama.modeling_mllamar   r   r   r   r   r   r   Úconfiguration_bltr    r!   r"   r#   r$   Ú
get_loggerr¥   Úloggerr@   rS   rk   Úfloat32r„   r    r¢   r¬   r®   rÊ   rÔ   ré   r  rE  r  r‘  r¢  rÐ  r  Ú__all__r£   r-   r+   Ú<module>r1     s”  ðñ Cå $ã Û Ý ß Ð å &ß CÑ CÝ )Ý /ß OÝ 6ß FÝ &ß RÓ Rß Gß EÝ 2Ý 7÷÷ ñ ÷õ ð 
×	Ò	˜HÓ	%€÷:÷:õ$#ðT  %ØØŸ™÷K õ\,(ô^	ˆ]ô 	ô	Ð"ô 	ô<Ð-ô <ô"`Ð9ô `ô,Ð.ô ,ô
,)Ð0ô ,)ð^ ô^Ð.ó ^ó ð^ôBn"Ð(ô n"ôbIÐ(ô IôX0Ð-ô 0ôfBÐ#ô BôJliÐ!ô liñ^ ðôô
g
Ð'¨ó g
óð
g
òT‚r-   