+
    œ°³j7û  ã                   ó€  € ^ RI t ^ RIHt ^ RIt^ RIHt ^ RIHu Ht ^ RI	H
t
 ^RIHt ^RIHt ^RIHtHt ^RIHt ^RIHt ^R	IHt ^R
IHt ^RIHtHtHt ^RIH t  ^RI!H"t"H#t# ^RI$H%t%H&t& ^RI'H(t( ^RI)H*t*H+t+H,t,H-t-H.t. ^RI/H0t0 ^RI1H2t2 ^RI3H4t4H5t5H6t6H7t7 ^RI8H9t9 ^RI:H;t;H<t< ^RI=H>t> ^RI?H@t@HAtAHBtB ]-P†                  ! ]D4      tE]+! RR7      ]
 ! R R]4      4       4       tF]+! RR7      ]
 ! R R]4      4       4       tG]+! RR7      ]
 ! R R]4      4       4       tH]+! RR R!7      ]
 ! R" R#]4      4       4       tI ! R$ R%]74      tJ ! R& R']44      tK ! R( R)];4      tL ! R* R+]Pš                  4      tNRVR, R- lltO ! R. R/]Pš                  4      tP ! R0 R1]64      tQ ! R2 R3]>4      tR ! R4 R5]Pš                  4      tS ! R6 R7]Pš                  4      tT ! R8 R9]Pš                  4      tU]! ]B]A.4       ! R: R;]Pš                  4      4       tV ! R< R=] 4      tW]+ ! R> R?]&4      4       tX]+ ! R@ RA]X4      4       tY ! RB RC]54      tZ ! RD RE]94      t[]+ ! RF RG]X4      4       t\ ! RH RI]Pš                  4      t]RJ RK lt^RWRL RM llt_]+ ! RN RO]X4      4       t`]+! RPRQ7       ! RR RS]X4      4       ta]+! RPRQ7       ! RT RU]X]4      4       tb. RXOtcR# )Yé    N)ÚCallable)Ústrict)Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)ÚGenerationMixin)Úuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚtorch_compilable_check)Úmerge_with_config_defaults)Úcapture_outputs)ÚGemma3CausalLMOutputWithPastÚGemma3ForCausalLMÚ	Gemma3MLPÚGemma3ModelOutputWithPast)ÚHiggsAudioV2Embeddings)ÚLlamaRMSNormÚ	repeat_kv)ÚMixtralExperts)Úapply_mask_to_padding_statesÚcausal_conv1d_fnÚcausal_conv1d_updatezthinkingmachines/Inkling)Ú
checkpointc                   ó�  a a€ ] tR t^<t oRtRtRtRRRRRR	R
RRRRRRR	RRRRRRRR	/tRR.R.3RRR.R.3RR.R.3/tRRRRRRR
R/t	RRRR R!R"R#R$R%R&/t
R'tR(tR)t^Bt^@t^t^€t^@t^t^€tR*t^tR+tR(tR,tR(tR(tR-tR.t^tR(tR/t R0t!R1t"R2t#^t$^t%R3t&R4t'R5t(R.t)R6t*R7t+R(t,^t-^t.R(t/R8t0R3t1R(t2V 3R9 lt3]4R: 4       t5]4R; 4       t6V3R< lt7R=t8Vt9V ;t:# )>ÚInklingTextConfiga$  
unpadded_vocab_size (`int`, *optional*, defaults to `None`):
    Number of rows the checkpoint's unembedding matrix actually holds when the head is not padded to
    `vocab_size`. Logits beyond it are dropped. If `None`, the head is not padded.
swa_num_attention_heads (`int`, *optional*, defaults to 64):
    Number of attention heads in the sliding-window layers.
swa_num_key_value_heads (`int`, *optional*, defaults to 16):
    Number of key/value heads in the sliding-window layers.
swa_head_dim (`int`, *optional*, defaults to 128):
    Dimension of query and key heads in the sliding-window layers.
sliding_window_size (`int`, *optional*, defaults to 512):
    Size of the sliding attention window used by layers whose `layer_types` entry is `"hybrid_sliding"`.
d_rel (`int`, *optional*, defaults to 16):
    Per-head dimension of the relative states that are mixed into the relative position bias.
rel_extent (`int`, *optional*, defaults to 1024):
    Backward distance, in tokens, over which the relative position bias is applied. The bias is zero beyond it.
log_scaling_n_floor (`int`, *optional*, defaults to `None`):
    Position from which logits start being scaled up logarithmically in the full-attention layers. If `None`,
    the scaling is disabled.
log_scaling_alpha (`float`, *optional*, defaults to 0.1):
    Strength of the logarithmic logit scaling controlled by `log_scaling_n_floor`.
local_layer_ids (`list[int]`, *optional*, defaults to `None`):
    Indices of the layers using sliding window attention. Used to derive `layer_types` when it is not provided.
    If `None`, every layer whose index is not a multiple of 6 uses sliding window attention.
mlp_layer_types (`list[str]`, *optional*, defaults to `None`):
    MLP type pattern for each layer (`"dense"` or `"sparse"`). If `None`, every layer is sparse.
shared_expert_sink (`bool`, *optional*, defaults to `True`):
    Whether the router scores the shared experts alongside the routed ones, so that they act as a sink in the
    softmax over expert weights.
logits_mup_width_multiplier (`float`, *optional*, defaults to 24.0):
    muP width multiplier the final hidden states are divided by before the language modeling head.
rms_norm_eps_moe_gate (`float`, *optional*, defaults to 1e-6):
    Epsilon of the RMS normalization applied inside the mixture-of-experts router.
num_mtp_layers (`int`, *optional*, defaults to `None`):
    Number of multi-token-prediction layers. If `None`, multi-token prediction is disabled.
chain_hidden_post_norm (`bool`, *optional*, defaults to `False`):
    Whether the hidden states chained between multi-token-prediction layers are normalized after each layer.
mtp_hidden_states_first (`bool`, *optional*, defaults to `True`):
    Whether the hidden states come before the token embeddings when the two are concatenated as the input of a
    multi-token-prediction layer.
mtp_local_layer_ids (`list[int]`, *optional*, defaults to `None`):
    Indices of the multi-token-prediction layers using sliding window attention. If `None`, every
    multi-token-prediction layer uses full attention.
Úinkling_textÚtext_configÚembed_tokensÚembedding_rowwisez!layers.*.mlp.experts.gate_up_projÚcolwisezlayers.*.mlp.experts.down_projÚrowwisezlayers.*.mlp.expertsÚmoe_tp_expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.shared_expertsÚ
all_reducezlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚlayersÚhidden_statesÚattention_maskÚnormzlayers.*.mlp.gateÚ	ep_routerÚgrouped_gemmÚembedding_multiplierÚlogits_mup_width_multiplierÚsliding_windowÚsliding_window_sizeÚnum_local_expertsÚn_routed_expertsÚsconv_kernel_sizeÚconv_kernel_sizeÚmodel_max_lengthÚmax_position_embeddingsi@ Né   i   é   gš™™™™™¹?i   ç�íµ ÷Æ°>i `  Úsilui   é   Tg       @g      8@ç        ç{®Gáz”?Fc                óˆ  <€ V P                   f’   V P                  e   \        V P                  4      pM;\        V P                  4       Uu0 uF  q3^,           ^,          '       g   K  VkK  	  pp\        V P                  4       Uu. uF  q3V9   d   RMRNK  	  upV n         V P
                  fE   VP                  R^ 4      p\        V P                  4       Uu. uF  q3V8  d   RMRNK  	  upV n        VP                  R4      e   VP                  R4      V n        ^V n	        \        SV `,  ! R/ VB  R # u upi u upi u upi )NÚhybrid_slidingÚhybridÚdense_mlp_idxÚdenseÚsparseÚdense_intermediate_size© )Úlayer_typesÚlocal_layer_idsÚsetÚrangeÚnum_hidden_layersÚmlp_layer_typesÚpopÚgetÚintermediate_sizeÚnumber_of_conv_statesÚsuperÚ__post_init__)ÚselfÚkwargsrU   ÚirO   Ú	__class__s   &,   €Úl/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/inkling/modular_inkling.pyr_   ÚInklingTextConfig.__post_init__¿   s*  ø€ Ø×ÑÒ#Ø×#Ñ#Ò/Ü"% d×&:Ñ&:Ó";‘ä.3°D×4JÑ4JÔ.KÓ"[Ñ.K¨ÐTUÕPUÐYZÏ{É{§1 1Ñ.K�Ð"[äPUÐVZ×VlÑVlÔPmó ÙPmÈ1¨Ô$8Ñ ¸hÒFÑPmñ ˆDÔð ×ÑÒ'Ø"ŸJ™J ¸Ó:ˆMÜX]Ð^b×^tÑ^tÔXuÓ#vÑXuÐST°=Ô/@¡GÀhÒ$NÑXuÑ#vˆDÔ à�:‰:Ð/Ó0Ò<Ø%+§Z¡ZÐ0IÓ%JˆDÔ"ð &'ˆÔ"ä‰ÒÑ' Ô'ùò #\ùò ùò
 $ws   ÁD5Á'D5ÂD:ÃD?c                óÞ   € V P                   eZ   V P                  f   R.V P                   ,          # \        V P                   4       Uu. uF  qV P                  9   d   RMRNK  	  up# R # u upi )NrN   rM   )Únum_mtp_layersÚmtp_local_layer_idsrW   )r`   rb   s   & rd   Úmtp_layer_typesÚ!InklingTextConfig.mtp_layer_typesÔ   sv   € à×ÑÒ*Ø×'Ñ'Ò/Ø �z D×$7Ñ$7Õ7Ð7ô ^cÐcg×cvÑcvÔ]wóÙ]wÐXY¨T×-EÑ-EÔ(EÑ$È8ÒSÑ]wñð ñ ùòs   ÁA*c                óJ   € V P                   e   R.V P                   ,          # R # )NrP   )rg   )r`   s   &rd   Úmtp_mlp_layer_typesÚ%InklingTextConfig.mtp_mlp_layer_typesà   s$   € à×ÑÒ*Ø�9˜t×2Ñ2Õ2Ð2Ùó    c                óÆ  <€ V ^8„  d   Qh/ S[ ;R&   S[ R,          ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R	&   S[ ;R
&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ R,          ;R&   S[;R&   S[S[ ,          R,          ;R&   S[S[,          R,          ;R&   S[ ;R&   S[;R&   S[ ;R&   S[S[,          R,          ;R&   S[ ;R&   S[;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R &   S[;R!&   S[;R"&   S[ R,          ;R#&   S[ R,          ;R$&   S[ R,          ;R%&   S[ R,          ;R&&   S[;R'&   S[;R(&   S[S[ ,          R,          ;R)&   # )*é   Ú
vocab_sizeNÚunpadded_vocab_sizeÚhidden_sizerX   Únum_attention_headsÚnum_key_value_headsÚhead_dimÚswa_num_attention_headsÚswa_num_key_value_headsÚswa_head_dimr>   Úd_relÚ
rel_extentÚlog_scaling_n_floorÚlog_scaling_alpharU   rT   rD   Úrms_norm_epsrB   rY   r\   Ú
hidden_actÚmoe_intermediate_sizer@   Únum_experts_per_tokÚn_shared_expertsÚshared_expert_sinkÚroute_scaler<   Úrms_norm_eps_moe_gateÚattention_dropoutÚinitializer_rangeÚpad_token_idÚbos_token_idÚeos_token_idrg   Úchain_hidden_post_normÚmtp_hidden_states_firstrh   )ÚintÚfloatÚlistÚstrÚbool)ÚformatÚ__classdict__s   "€rd   Ú__annotate__ÚInklingTextConfig.__annotate__<   s˜  ø‡ ‚ ñl Ññm ñp ˜t�Ñ*ñq ñr Ññs ñt Ññu ñv Ñ!ñw ñx Ñ ñy ñz Ññ{ ñ| !Ñ%ñ} ñ~ !Ñ%ñ ñ@ ÑñA ñB Ñ"ñC ñD �OñE ñF ÑñG ñH ˜t�Ñ*ñI ñJ Ñ"ñK ñL ™#•Y Õ%Ñ,ñM ñN ‘c•˜TÕ!Ñ(ñO ñP !Ñ)ñQ ñR ÑñS ñT ÑñU ñV ™#•Y Õ%Ñ,ñW ñX Ñ"ñY ñZ Ññ[ ñ^ Ñ%ñ_ ñ` Ñña ñb Ñ ñc ñd Ññe ñf Ñ#ñg ñh Ññi ñl "'Ñ-ñm ñn !Ñ'ño ñp Ñ"ñq ñr Ñ#ñs ñt ˜•*Ñ#ñu ñv ˜•*Ñ ñw ñx ˜•*Ñ ñy ñ| ˜$•JÑ%ñ} ñ~ !Ñ(ñ ñ@ "Ñ(ñA ñB ™c� TÕ)Ñ0òC rn   )r\   rT   rY   r]   );Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚbase_config_keyÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_maprq   rr   rs   rX   rt   ru   rv   rw   rx   ry   r>   rz   r{   r|   r}   rU   rT   rD   r~   rB   rY   r\   r   r€   r@   r�   r‚   rƒ   r„   r<   r…   r†   r‡   rˆ   r‰   rŠ   rg   r‹   rŒ   rh   r_   Úpropertyri   rl   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rc   r“   s   @@rd   r*   r*   <   só  ù‡ € ñ+ðZ  €JØ#€OàÐ+Ø+¨YØ(¨)ØÐ 0Ø/°Ø-¨yØ/°Ø% |Ø  )Ø 	Ø  )ðÐð 	˜+˜¨Ð(9Ð:Ø�OÐ%5Ð6¸Ð8IÐJØ�/Ð" _Ð$5Ð6ðÐð 	˜[Ø+¨^Ø(¨.ØÐ 0ð	Ðð 	Ð =ØÐ/ØÐ/ð 	Ð/àÐ5ð	€Mð €Jà&*ÐØ€KØÐØ!ÐØ ÐØ€HØ#%ÐØ#%ÐØ€LØ"ÐØ€EØ€JØ&*ÐØ"ÐØ(,€OØ$(€KØ#)ÐØ€LØÐØ(,€OØ"ÐØ€Jà!%ÐØÐØ ÐØÐØ#ÐØ€Kà)-ÐØ#'ÐØ"ÐØ#ÐØ#€LØ €LØ €Là!%€NØ#(ÐØ$(ÐØ,0Ðõ(ð* ñó ðð ñó ð÷K … rn   r*   c                   ó\   a € ] tR t^çt o RtRtRtRRRRRR	/t^PtR
t	Rt
RtRtV 3R ltRtV tR# )ÚInklingAudioConfigaz  
n_mel_bins (`int`, *optional*, defaults to 80):
    Number of mel-frequency bins per audio frame.
mel_vocab_size (`int`, *optional*, defaults to 256):
    Number of discrete bins each mel value is quantized into before being embedded.
text_hidden_size (`int`, *optional*, defaults to 6144):
    Dimensionality the audio embeddings are projected to, matching the text backbone.
Úinkling_audioÚaudio_configÚnum_codebooksÚ
n_mel_binsÚcodebook_sizeÚmel_vocab_sizers   Útext_hidden_sizerI   rE   rG   rK   c                óV   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[;R&   S[;R&   # )rp   r¬   r®   r¯   r~   r‡   ©r�   rŽ   )r’   r“   s   "€rd   r”   ÚInklingAudioConfig.__annotate__ç   sF   ø‡ ‚ ñ( Ññ) ñ* Ññ+ ñ, Ñ ñ- ñ. Ññ/ ñ0 Ñ#ò1 rn   rS   N)r–   r—   r˜   r™   rš   r›   rœ   r    r¬   r®   r¯   r~   r‡   r¢   r£   r¤   ©r“   s   @rd   r¨   r¨   ç   sR   ø‡ € ñð !€JØ$€Oà˜ØÐ)ØÐ)ð€Mð €JØ€NØ ÐØ€LØ#Ð÷1 ƒ rn   r¨   c                   ód   a € ] tR tRt o RtRtRtRR/tRt^(t	^t
^tRt^t^tR	tR
tV 3R ltRtV tR# )ÚInklingVisionConfigi  z¨
text_hidden_size (`int`, *optional*, defaults to 6144):
    Dimensionality the vision features are projected to by the last encoder layer, matching the text backbone.
Úinkling_visionÚvision_configrX   Ún_layersrE   rF   rG   rK   c                ó†   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[;R&   S[;R	&   # )
rp   r¯   Ú
patch_sizeÚtemporal_patch_sizeÚnum_channelsrs   rX   rt   r~   r‡   r±   )r’   r“   s   "€rd   r”   Ú InklingVisionConfig.__annotate__  sz   ø‡ ‚ ñ Ñ ñ ñ Ññ ñ Ñ ñ ñ Ññ ñ  Ññ! ñ" Ññ# ñ$ Ñ!ñ% ñ& Ññ' ñ( Ñ#ò) rn   rS   N)r–   r—   r˜   r™   rš   r›   rœ   r    r¯   rº   r»   r¼   rs   rX   rt   r~   r‡   r¢   r£   r¤   r³   s   @rd   rµ   rµ     sY   ø‡ € ñð
 "€JØ%€OØ(¨*Ð5€Mà ÐØ€JØ ÐØ€LØ€KØÐØ!ÐØ€LØ#Ð÷) ƒ rn   rµ   zETop-level multimodal config (`InklingMMConfig` in the SGLang source).)r(   Úcustom_introc                   ór   a a€ ] tR tRt oRtRtR]R]R]/t	Rt
RtRtRtR	tR
tRtV 3R ltV3R ltRtVtV ;t# )ÚInklingConfigi  a  
image_bos_token_id (`int`, *optional*, defaults to 200005):
    The beginning-of-image token index used to mark the start of image spans.
audio_bos_token_id (`int`, *optional*, defaults to 200020):
    The beginning-of-audio token index used to mark the start of audio spans.
Úinkling_mm_modelr,   rª   r·   Niv iu iE iT c                óî  <€ VP                  R 4      ;'       g    / p\        V P                  \        4      '       dƒ   V P                  P	                  RVP                  R4      4       V P                  P	                  RVP                  RR4      4       V P                  P	                  RVP                  R4      4       \        V P
                  \        4      '       d*   V P                  R,          ! R/ V P
                  B V n        M+V P
                  f   V P                  R,          ! 4       V n        \        V P                  \        4      '       d*   V P                  R	,          ! R/ V P                  B V n        M+V P                  f   V P                  R	,          ! 4       V n        \        V P                  \        4      '       d*   V P                  R
,          ! R/ V P                  B V n        M+V P                  f   V P                  R
,          ! 4       V n        V P                  P                  V P                  n	        V P                  P                  V P
                  n	        \        SV `,  ! R/ VB  R# )Ú
mtp_configrg   Únum_nextn_predict_layersr‹   Frh   rU   rª   Nr·   r,   rS   )r[   Ú
isinstancer,   ÚdictÚ
setdefaultrª   Úsub_configsr·   rs   r¯   r^   r_   )r`   ra   rÃ   rc   s   &, €rd   r_   ÚInklingConfig.__post_init__5  sØ  ø€ à—Z‘Z Ó-×3Ð3°ˆ
Ü�d×&Ñ&¬×-Ò-Ø×Ñ×'Ñ'Ð(8¸*¿.¹.ÐIcÓ:dÔeØ×Ñ×'Ñ'Ð(@À*Ç.Á.ÐQiÐkpÓBqÔrØ×Ñ×'Ñ'Ð(=¸z¿~¹~ÐN_Ó?`Ôaä�d×'Ñ'¬×.Ò.Ø $× 0Ñ 0°Ö @Ñ UÀ4×CTÑCTÑ UˆDÕØ×ÑÒ&Ø $× 0Ñ 0°Ö @Ó BˆDÔä�d×(Ñ(¬$×/Ò/Ø!%×!1Ñ!1°/Ö!BÑ!XÀT×EWÑEWÑ!XˆDÕØ×ÑÒ'Ø!%×!1Ñ!1°/Ö!BÓ!DˆDÔä�d×&Ñ&¬×-Ò-Ø#×/Ñ/°Ö>ÑRÀ×AQÑAQÑRˆDÕØ×ÑÒ%Ø#×/Ñ/°Ö>Ó@ˆDÔà.2×.>Ñ.>×.JÑ.Jˆ×ÑÔ+Ø-1×-=Ñ-=×-IÑ-Iˆ×ÑÔ*Ü‰ÒÑ' Ô'rn   c                óÈ   <€ V ^8„  d   Qh/ S[ S[,          R,          ;R&   S[S[,          R,          ;R&   S[S[,          R,          ;R&   S[;R&   S[;R&   S[;R&   S[;R&   # )	rp   Nr,   rª   r·   Úimage_token_idÚaudio_token_idÚimage_bos_token_idÚaudio_bos_token_id)r*   rÆ   r¨   rµ   r�   )r’   r“   s   "€rd   r”   ÚInklingConfig.__annotate__  s~   ø‡ ‚ ñ( #¡TÕ)¨DÕ0Ñ7ñ) ñ* %¡tÕ+¨dÕ2Ñ9ñ+ ñ, '©Õ-°Õ4Ñ;ñ- ñ. Ñ ñ/ ñ0 Ñ ñ1 ñ2 Ñ$ñ3 ñ4 Ñ$ò5 rn   )rª   r,   r·   )r–   r—   r˜   r™   rš   r›   r*   r¨   rµ   rÈ   r,   rª   r·   rË   rÌ   rÍ   rÎ   r_   r¢   r£   r¤   r¥   r¦   s   @@rd   rÀ   rÀ     s]   ù‡ € ñð $€JàÐ(ØÐ*ØÐ,ð€Kð 48€KØ59€LØ7;€MØ €NØ €NØ$ÐØ$Ðõ(÷9 … rn   rÀ   c                   ó   € ] tR tRtRtR# )ÚInklingModelOutputWithPastiQ  rS   N©r–   r—   r˜   r™   r£   rS   rn   rd   rÑ   rÑ   Q  ó   † Ûrn   rÑ   c                   ó   € ] tR tRtRtR# )ÚInklingCausalLMOutputWithPastiU  rS   NrÒ   rS   rn   rd   rÕ   rÕ   U  rÓ   rn   rÕ   c                   ó   € ] tR tRtRtR# )ÚInklingRMSNormiY  rS   NrÒ   rS   rn   rd   r×   r×   Y  rÓ   rn   r×   c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚInklingRelativeLogitsi]  al  hidden states conditioned relative position bias. `proj` is a trained bank of bias-vs-distance profiles; each token's
`relative_states` mixes them into one bias value per backward distance
(`sglang RelLogitsProj` + the FA4 `score_mod`, materialized densely). The bias is zero
outside `0 <= distance < rel_extent`; causality and padding stay in the attention mask.
c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )rp   rz   r{   ©r�   )r’   r“   s   "€rd   r”   Ú"InklingRelativeLogits.__annotate__d  s   ø€ ÷ Añ A™cð A©sñ Arn   c                óŽ   <€ \         SV `  4        W n        \        P                  ! \
        P                  ! W4      4      V n        R # ©N)r^   Ú__init__r{   ÚnnÚ	ParameterÚtorchÚemptyÚproj)r`   rz   r{   rc   s   &&&€rd   rß   ÚInklingRelativeLogits.__init__d  s,   ø€ Ü‰ÑÔØ$ŒÜ—L’L¤§¢¨UÓ!?Ó@ˆŽ	rn   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )rp   Úrelative_statesÚquery_positionsÚkey_positionsÚreturn©râ   ÚTensor)r’   r“   s   "€rd   r”   rÜ   i  sI   ø€ ÷ ^ñ ^áŸ™ð^ñ Ÿ™ð^ñ —|‘|ð	^ñ
 
�‰ñ^rn   c                óˆ  € WP                   ,          P                  ^^4      pVR,          VR,          ,
          R,          pVP                  ^ V P                  ^,
          4      P                  ! . VP
                  R,          ORNRN5!  pVP                  RV4      pVP                  V^ 8  WPP                  8¬  ,          R4      # )é   :Nrp   NrJ   )ºNNNN)Nrï   )NNrï   rï   éÿÿÿÿ)rä   Ú	transposeÚclampr{   ÚexpandÚshapeÚgatherÚmasked_fill)r`   rç   rè   ré   Ú
rel_logitsÚdistanceÚgather_indexÚposition_biass   &&&&    rd   ÚforwardÚInklingRelativeLogits.forwardi  s¬   € ð &¯	©	Õ1×<Ñ<¸QÀÓBˆ
Ø# GÕ,¨}¸WÕ/EÕEÐGWÕXˆØ—~‘~ a¨¯©¸1Õ)<Ó=×DÒDÐcÀj×FVÑFVÐWYÕFZÐcÐ\^ÐcÐ`bÓcˆØ"×)Ñ)¨"¨lÓ;ˆØ×(Ñ(¨(°Q©,¸8ÇÁÑ;VÕ)WÐY\Ó]Ð]rn   )rä   r{   ©
r–   r—   r˜   r™   rš   rß   rû   r£   r¤   r¥   r¦   s   @@rd   rÙ   rÙ   ]  s(   ù‡ € ñ÷Aó A÷
^÷ ^ð ^rn   rÙ   c                ó  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        P                  R\        P                  R,          R\        R\        R	\        P                  R,          /# )
rp   ÚmoduleÚqueryÚkeyÚvaluer7   NÚscalingÚdropoutrú   )rà   ÚModulerâ   rì   rŽ   )r’   s   "rd   r”   r”   w  sz   € ÷ %ñ %Ü�I‰Ið%ä�<‰<ð%ô 
�‰ð%ô �<‰<ð	%ô
 —L‘L 4Õ'ð%ô ð%ô ð%ô —<‘< $Õ&ñ%rn   c                 ó:  € \        W P                  4      p	\        W0P                  4      p
\        P                  ! WP	                  ^^4      4      V,          pVe	   W·,           pVe	   W´,           p\
        P                  P                  VR\        P                  R7      P                  VP                  4      p\
        P                  P                  W¶V P                  R7      p\        P                  ! Wº4      pVP	                  ^^4      P                  4       pWË3# )rp   )ÚdimÚdtype)ÚpÚtrainingrð   )r#   Únum_key_value_groupsrâ   Úmatmulrñ   rà   Ú
functionalÚsoftmaxÚfloat32Útor  r  r
  Ú
contiguous)rÿ   r   r  r  r7   r  r  rú   ra   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs   &&&&&&&&,    rd   Úeager_attention_forwardr  w  sà   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÕL€LØÒ Ø#Õ3ˆØÒ!Ø#Õ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ó[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$rn   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚInklingAttentioni“  c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©rp   ÚconfigÚ	layer_idx©r*   r�   )r’   r“   s   "€rd   r”   ÚInklingAttention.__annotate__”  s    ø€ ÷ Tñ TÑ0ð T¹Sñ Trn   c                óP  <€ \         SV `  4        Wn        W n        VP                  V P                  ,          R 8H  V n        V P
                  '       d   VP                  MVP                  V n        V P
                  '       d   VP                  MVP                  V n
        V P
                  '       d   VP                  MVP                  V n        V P                  V P                  ,          V n        V P
                  '       d   VP                  MRV n        V P
                  '       d   VP                  MVP                   V n        RV P                  ,          V n        VP$                  V n        RV n        \(        P*                  ! VP,                  V P                  V P                  ,          RR7      V n        \(        P*                  ! VP,                  V P                  V P                  ,          RR7      V n        \(        P*                  ! VP,                  V P                  V P                  ,          RR7      V n        \(        P*                  ! VP,                  V P                  VP4                  ,          RR7      V n        \(        P*                  ! V P                  V P                  ,          VP,                  RR7      V n        \;        V P                  V P                  ,          VP<                  V^ R7      V n        \;        V P                  V P                  ,          VP<                  V^R7      V n         \C        V P                  VPD                  R7      V n#        \C        V P                  VPD                  R7      V n$        \K        VP4                  V P                   4      V n&        R# )rM   Nç      ð?TF©Úbias)Úconv_idx©Úeps)'r^   rß   r  r  rT   Ú
is_slidingry   rv   rw   rt   Ú	num_headsrx   ru   r  r>   r=   r{   r  r†   Ú	is_causalrà   ÚLinearrs   Úq_projÚk_projÚv_projrz   Úr_projÚo_projÚInklingShortConvolutionrA   Úk_sconvÚv_sconvr×   r~   Úq_normÚk_normrÙ   Úrel_logits_proj©r`   r  r  rc   s   &&&€rd   rß   ÚInklingAttention.__init__”  sm  ø€ Ü‰ÑÔØŒØ"ŒØ ×,Ñ,¨T¯^©^Õ<Ð@PÑPˆŒØ/3¯¯¨˜×+Ò+ÀFÇOÁOˆŒØ;?¿?¿?¸?˜×7Ò7ÐPV×PjÑPjˆŒØEIÇ_Ç_À_ 6×#AÒ#AÐZ`×ZtÑZtˆÔ Ø$(§N¡N°d×6NÑ6NÕ$NˆÔ!Ø<@¿O¿O¸O˜f×8Ò8ÐQUˆÔØ8<¿¿¸˜&×4Ò4ÈV×M^ÑM^ˆŒà˜TŸ]™]Õ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’i × 2Ñ 2°D·N±NÀTÇ]Á]Õ4RÐY^Ô_ˆŒÜ—i’i × 2Ñ 2°D×4LÑ4LÈtÏ}É}Õ4\ÐchÔiˆŒÜ—i’i × 2Ñ 2°D×4LÑ4LÈtÏ}É}Õ4\ÐchÔiˆŒÜ—i’i × 2Ñ 2°D·N±NÀVÇ\Á\Õ4QÐX]Ô^ˆŒÜ—i’i §¡°·±Õ >À×@RÑ@RÐY^Ô_ˆŒÜ.Ø×$Ñ$ t§}¡}Õ4°f×6NÑ6NÐPYÐdeô
ˆŒô /Ø×$Ñ$ t§}¡}Õ4°f×6NÑ6NÐPYÐdeô
ˆŒô % T§]¡]¸×8KÑ8KÔLˆŒÜ$ T§]¡]¸×8KÑ8KÔLˆŒÜ4°V·\±\À4Ç?Á?ÓSˆÖrn   c                ó   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[ P                  S[ P                  R,          3,          /# ©rp   r6   r7   NÚ	conv_maskÚpast_key_valuesra   rê   )râ   rì   r   r   r   Útuple)r’   r“   s   "€rd   r”   r  ³  s†   ø€ ÷ A)ñ A)á—|‘|ðA)ñ Ÿ™ tÕ+ðA)ñ —<‘< $Õ&ð	A)ñ
  �ðA)ñ Ñ+Õ,ðA)ñ 
‰u�|‰|™UŸ\™\¨DÕ0Ð0Õ	1ñA)rn   c                ó¸  € VP                   R R
 p. VOR
NV P                  N5pV P                  V4      pV P                  V P	                  V4      WCR7      p	V P                  V P                  V4      WCR7      p
V P                  V4      pV P                  VP                  V4      4      P                  ^^4      pV P                  V	P                  V4      4      P                  ^^4      p	V
P                  V4      P                  ^^4      p
VP                   ^,          pVeX   VP                  WÀP                  4      w  rÞVP                  V P                  4      pVP                  WšV P                  4      w  ršMV	P                   ^,          p^ ^ rï\         P"                  ! WÑP$                  R7      V,           p\         P"                  ! WÁP$                  R7      V,           pVP                  ! . VOV P&                  NR
N5!  pV P)                  VVV4      pV P*                  '       Eg   V P,                  P.                  eï   V^,           P1                  4       pRV P,                  P2                  \         P4                  ! VV P,                  P.                  ,          P7                  RR7      4      ,          ,           pVP                  ^^R
^4      pVP1                  4       V,          P9                  VP:                  4      pVP1                  4       V,          P9                  VP:                  4      p\<        P>                  ! V P,                  P@                  \B        4      pT! V VV	V
V3RV PD                  '       g   RMV PF                  RV PH                  RV PJ                  R	V/VB w  ppVPL                  ! . VOR
N5!  PO                  4       pV PQ                  V4      pVV3# )N©r:  r9  ©Údevicer   )Úminr  rJ   r  r=   rú   rð   ))rô   rv   r*  r0  r+  r1  r,  r-  r2  Úviewrñ   r3  Úget_mask_sizesr  Úget_query_offsetÚupdaterâ   Úaranger?  r'  r4  r&  r  r|   rŽ   r}   Úlogrò   r  r  r   Úget_interfaceÚ_attn_implementationr  r
  r†   r  r=   Úreshaper  r.  )r`   r6   r7   r9  r:  ra   Úinput_shapeÚhidden_shapeÚquery_statesr  r  rç   Úq_lengthÚ	kv_lengthÚ	kv_offsetÚq_offsetÚkv_positionsÚq_positionsrú   Úeffective_nÚtauÚattention_interfacer  r  s   &&&&&,                  rd   rû   ÚInklingAttention.forward³  s_  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1ˆØ—\‘\ $§+¡+¨mÓ"<Èo�\Ósˆ
Ø—|‘| D§K¡K°Ó$>ÐP_�|ÓuˆØŸ+™+ mÓ4ˆà—{‘{ <×#4Ñ#4°\Ó#BÓC×MÑMÈaÐQRÓSˆØ—[‘[ §¡°Ó!>Ó?×IÑIÈ!ÈQÓOˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà×%Ñ% aÕ(ˆØÒ&à#2×#AÑ#AÀ(ÏNÉNÓ#[Ñ ˆIØ&×7Ñ7¸¿¹ÓGˆHà'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJ˜à"×(Ñ(¨Õ+ˆIØ"# Q�iä—|’| I×6JÑ6JÔKÈiÕWˆÜ—l’l 8×4HÑ4HÔIÈHÕTˆØ)×.Ò.ÐP°ÐP¸T¿^¹^ÐPÈRÓPˆØ×,Ñ,¨_¸kÈ<ÓXˆð ��‰ 4§;¡;×#BÑ#BÒ#NØ&¨�?×1Ñ1Ó3ˆKØ˜Ÿ™×5Ñ5¼¿	º	Ø˜tŸ{™{×>Ñ>Õ>×EÑEÈ#ÐEÓNó9õ õ ˆCð —(‘(˜1˜a  QÓ'ˆCØ(×.Ñ.Ó0°3Õ6×:Ñ:¸<×;MÑ;MÓNˆLØ*×0Ñ0Ó2°SÕ8×<Ñ<¸]×=PÑ=PÓQˆMä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ%
ð  $Ÿ}Ÿ}˜}‘C°$×2HÑ2Hð%
ð —L‘Lð%
ð  ×.Ñ.ð%
ð (ð%
ð ñ%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ó;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rn   )r†   r  rv   r(  r&  r3  r+  r0  r  r'  r  ru   r.  r2  r*  r-  r{   r4  r  r=   r,  r1  ©NN©	r–   r—   r˜   r™   rß   rû   r£   r¤   r¥   r¦   s   @@rd   r  r  “  s#   ù‡ € ÷Tó T÷>A)÷ A)ò A)rn   r  c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú
InklingMLPi÷  c                ó    <€ V ^8„  d   QhRS[ /# ©rp   r  ©r*   )r’   r“   s   "€rd   r”   ÚInklingMLP.__annotate__ø  s   ø€ ÷ 8ñ 8Ñ0ñ 8rn   c                ó¼   <€ \         SV `  V4       \        VP                  ,          V n        \
        P                  ! \        P                  ! ^4      4      V n	        R# ©rî   N)
r^   rß   r   r   Úact_fnrà   rá   râ   ÚonesÚglobal_scale©r`   r  rc   s   &&€rd   rß   ÚInklingMLP.__init__ø  s;   ø€ Ü‰Ñ˜Ô Ü˜V×.Ñ.Õ/ˆŒÜŸLšL¬¯ª°A«Ó7ˆÖrn   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©rp   r6   rê   rë   )r’   r“   s   "€rd   r”   r^  ý  s#   ø€ ÷ 1ñ 1¡U§\¡\ð 1±e·l±lñ 1rn   c                ó°   € V P                  V P                  V P                  V4      4      V P                  V4      ,          4      pWP                  ,          # rÞ   )Ú	down_projra  Ú	gate_projÚup_projrc  ©r`   r6   s   &&rd   rû   ÚInklingMLP.forwardý  sC   € ØŸ™ t§{¡{°4·>±>À-Ó3PÓ'QÐTX×T`ÑT`ÐanÓToÕ'oÓpˆØ×0Ñ0Õ0Ð0rn   )ra  rc  rX  r¦   s   @@rd   rZ  rZ  ÷  s   ù‡ € ÷8ó 8÷
1÷ 1ð 1rn   rZ  c                   ó>   a a€ ] tR tRt oV3R lV 3R lltRtVtV ;t# )ÚInklingExpertsi  c                ó    <€ V ^8„  d   QhRS[ /# r\  r]  )r’   r“   s   "€rd   r”   ÚInklingExperts.__annotate__  s   ø€ ÷ =ñ =Ñ0ñ =rn   c                ój   <€ \         SV `  V4       VP                  V n        VP                  V n        R # rÞ   )r^   rß   r@   Únum_expertsr€   Úintermediate_dimrd  s   &&€rd   rß   ÚInklingExperts.__init__  s,   ø€ Ü‰Ñ˜Ô Ø!×2Ñ2ˆÔØ &× <Ñ <ˆÖrn   )rt  rs  )r–   r—   r˜   r™   rß   r£   r¤   r¥   r¦   s   @@rd   ro  ro    s   ù‡ € ÷=÷ =ó =rn   ro  c                   óD   a a€ ] tR tRt oV 3R ltV3R lR ltRtVtV ;t# )ÚInklingTopkRouteri	  c                ój  <€ \         SV `  4        VP                  V n        VP                  V n        V P                  V P                  ,           V n        VP                  V n        VP                  V n        VP                  V n
        \        P                  ! \        P                  ! V P
                  VP                  4      4      V n        \        P                  ! \        P                   ! ^4      4      V n        \        P$                  ! \        P&                  ! V P                  4      4      V n        R# r`  )r^   rß   r@   rs  r‚   Ún_total_expertsrs   Ú
hidden_dimr„   r�   Útop_krà   rá   râ   rã   Úweightrb  rc  ÚBufferÚzerosÚe_score_correction_biasrd  s   &&€rd   rß   ÚInklingTopkRouter.__init__
  sË   ø€ Ü‰ÑÔØ!×2Ñ2ˆÔØ &× 7Ñ 7ˆÔØ#×/Ñ/°$×2GÑ2GÕGˆÔØ ×,Ñ,ˆŒØ!×-Ñ-ˆÔØ×/Ñ/ˆŒ
ä—l’l¤5§;¢;¨t×/CÑ/CÀV×EWÑEWÓ#XÓYˆŒÜŸLšL¬¯ª°A«Ó7ˆÔÜ')§y¢y´·²¸T×=MÑ=MÓ1NÓ'OˆÖ$rn   c                óv   <€ V ^8„  d   QhRS[ S[P                  S[P                  S[P                  3,          /# ©rp   rê   )r;  râ   rì   )r’   r“   s   "€rd   r”   ÚInklingTopkRouter.__annotate__  s/   ø€ ÷ Hñ H©©e¯l©l¹E¿L¹LÉ%Ï,É,Ð.VÕ(Wñ Hrn   c           	     ón  € VP                  RV P                  4      p\        P                  ! W P                  4      pVP                  4       pVRRV P                  ) 13,          pWPP                  ,           p\        P                  ! W`P                  RRR7      ^,          pVRRV P                  ) 13,          pVRV P                  ) R13,          p	\        P                  ! VP                  RV4      V	.RR7      p
\        P                  ! V
4      p\        P                  ! V\        P                  ! VRRR7      ,
          4      pWÀP                   ,          V P"                  ,          pVRV P                  ) R13,          P%                  4       pVRRV P                  13,          P%                  4       pWŒW}3# )	rî   .NF)r  Úsorted©r  T)r  Úkeepdimrð   )rI  rz  ÚFÚlinearr|  Úsigmoidr‚   r  râ   Útopkr{  Úcatrõ   Ú
logsigmoidÚexpÚ	logsumexpr„   rc  r  )r`   r6   ÚflatÚrouter_logitsÚscoresÚrouted_scoresÚscores_for_choiceÚtopk_indicesÚrouted_logitsÚshared_logitsÚtopk_logitsÚtopk_log_probsÚtopk_weightsÚshared_gammass   &&            rd   rû   ÚInklingTopkRouter.forward  sy  € Ø×$Ñ$ R¨¯©Ó9ˆÜŸš §{¡{Ó3ˆð ×&Ñ&Ó(ˆØ˜sÐ$< t×'<Ñ'<Ð&<Ð$<Ð<Õ=ˆØ)×,HÑ,HÕHÐÜ—z’zÐ"3·Z±ZÀRÐPUÔVÐWXÕYˆà% cÐ+C¨d×.CÑ.CÐ-CÐ+CÐ&CÕDˆØ% c¨D×,AÑ,AÐ+AÑ+CÐ&CÕDˆÜ—i’i ×!5Ñ!5°b¸,Ó!GÈÐ WÐ]_Ô`ˆÜŸš kÓ2ˆÜ—y’y ´%·/²/À.ÐVXÐbfÔ2gÕ!gÓhˆà#×&6Ñ&6Õ6¸×9JÑ9JÕJˆà$ S¨4×+@Ñ+@Ð*@Ñ*BÐ%BÕC×NÑNÓPˆØ# C¨¨4¯:©:¨Ð$5Õ6×AÑAÓCˆà¨LÐGÐGrn   )	r  rc  rz  r‚   ry  rs  r„   r{  r|  rX  r¦   s   @@rd   rw  rw  	  s   ù‡ € õP÷H÷ Hð Hrn   rw  c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚInklingSharedExpertsi/  c                ó0  <€ \         SV `  4        VP                  V n        VP                  p\        P
                  ! \        P                  ! VP                  W!P                  4      4      V n	        \        P
                  ! \        P                  ! VP                  W!P                  4      4      V n
        \        P
                  ! \        P                  ! VP                  VP                  V4      4      V n        \        VP                  ,          V n        R # rÞ   )r^   rß   r‚   r€   rà   rá   râ   rã   rs   rj  rk  ri  r   r   ra  )r`   r  rt  rc   s   && €rd   rß   ÚInklingSharedExperts.__init__0  s¶   ø€ Ü‰ÑÔØ &× 7Ñ 7ˆÔØ!×7Ñ7Ðô Ÿš¤e§k¢k°&×2IÑ2IÐK[×]oÑ]oÓ&pÓqˆŒÜ—|’|¤E§K¢K°×0GÑ0GÐIY×[mÑ[mÓ$nÓoˆŒÜŸš¤e§k¢k°&×2IÑ2IÈ6×K]ÑK]Ð_oÓ&pÓqˆŒÜ˜V×.Ñ.Õ/ˆŽrn   c                óÐ  € VP                   pVP                  ^RVR,          4      P                  V P                  RR4      pVP                  RV P                  ^4      P	                  ^ ^4      p\
        P                  ! WP                  P	                  ^^4      4      p\
        P                  ! WP                  P	                  ^^4      4      pV P                  V4      V,          V,          p\
        P                  ! W`P                  P	                  ^^4      4      pVP                  4       P                  ^ R7      P                  VP                  4      pVP                  V4      # )rî   r†  rð   )rô   rI  ró   r‚   rñ   râ   Úbmmrj  rk  ra  ri  rŽ   Úsumr  r  rA  )	r`   r6   ÚgammasrJ  ÚgateÚupÚ	activatedÚdownÚouts	   &&&      rd   rû   ÚInklingSharedExperts.forward<  s  € Ø#×)Ñ)ˆØ%×-Ñ-¨a°°[Àµ_ÓE×LÑLÈT×MbÑMbÐdfÐhjÓkˆØ—‘  D×$9Ñ$9¸1Ó=×GÑGÈÈ1ÓMˆä�yŠy˜¯©×(@Ñ(@ÀÀAÓ(FÓGˆÜ�YŠY�}§l¡l×&<Ñ&<¸QÀÓ&BÓCˆØ—K‘K Ó%¨Õ*¨VÕ3ˆ	Ü�yŠy˜§N¡N×$<Ñ$<¸QÀÓ$BÓCˆà�j‰j‹l×Ñ 1ÐÓ%×(Ñ(¨×)<Ñ)<Ó=ˆØ�x‰x˜Ó$Ð$rn   )ra  ri  rj  r‚   rk  rX  r¦   s   @@rd   rž  rž  /  s   ù‡ € õ
0÷%ò %rn   rž  c                   óH   a a€ ] tR tRt oRtV 3R ltV3R lR ltRtVtV ;t	# )Ú
InklingMoEiJ  z7Gate -> routed experts (+ shared experts), TML flavour.c                ó�   <€ \         SV `  4        Wn        \        V4      V n        \        V4      V n        \        V4      V n        R # rÞ   )	r^   rß   r  rw  r¥  ro  Úexpertsrž  Úshared_expertsrd  s   &&€rd   rß   ÚInklingMoE.__init__M  s8   ø€ Ü‰ÑÔØŒÜ% fÓ-ˆŒ	Ü% fÓ-ˆŒÜ2°6Ó:ˆÖrn   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r‚  rë   )r’   r“   s   "€rd   r”   ÚInklingMoE.__annotate__T  s   ø€ ÷ ñ ©¯©ñ rn   c                ó   € TpVP                   pV P                  V4      w  rErgVP                  RVP                   R,          4      pV P                  WV4      P                  ! V!  pWP	                  W'R7      ,           pV# )rî   )r¤  rð   )rô   r¥  rA  r®  r¯  )r`   r6   Ú	residualsrJ  Ú_rš  r•  r›  s   &&      rd   rû   ÚInklingMoE.forwardT  sz   € Ø!ˆ	Ø#×)Ñ)ˆØ7;·y±yÀÓ7OÑ4ˆ˜Ø%×*Ñ*¨2¨}×/BÑ/BÀ2Õ/FÓGˆØŸ™ ]À,ÓO×TÒTÐVaÑbˆØ%×(;Ñ(;¸IÐ(;Ó(\Õ\ˆØÐrn   )r  r®  r¥  r¯  rý   r¦   s   @@rd   r¬  r¬  J  s   ù‡ € ÙAõ;÷÷ ð rn   r¬  c                   ój   a a€ ] tR tRt oV3R lV 3R llt]! R4      RV3R lR ll4       tRtVtV ;t	# )	r/  i^  c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ /# )rp   rs   rB   r  r#  rÛ   )r’   r“   s   "€rd   r”   Ú$InklingShortConvolution.__annotate__`  s*   ø€ ÷ 
ñ 
¡Cð 
¹3ð 
É3ð 
ÑZ]ñ 
rn   c           	     ó˜   <€ \         SV `  4        W0n        W@n        W n        \
        P                  ! VVVVV^,
          RR7      V n        R# )rî   F)Úin_channelsÚout_channelsÚkernel_sizeÚgroupsÚpaddingr"  N)r^   rß   r  r#  rB   rà   ÚConv1dÚconv1d)r`   rs   rB   r  r#  rc   s   &&&&&€rd   rß   Ú InklingShortConvolution.__init__`  sE   ø€ Ü‰ÑÔØ"ŒØ ŒØ 0Ôä—i’iØ#Ø$Ø(ØØ$ qÕ(Øô
ˆŽrn   rÁ  c          	      ó†   <€ V ^8„  d   QhRS[ P                  RS[R,          RS[ P                  R,          RS[S[,          /# )rp   r6   r:  Nr9  ra   ©râ   rì   r   r   r   )r’   r“   s   "€rd   r”   r¹  p  sJ   ø€ ÷ *ñ *á—|‘|ð*ñ  �ð*ñ —<‘< $Õ&ð	*ñ
 Ñ+Õ,ñ*rn   c                ó  € VP                   pVP                  4       pTp\        W4      pVP                  ^,          pVP	                  ^^4      pVRJ;'       d'    VP                  V P                  V P                  4      pV'       d²   V^8X  d«   VP                  V P                  ,          P                  '       g~   VP                  V P                  ,          P                  V P                  ,          p	\        WV P                  P                  P                  ^4      V P                  P                  4      pM�Ve3   VP!                  WP                  V P                  V P"                  R7      p\%        WP                  P                  P                  ^4      V P                  P                  VP'                  R4      R7      pVe   VRRV) R13,          pVP	                  ^^4      pW,           P)                  VR7      pV# )rî   N)Ú	state_idxrB   Úseq_idx)rÇ  rï   )r  )r  rŽ   r%   rô   rñ   Úhas_previous_stater  r#  r5   Úrecord_pastÚconv_statesr'   rÁ  r|  Úsqueezer"  Úupdate_conv_staterB   r&   r[   r  )
r`   r6   r:  r9  ra   Úinput_dtypeÚresidualÚseq_lenÚuse_precomputed_statesÚ
conv_states
   &&&&,     rd   rû   ÚInklingShortConvolution.forwardo  s¶  € ð $×)Ñ)ˆØ%×+Ñ+Ó-ˆà ˆÜ4°]ÓNˆØ×%Ñ% aÕ(ˆØ%×/Ñ/°°1Ó5ˆà!0¸Ð!<÷ "
ð "
À×AcÑAcØ�N‰N˜DŸM™MóB
Ð÷ " g°¤l¸?×;QÑ;QÐRV×R`ÑR`Õ;a×;m×;mÐ;mØ(×/Ñ/°·±Õ?×KÑKÈDÏMÉMÕZˆJä0Ø¨4¯;©;×+=Ñ+=×+EÑ+EÀaÓ+HÈ$Ï+É+×JZÑJZó‰Mð Ò*Ø /× AÑ AØ!§>¡>¸T¿]¹]Ð]a×]rÑ]rð !Bó !�ô -ØŸ{™{×1Ñ1×9Ñ9¸!Ó<¸d¿k¹k×>NÑ>NÐX^×XbÑXbÐclÓXmôˆMð
 Ò*Ø -¨a°°W°H±I¨oÕ >�à%×/Ñ/°°1Ó5ˆØ&Õ1×5Ñ5¸KÐ5ÓHˆØÐrn   )rÁ  r#  rB   r  rW  )
r–   r—   r˜   r™   rß   r   rû   r£   r¤   r¥   r¦   s   @@rd   r/  r/  ^  s2   ù‡ € ÷
ó 
ñ ˜HÓ%÷*ñ *ó &÷*ð *rn   r/  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚInklingDecoderLayeri�  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r  r  )r’   r“   s   "€rd   r”   Ú InklingDecoderLayer.__annotate__ž  s   ø€ ÷ 
ñ 
Ñ0ð 
¹Sñ 
rn   c                ó<  <€ \         SV `  4        VP                  V n        \        W4      V n        VP
                  V,          R 8X  d   \        V4      V n        M\        V4      V n        \        VP                  VP                  4      V n        \        VP                  VP                  4      V n        VP                  V,          V n        \        VP                  VP                   V^R7      V n        \        VP                  VP                   V^R7      V n        R# )rQ   )r  r#  N)r^   rß   rs   r  Ú	self_attnrY   r¬  ÚmlprZ  r×   r~   Úinput_layernormÚpost_attention_layernormrT   Ú
layer_typer/  rB   Ú
attn_sconvÚ	mlp_sconvr5  s   &&&€rd   rß   ÚInklingDecoderLayer.__init__ž  sá   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ)¨&Ó<ˆŒà×!Ñ! )Õ,°Ô8Ü! &Ó)ˆD�Hä! &Ó)ˆDŒHä-¨f×.@Ñ.@À&×BUÑBUÓVˆÔÜ(6°v×7IÑ7IÈ6×K^ÑK^Ó(_ˆÔ%Ø ×,Ñ,¨YÕ7ˆŒÜ1Ø×Ñ × 7Ñ 7À9ÐWXô
ˆŒô 1Ø×Ñ × 7Ñ 7À9ÐWXô
ˆŽrn   c                óÈ   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[ P                  /# r8  rÄ  )r’   r“   s   "€rd   r”   rÖ  ²  sk   ø€ ÷ ñ á—|‘|ðñ Ÿ™ tÕ+ðñ —<‘< $Õ&ð	ñ
  �ðñ Ñ+Õ,ðñ 
�‰ñrn   c                ó  € TpV P                  V4      pV P                  ! RR VRVRVRV/VB w  rV P                  WVR7      pWa,           pTpV P                  V4      pV P	                  V4      pV P                  WVR7      pWa,           pV# )r6   r7   r9  r:  r=  rS   )rÚ  rØ  rÝ  rÛ  rÙ  rÞ  )r`   r6   r7   r9  r:  ra   rÎ  rµ  s   &&&&&,  rd   rû   ÚInklingDecoderLayer.forward²  s¹   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ>š>ñ 
Ø'ð
à)ð
ð  ð
ð ,ð	
ð
 ñ
Ñˆð Ÿ™¨Ðbk˜ÓlˆØ Õ0ˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØŸ™ }Ðaj˜ÓkˆØ Õ0ˆØÐrn   )rÝ  rs   rÚ  rÜ  rÙ  rÞ  rÛ  rØ  )NNNrX  r¦   s   @@rd   rÔ  rÔ  �  s   ù‡ € ÷
ó 
÷(÷ ò rn   rÔ  c                   ó    a a€ ] tR tRt o]tRtRtR.tR.t	Rt
RtRtRtRtR.t. ROtR]R	]/t]P*                  ! 4       V 3R
 l4       tRtVtV ;t# )ÚInklingPreTrainedModeliÎ  ÚmodelTrÔ  r:  Fzmodel\.mtp\..*r6   Ú
attentionsc                ó  <€ \         SV `  V4       V P                  P                  4       P                  p\        V\        4      '       d&   \        P                  ! VP                  R VR7       R# \        V\        4      '       d#   \        P                  ! VP                  4       R# \        V\        4      '       dI   \        P                  ! VP                  R VR7       \        P                  ! VP                  R VR7       R# \        V\         4      '       df   \        P                  ! VP"                  R VR7       \        P                  ! VP                  4       \        P$                  ! VP&                  4       R# \        V\(        4      '       dl   \        P                  ! VP*                  R VR7       \        P                  ! VP,                  R VR7       \        P                  ! VP                  R VR7       R# \        V\.        4      '       dt   \1        V P                  RV P                  4      p\        P2                  ! VP4                  \6        P8                  ! VP:                  4      VP<                  ,          4       R# R# )rJ   )ÚmeanÚstdrª   N)r^   Ú_init_weightsr  Úget_text_configr‡   rÅ   rÙ   ÚinitÚnormal_rä   rZ  Úones_rc  ro  Úgate_up_projri  rw  r|  Úzeros_r  rž  rj  rk  ÚInklingAudioModelEmbeddingsÚgetattrÚcopy_Úaudio_tokens_offsetsrâ   rE  r¬   r®   )r`   rÿ   ré  rª   rc   s   &&  €rd   rê  Ú$InklingPreTrainedModel._init_weightsã  s“  ø€ ä‰Ñ˜fÔ%Ø�k‰k×)Ñ)Ó+×=Ñ=ˆÜ�fÔ3×4Ò4Ü�LŠL˜Ÿ™¨3°C×8Ü˜¤
×+Ò+Ü�JŠJ�v×*Ñ*Ö+Ü˜¤×/Ò/Ü�LŠL˜×,Ñ,°3¸CÕ@Ü�LŠL˜×)Ñ)°¸×=Ü˜Ô 1×2Ò2Ü�LŠL˜Ÿ™¨S°cÕ:Ü�JŠJ�v×*Ñ*Ô+Ü�KŠK˜×6Ñ6Ö7Ü˜Ô 4×5Ò5Ü�LŠL˜×)Ñ)°¸Õ=Ü�LŠL˜Ÿ™¨c°sÕ;Ü�LŠL˜×)Ñ)°¸×=Ü˜Ô ;×<Ò<ô # 4§;¡;°ÀÇÁÓLˆLÜ�JŠJØ×+Ñ+Ü—’˜\×4Ñ4Ó5¸×8SÑ8SÕSöñ	 =rn   rS   )rÝ  rÞ  r0  r1  )r–   r—   r˜   r™   rÀ   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚ"_keys_to_ignore_on_load_unexpectedÚ_keep_in_fp32_modules_strictrÔ  r  Ú_can_record_outputsrâ   Úno_gradrê  r£   r¤   r¥   r¦   s   @@rd   rä  rä  Î  s‰   ù‡ € à €LØÐØ&*Ð#Ø.Ð/ÐØ#4Ð"5Ðð !ÐØ€NØÐØ"ÐØ"'ÐØ*;Ð)<Ð&Ú#TÐ àÐ,ØÐ&ðÐð
 ‡]‚]ƒ_ôó ÷ð rn   rä  c                   ó~   a a€ ] tR tRt oV3R lV 3R llt]]]RV3R lR ll4       4       4       tV3R lt	Rt
VtV ;t# )	ÚInklingTextModeli   c                ó    <€ V ^8„  d   QhRS[ /# r\  r]  )r’   r“   s   "€rd   r”   ÚInklingTextModel.__annotate__  s   ø€ ÷ ñ Ñ0ñ rn   c           	     ó>  <€ \         SV `  V4       VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  V P                  4      V n        \
        P                  ! \        VP                  4       Uu. uF  p\        W4      NK  	  up4      V n        \        VP                  VP                  R 7      V n        \        VP                  VP                  R 7      V n        RV n        V P'                  4        R# u upi )r$  FN)r^   rß   rˆ   Úpadding_idxrq   rà   Ú	Embeddingrs   r-   Ú
ModuleListrW   rX   rÔ  r5   r×   r~   r8   Ú
embed_normÚgradient_checkpointingÚ	post_initr5  s   && €rd   rß   ÚInklingTextModel.__init__  sØ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜEJÈ6×KcÑKcÔEdÓeÑEd¸	Ô  Ö3ÑEdÑeó
ˆŒô # 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ü(¨×);Ñ);À×ATÑATÔUˆŒØ&+ˆÔ#ð 	�‰Öùò fs   ÂDc                óþ   <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[R,          RS[S[,          R	S[/# )
rp   r3   Nr7   Úposition_idsr:  r4   Ú	use_cachera   rê   )	râ   Ú
LongTensorrì   r   ÚFloatTensorr‘   r   r   r   )r’   r“   s   "€rd   r”   r    s“   ø€ ÷ 8
ñ 8
á×#Ñ# dÕ*ð8
ñ Ÿ™ tÕ+ð8
ñ ×&Ñ&¨Õ-ð	8
ñ
  �ð8
ñ ×(Ñ(¨4Õ/ð8
ñ ˜$•;ð8
ñ Ñ+Õ,ð8
ñ 
!ñ8
rn   c           
     ó:  € VR J VR J,          '       d   \        R4      hVf   V P                  V4      pV P                  V4      pV'       d   Vf   \        V P                  R7      pVfb   Ve   VP                  4       M^ p\        P                  ! VP                  ^,          VP                  R7      V,           pVP                  ^ 4      p\        T;p	\        4      '       g:   RV P                  RVRVRVRV/p
R	\        R/ V
B R
\        R/ V
B R\        R/ V
B /p	Tp\!        V P"                  4       FG  w  rÍV P                  P$                  V,          R8X  d   R	MR
pV! V3RWž,          RV	R,          RV/VB pKI  	  V P'                  V4      p\)        VVR7      # )Nú:You must specify exactly one of input_ids or inputs_embeds)r  r>  r  r4   r7   r:  r  Úfull_attentionÚsliding_attentionÚlinear_attentionrN   r9  )Úlast_hidden_stater:  rS   )Ú
ValueErrorr-   r  r   r  Úget_seq_lengthrâ   rE  rô   r?  Ú	unsqueezerÅ   rÆ   r   r   r   Ú	enumerater5   rT   r8   r   )r`   r3   r7   r  r:  r4   r  ra   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr6   rb   Údecoder_layerÚattention_types   &&&&&&&,       rd   rû   ÚInklingTextModel.forward  s»  € ð ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×-Ñ-¨iÓ8ˆMàŸ™¨Ó6ˆç˜Ò0Ü*°$·+±+Ô>ˆOàÒØCRÒC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÕ(>À}×G[ÑG[Ô\Ð_oÕoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-Ä×FÒFà˜$Ÿ+™+Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%UÈÑ%UØ"Ô$CÑ$RÀkÑ$Rð#Ðð &ˆÜ )¨$¯+©+Ö 6ÑˆAØ15·±×1HÑ1HÈÕ1KÈxÔ1WÑ-Ð]pˆNÙ)Øñà2ÕBðð .Ð.@ÕAðð !0ð	ð
 ñŠMñ !7ð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
rn   c                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # r\  r]  )r’   r“   s   "€rd   r”   r     s   ø‡ ‚ áÑò rn   )r  r-   r  r5   r8   r	  rq   )NNNNNN)r–   r—   r˜   r™   rß   r   r   r   rû   r¢   r£   r¤   r¥   r¦   s   @@rd   r  r     s?   ù‡ € ÷ó ð   ØØ÷8
ñ 8
ó ó ó  ð8
÷/ … rn   r  c                   ó^   a € ] tR tRt o / tRR/tRR/t]]R	V 3R lR ll4       4       t	Rt
V tR# )
ÚInklingForCausalLMiR  Úlm_headÚrowwise_split_inputÚkeep_full_weightNc                óP  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[S[ P                  ,          R
S[S[,          RS[	/
# )rp   r3   Nr7   r  r:  r4   Úlabelsr  Úlogits_to_keepra   rê   )
râ   r  rì   r   r  r‘   r�   r   r   rÕ   )r’   r“   s   "€rd   r”   ÚInklingForCausalLM.__annotate__Z  s»   ø€ ÷ 9
ñ 9
á×#Ñ# dÕ*ð9
ñ Ÿ™ tÕ+ð9
ñ ×&Ñ&¨Õ-ð	9
ñ
  �ð9
ñ ×(Ñ(¨4Õ/ð9
ñ × Ñ  4Õ'ð9
ñ ˜$•;ð9
ñ ™eŸl™lÕ*ð9
ñ Ñ+Õ,ð9
ñ 
'ñ9
rn   c	                óB  € V P                   ! RRVRVRVRVRVRV/V	B p
V
P                  V P                  P                  ,          p\	        V\
        4      '       d   \        V) R4      MTpV P                  VRVR3,          4      pV P                  P                  pVe%   WíP                  R,          8  d   VR	RV13,          pRpVe*   V P                  ! RR
VRVRVP                  R,          /V	B p\        VVV
P                  V
P                  V
P                  R7      # )a:  
Example:

```python
>>> from transformers import AutoTokenizer, InklingForCausalLM

>>> model = InklingForCausalLM.from_pretrained("thinkingmachines/Inkling-NVFP4")
>>> tokenizer = AutoTokenizer.from_pretrained("thinkingmachines/Inkling-NVFP4")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(**inputs, max_new_tokens=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```r3   r7   r  r:  r4   r  Nrï   .Úlogitsr,  rq   )Úlossr0  r:  r6   ræ  rS   rð   )rå  r  r  r<   rÅ   r�   Úslicer(  rr   rô   Úloss_functionrÕ   r:  r6   ræ  )r`   r3   r7   r  r:  r4   r,  r  r-  ra   Úoutputsr6   Úslice_indicesr0  rr   r1  s   &&&&&&&&&,      rd   rû   ÚInklingForCausalLM.forwardX  sJ  € ð> —*’*ñ 
Øð
à)ð
ð &ð
ð ,ð	
ð
 (ð
ð  ð
ð ñ
ˆð  ×1Ñ1°D·K±K×4[Ñ4[Õ[ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆØ"Ÿk™k×=Ñ=ÐØÒ*Ð/BÇ\Á\ÐRTÕEUÔ/UØ˜CÐ!5Ð"5Ð!5Ð5Õ6ˆFàˆØÒØ×%Ò%Ñj¨VÐj¸FÐjÈvÏ|É|Ð\^ÕO_ÐjÐciÑjˆDä,ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rn   rS   )NNNNNNNr   )r–   r—   r˜   r™   Ú_tied_weights_keysÚ_tp_planÚ
_fsdp_planr   r   rû   r£   r¤   r³   s   @rd   r'  r'  R  s@   ø‡ € àÐØÐ0Ð1€HØÐ/Ð0€JàØ÷9
ñ 9
ó ó ö9
rn   r'  c                   ó   € ] tR tRtRtR# )rñ  i–  rS   NrÒ   rS   rn   rd   rñ  rñ  –  s   † »3rn   rñ  c                   óZ   a a€ ] tR tRt oV3R lV 3R llt]V3R lR l4       tRtVtV ;t	# )ÚInklingAudioModeli™  c                ó    <€ V ^8„  d   QhRS[ /# r\  )r¨   )r’   r“   s   "€rd   r”   ÚInklingAudioModel.__annotate__›  s   ø€ ÷ Fñ FÑ1ñ Frn   c                ó~   <€ \         SV `  V4       \        V4      V n        \	        VP
                  R R7      V n        R# )rG   r$  N)r^   rß   rñ  Úembed_audio_tokensr×   r¯   r8   rd  s   &&€rd   rß   ÚInklingAudioModel.__init__›  s1   ø€ Ü‰Ñ˜Ô Ü"=¸fÓ"EˆÔÜ" 6×#:Ñ#:ÀÔEˆŽ	rn   c                ó:   <€ V ^8„  d   QhRS[ P                  RS[/# )rp   Úaudio_input_idsrê   )râ   rì   r   )r’   r“   s   "€rd   r”   r>  ¡  s    ø€ ÷ 

ñ 

¡u§|¡|ð 

ÑB\ñ 

rn   c                ó`   € V P                  V4      pV P                  V4      p\        VVR7      # )z‡
audio_input_ids (`torch.Tensor` of shape `(num_audios, max_num_frames, n_mel_bins)`):
    Mel-spectrogram frames of the input audios.
©r  Úpooler_output)r@  r8   r   )r`   rC  ra   r6   s   &&, rd   rû   ÚInklingAudioModel.forward   s5   € ð ×/Ñ/°Ó@ˆØŸ	™	 -Ó0ˆÜ)Ø+Ø'ô
ð 	
rn   )r@  r8   ©
r–   r—   r˜   r™   rß   r   rû   r£   r¤   r¥   r¦   s   @@rd   r<  r<  ™  s*   ù‡ € ÷Fó Fð
 ÷

ó ÷

ð 

rn   r<  c                   ób   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltV3R lR ltRtVtV ;t	# )	ÚInklingVisionEncoderLayeri®  c          
      ó8   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[/# )rp   Ú	input_dimÚ
output_dimÚt_foldÚhw_foldÚadd_norm)r�   r‘   )r’   r“   s   "€rd   r”   Ú&InklingVisionEncoderLayer.__annotate__¯  s1   ø€ ÷ !ñ !¡#ð !±3ð !Áð !Écð !Ñ]añ !rn   c                ó²   <€ \         SV `  4        \        P                  ! WR R7      V n        V'       d   \        V4      V n        W@n        W0n        WPn	        R# ©Fr!  N)
r^   rß   rà   r)  Ú
projectionr×   Ú
layer_normrO  rN  rP  )r`   rL  rM  rN  rO  rP  rc   s   &&&&&&€rd   rß   Ú"InklingVisionEncoderLayer.__init__¯  s>   ø€ Ü‰ÑÔÜŸ)š) IÀÔFˆŒßÜ,¨ZÓ8ˆDŒOØŒØŒØ Žrn   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rg  rë   )r’   r“   s   "€rd   r”   rQ  ¸  s#   ø€ ÷ ñ ±U·\±\ð ÁeÇlÁlñ rn   c           
     ó´  € VP                   w  r#rEpW0P                  ,          pW@P                  ,          pWPP                  ,          p	VP                  W'V P                  W€P                  W�P                  V4      pVP	                  ^ ^^^^^^^4      pVP                  W'W‰V P                  V P                  ,          V P                  ,          V,          4      pV# )zp
Convert a tensor of shape (B, T, H, W, C) to a tensor of shape (B, T // t, H // hw, W //  hw, C * (t * hw**2))
)rô   rN  rO  rI  Úpermute)
r`   r6   ÚBÚTÚHÚWÚCÚt_newÚh_newÚw_news
   &&        rd   Úfold_timespace_to_depthÚ1InklingVisionEncoderLayer.fold_timespace_to_depth¸  s»   € ð &×+Ñ+‰ˆˆa�Aà—[‘[Õ ˆØ—\‘\Õ!ˆØ—\‘\Õ!ˆà%×-Ñ-¨a¸¿¹ÀUÏLÉLÐZ_×amÑamÐopÓqˆà%×-Ñ-¨a°°A°q¸!¸QÀÀ1ÓEˆØ%×-Ñ-¨a¸ÀdÇkÁkÐTX×T`ÑT`ÕF`Ðcg×coÑcoÕFoÐrsÕFsÓtˆØÐrn   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rg  rë   )r’   r“   s   "€rd   r”   rQ  È  s#   ø€ ÷ ñ ¡U§\¡\ð ±e·l±lñ rn   c                ó   € V P                   ^8”  g   V P                  ^8”  d   V P                  V4      pV P                  V4      pV P                  '       d(   V P                  V4      p\        P                  ! V4      pV# )rî   )rO  rN  rb  rT  rP  rU  rˆ  Úgelurl  s   &&rd   rû   Ú!InklingVisionEncoderLayer.forwardÈ  sa   € Ø�<‰<˜!Ô˜tŸ{™{¨QœØ ×8Ñ8¸ÓGˆMàŸ™¨Ó6ˆØ�=�=ˆ=Ø ŸO™O¨MÓ:ˆMÜŸFšF =Ó1ˆMØÐrn   )rP  rO  rU  rT  rN  )
r–   r—   r˜   r™   rß   rb  rû   r£   r¤   r¥   r¦   s   @@rd   rJ  rJ  ®  s(   ù‡ € ÷!ó !÷ð ÷ ÷ ð rn   rJ  c                óF   € V ^8„  d   QhR\         R\        \         ,          /# )rp   Únumberrê   )r�   r�   )r’   s   "rd   r”   r”   Ó  s   € ÷ ñ œ#ð ¤$¤s¥)ñ rn   c                 ó<  € . pV ^,          ^ 8X  d   VP                  ^4       V ^,          p K*  \        ^\        P                  ! V 4      ^,           ^4       F+  pW,          ^ 8X  g   K  VP                  V4       W,          p K*  	  V ^8”  d   VP                  V 4       V# )rp   )ÚappendrW   ÚmathÚisqrt)ri  Úfactorsr	  s   &  rd   Úprime_factorsro  Ó  s~   € Ø€Gà
�1�*˜Œ/Ø�‰�qÔØ�1�Šä�1”d—j’j Ó(¨1Õ,¨aÖ0ˆØ�j˜AŽoØ�N‰N˜1ÔØ�LŠFñ 1ð
 �„zØ�‰�vÔØ€Nrn   c          
      óh   € V ^8„  d   QhR\         R\         R\         R\         R\        P                  /# )rp   r»   rº   r¸   Ú
n_channelsrê   )r�   râ   r  )r’   s   "rd   r”   r”   ä  s:   € ÷ ;ñ ;Üð;Ü*-ð;Ü9<ð;ÜJMð;ä
×Ññ;rn   c           
     ó„  € \         P                  ! \         P                  ! \        V4      RRR1,          VR7      ^ R7      p\         P                  ! \         P                  ! \        V 4      RRR1,          VR7      ^ R7      p\         P                  ! V^,          V,          ^@,          4      P                  4       ^@,          p\         P                  ! VR,          ^,          V,          V,          4      P                  4       ^@,          p\         P                  ! ^^^V..VR7      p	\         P                  ! \         P                  ! V4      WUV.^R7      p
\         P                  ! V\         P                  ! WeR,          4      \         P                  ! WeR,          4      V.^R7      p\         P                  ! WšV.^ R7      p\         P                  ! VRRR13,          ^R7      P                  4       pW,          V ,          V,          p\         P                  ! ^ \         P                  ! \         P                  ! WäR7      4      V^,           VR7      p\         P                  ! VP                  ^4      \         P                  ! V4      P                  ^ 4      ,
          4      pW,P                   ^ ,          8¼  d   \         P"                  ! V^R7      pME^ RIHp V! VP)                  4       P+                  4       4      w  pp\         P                  ! VVR7      p^ V^ &   VP                   ^ ,          ^,
          VR&   VV,          # )aÎ  
Plan out the dimensions for each layer in the HMLP encoder.

This function determines the progression of dimensions (temporal, height, width, channels)
for a multi-layer perceptual model that processes image/video patches. It follows these
principles:
1. Start with small dimensions and increase to full size
2. Expand spatial dimensions (height/width) first, then temporal
3. Increase channel count to avoid information bottlenecks
4. Round channel dimensions to multiples of 64 for hardware efficiency

The function computes optimal assignments of scale configurations to layers using either:
- For n_layers >= len(scales): Individual best matching scales for each layer (allowing duplicates)
- For n_layers < len(scales): Global optimal assignment via linear_sum_assignment

The first and last scales are always fixed to ensure the proper input and output dimensions.

Args:
    temporal_patch_size: Temporal dimension of input patches
    patch_size: Spatial dimension (height/width) of input patches
    n_layers: Number of layers in the encoder
    n_channels: Number of input channels (default: 3 for RGB)

Returns:
    torch.LongTensor of shape `(n_layers + 1, 4)` where the last dim holds values for (t, h, w, c) grids.
Nr>  r†  rï   )Úlinear_sum_assignmentrð   )râ   ÚcumprodÚtensorro  Úceilr�   ÚstackÚ	ones_likeÚ	full_likerŒ  ÚprodrŽ   ÚlinspacerF  Úabsr  rô   ÚargminÚscipy.optimizers  ÚcpuÚnumpy)r»   rº   r¸   rq  r?  ÚhÚtÚh_chÚt_chÚbaseÚspatialÚtemporalÚscalesÚsize_reductionÚtotal_elementsÚlog_ideal_scalesÚcost_matrixÚidxsrs  rµ  Úidxs_nps   &&&&&                rd   Úplan_out_scalesr�  ä  sO  € ô: 	�Š”e—l’l¤=°Ó#<¹T¸r¸TÕ#BÈ6ÔRÐXYÔZ€AÜ�Š”e—l’l¤=Ð1DÓ#EÁdÈÀdÕ#KÐTZÔ[ÐabÔc€Aä�:Š:�a˜•d˜ZÕ'¨"Õ,Ó-×1Ñ1Ó3°bÕ8€DÜ�:Š:�a˜•e˜q•j :Õ-°Õ1Ó2×6Ñ6Ó8¸2Õ=€Dä�<Š<˜!˜Q  :Ð.Ð/¸Ô?€DÜ�kŠkœ5Ÿ?š?¨1Ó-¨q°TÐ:ÀÔB€GÜ�{Š{˜AœuŸš¨q°Bµ%Ó8¼%¿/º/È!ÈrÍUÓ:SÐUYÐZÐ`aÔb€HÜ�YŠY˜ xÐ0°aÔ8€Fä—Z’Z  q¨#¨2¨# v¥°AÔ6×<Ñ<Ó>€NàÕ,Ð/BÕBÀZÕO€NÜ—~’~Ø	Œ5�9Š9”U—\’\ .Ô@ÓAÀ8ÈaÅ<ÐX^ôÐô —)’)Ð,×6Ñ6°qÓ9¼E¿IºIÀnÓ<U×<_Ñ<_Ð`aÓ<bÕbÓc€Kà—<‘< •?Ô"Ü�|Š|˜K¨QÔ/‰å8á*¨;¯?©?Ó+<×+BÑ+BÓ+DÓE‰
ˆˆ7Ü�|Š|˜G¨FÔ3ˆð €Dˆ�GØ�|‰|˜A� Õ"€Dˆ�HØ�$�<Ðrn   c                   óZ   a a€ ] tR tRt oV3R lV 3R llt]V3R lR l4       tRtVtV ;t	# )ÚInklingVisionModeli"  c                ó    <€ V ^8„  d   QhRS[ /# r\  )rµ   )r’   r“   s   "€rd   r”   ÚInklingVisionModel.__annotate__$  s   ø€ ÷ ñ Ñ2ñ rn   c                ó°  <€ \         S	V `  V4       \        VP                  VP                  VP
                  VP                  4      V n        \        P                  ! 4       V n
        \        \        V P                  R R V P                  R,          4      4       EF  w  pw  r4V^ ,          V^ ,          ,          V^,          V^,          ,          ,          V^,          V^,          ,          ,          pW!P
                  ^,
          8X  d   VP                  MV^,          p\        V^,          V^,          ,          4      p\        V^ ,          V^ ,          ,          4      pV P                  P                  \!        V^,          V,          VVVW!P
                  ^,
          8g  R7      4       EK  	  \#        VP                  4      V n        V P'                  4        R # )N:rî   NN)rL  rM  rO  rN  rP  rð   )r^   rß   r�  r»   rº   rX   r¼   rˆ  rà   r  Úencoder_layersr  Úzipr¯   r�   rk  rJ  r×   Ú
final_normr  )
r`   r  rb   Ústart_scaleÚ	end_scaleÚshuffle_multrM  rO  rN  rc   s
   &&       €rd   rß   ÚInklingVisionModel.__init__$  sx  ø€ Ü‰Ñ˜Ô Ü%Ø×&Ñ&Ø×ÑØ×$Ñ$Ø×Ñó	
ˆŒô !Ÿmšm›oˆÔÜ+4´S¸¿¹ÀSÀbÐ9IÈ4Ï;É;ÐWYÍ?Ó5[×+\Ñ'ˆAÑ'�à˜1• ¨Q¥Õ/°I¸aµLÀKÐPQÅNÕ4RÕSÐW`ÐabÕWcÐgrÐstÕguÕWuÕvð ð 56×9QÑ9QÐTUÕ9UÔ4U˜×0Ò0Ð[dÐefÕ[gˆJÜ˜) A�,¨+°a­.Õ8Ó9ˆGÜ˜ 1�¨°Q­Õ7Ó8ˆFØ×Ñ×&Ñ&Ü)Ø)¨!�n¨|Õ;Ø)Ø#Ø!Ø×":Ñ":¸QÕ">Ñ>ô÷ñ ,]ô" )¨×)@Ñ)@ÓAˆŒØ�‰Örn   c                óP   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[/# ©rp   Úpixel_valuesra   rê   )râ   rì   r   r   r   )r’   r“   s   "€rd   r”   r“  D  s-   ø€ ÷ 
ñ 
¡E§L¡Lð 
¹FÑCUÕ<Vð 
Ñ[uñ 
rn   c                óÆ   € VP                   ^ ,          pTpV P                   F  pV! VR7      pK  	  V P                  V4      pVP                  V^R4      p\	        VVR7      # )r   )r6   rE  rð   )rô   r•  r—  rI  r   )r`   rž  ra   Únum_patchesr6   Úlayers   &&,   rd   rû   ÚInklingVisionModel.forwardC  sh   € à"×(Ñ(¨Õ+ˆØ$ˆØ×(Ô(ˆEÙ!°Ô>ŠMñ )ð Ÿ™¨Ó6ˆØ%×-Ñ-¨k¸1¸bÓAˆÜ)Ø+Ø'ô
ð 	
rn   )r•  r—  rˆ  rH  r¦   s   @@rd   r‘  r‘  "  s(   ù‡ € ÷ó ð> ÷
ó ÷
ð 
rn   r‘  zz
    The Base Inkling model which consists of a vision backbone and a language model without language modeling head.,
    ©r¾   c                   óê   a a€ ] tR tRt oRtV3R lV 3R llt]]! RR7      V3R lR l4       4       t]]! R	R7      RV3R
 lR ll4       4       t	V3R lR lt
]]RV3R lR ll4       4       tRtVtV ;t# )ÚInklingModeliR  Fc                ó    <€ V ^8„  d   QhRS[ /# r\  ©rÀ   )r’   r“   s   "€rd   r”   ÚInklingModel.__annotate__[  ó   ø€ ÷ ñ ™}ñ rn   c                ó  <€ \         SV `  V4       VP                  P                  V n        \	        VP                  4      V n        \        VP                  4      V n        \        VP                  4      V n        V P                  4        R # rÞ   )r^   rß   r,   rq   r  Úlanguage_modelr<  rª   Úaudio_towerr‘  r·   Úvision_towerr  rd  s   &&€rd   rß   ÚInklingModel.__init__[  sf   ø€ Ü‰Ñ˜Ô Ø ×,Ñ,×7Ñ7ˆŒÜ.¨v×/AÑ/AÓBˆÔÜ,¨V×-@Ñ-@ÓAˆÔÜ.¨v×/CÑ/CÓDˆÔØ�‰Örn   zOProjects the last hidden state from the vision model into language model space.r£  c                ó`   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[S[,          /# r�  )râ   r  r   r   r;  r   )r’   r“   s   "€rd   r”   r¨  e  s<   ø€ ÷ Fñ FÙ!×-Ñ-ðFÙ9?Ñ@RÕ9SðFá	Ñ+Õ	+ñFrn   c                ó*   € V P                   ! RR V/VB # )rž  rS   )r­  ©r`   rž  ra   s   &&,rd   Úget_image_featuresÚInklingModel.get_image_featuresc  s   € ð
 × Ò ÑE¨lÐE¸fÑEÐErn   zCProjects discretized dMel bin tokens into the language model space.c                ór   <€ V ^8„  d   QhRS[ P                  RS[ P                  R,          RS[S[,          /# )rp   rC  Úaudio_input_ids_maskNrê   )râ   r  rì   r;  r   )r’   r“   s   "€rd   r”   r¨  l  s>   ø€ ÷ 1ñ 1á×)Ñ)ð1ñ $Ÿl™l¨TÕ1ð1ñ 
Ñ+Õ	+ñ	1rn   c                ó    € Ve   WP                  4       ,          pM#VP                  RVP                  R,          4      pV P                  V4      # )a¼  
audio_input_ids (`torch.LongTensor` of shape `(num_audios, max_num_frames, n_mel_bins)`):
    Batch of (padded) dMel bin tokens produced by [`InklingProcessor`].
audio_input_ids_mask (`torch.Tensor` of shape `(num_audios, max_num_frames)`, *optional*):
    Mask marking valid (non-padding) frames. When provided, only valid frames are encoded so that the
    number of returned audio embeddings matches the number of audio placeholder tokens.
rð   )r‘   rI  rô   r¬  )r`   rC  rµ  s   &&&rd   Úget_audio_featuresÚInklingModel.get_audio_featuresj  sJ   € ð  Ò+Ø-×.GÑ.GÓ.IÕJ‰Oà-×5Ñ5°b¸/×:OÑ:OÐPRÕ:SÓTˆOØ×Ñ Ó0Ð0rn   c                ón   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[/# )rp   r3   r4   ÚfeaturesÚtoken_id)râ   r  r  r�   )r’   r“   s   "€rd   r”   r¨  ~  sE   ø€ ÷ ñ á×#Ñ#ðñ ×(Ñ(ðñ ×#Ñ#ð	ñ
 ñrn   c                ó  € Vfs   W P                   P                  V P                  4       ! \        P                  ! V\        P
                  VP                  R7      4      4      8H  pVP                  R4      pMW8H  pVP                  4       pVP                  R4      P                  V4      P                  VP                  4      p\        W%,          P                  4       VP                  4       8H  RV RVP                  ^ ,           24       V# )zÞ
Obtains a multimodal placeholder mask from `input_ids` or `inputs_embeds` for the given `token_id`, and checks
that the placeholder token count matches the length of `features`. If the lengths differ, an error is raised.
)r  r?  zAMultimodal features and placeholder tokens do not match, tokens: z, features: rð   )r«  r  Úget_input_embeddingsrâ   ru  Úlongr?  Úallr£  r  Ú	expand_asr  r   Únumelrô   )r`   r3   r4   rº  r»  Úspecial_maskÚn_tokenss   &&&&&  rd   Úget_placeholder_maskÚ!InklingModel.get_placeholder_mask~  sí   € ð ÒØ(×,?Ñ,?×,JÑ,JØ×)Ñ)Ô+¬E¯LªL¸ÌÏÉÐ\i×\pÑ\pÔ,qÓró-ñ ˆLð (×+Ñ+¨BÓ/‰Là$Ñ0ˆLà×#Ñ#Ó%ˆØ#×-Ñ-¨bÓ1×;Ñ;¸MÓJ×MÑMÈm×NbÑNbÓcˆÜØÕ'×-Ñ-Ó/°8·>±>Ó3CÑCØOÐPXÈzÐYeÐfn×ftÑftÐuvÕfwÐexÐyô	
ð Ðrn   c                óÖ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          R	S[ P                  R,          R
S[ P                  R,          RS[ P                  R,          RS[R,          RS[S[,          RS[S[	,          /# )rp   r3   Nrž  rC  rµ  r7   r  r:  Útoken_type_idsr4   r,  r  Ú	lm_kwargsrê   )
râ   r  r  rì   r   r‘   r   r   r;  rÑ   )r’   r“   s   "€rd   r”   r¨  ›  s  ø€ ÷ j
ñ j
á×#Ñ# dÕ*ðj
ñ ×'Ñ'¨$Õ.ðj
ñ ×)Ñ)¨DÕ0ð	j
ñ
 $Ÿl™l¨TÕ1ðj
ñ Ÿ™ tÕ+ðj
ñ ×&Ñ&¨Õ-ðj
ñ  �ðj
ñ ×(Ñ(¨4Õ/ðj
ñ ×(Ñ(¨4Õ/ðj
ñ × Ñ  4Õ'ðj
ñ ˜$•;ðj
ñ Ñ.Õ/ðj
ñ 
Ñ+Õ	+ñj
rn   c                ó  € VRJ V	RJ,          '       d   \        R4      hV	f   V P                  4       ! V4      p	V P                  P                  V	4      p	Vey   V P	                  V4      P
                  pVP                  V	P                  V	P                  4      pV P                  WWÐP                  P                  4      pV	P                  Wí4      p	RpVez   V P                  W44      P                  pVP                  V	P                  V	P                  4      pV P                  WWðP                  P                  4      pV	P                  VV4      p	\!        T;p\"        4      '       gH   RV P                  P%                  4       RV	RVRVRV/pR\'        R/ VB R	\)        R/ VB R
\+        R/ VB /pV P                  ! RRVRVRVRV	RV/VB p\-        VP                  VP.                  VP0                  VP2                  Ve   XR7      # RR7      # )ag  
audio_input_ids (`torch.LongTensor` of shape `(num_audios, max_num_frames, n_mel_bins)`, *optional*):
    Batch of (padded) discretized dMel bin tokens produced by [`InklingProcessor`].
audio_input_ids_mask (`torch.Tensor` of shape `(num_audios, max_num_frames)`, *optional*):
    Mask marking valid (non-padding) audio frames in `audio_input_ids`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, InklingForConditionalGeneration

>>> model = InklingForConditionalGeneration.from_pretrained("thinkingmachines/Inkling-NVFP4")
>>> processor = AutoProcessor.from_pretrained("thinkingmachines/Inkling-NVFP4")

>>> prompt = "Where is the cat standing?"
>>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read()))

>>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(**inputs,)
>>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Where is the cat standing?\nsnow"
```Nr  r  r4   r7   r:  r  r  r  r  r  )r  r:  r6   ræ  Úimage_hidden_statesrS   )r  r½  r«  r  r²  rF  r  r?  r  rÄ  r  rË   Úmasked_scatterr·  r  rÌ   rÅ   rÆ   rë  r   r   r   rÑ   r:  r6   ræ  )r`   r3   rž  rC  rµ  r7   r  r:  rÇ  r4   r,  r  rÈ  Úimage_featuresÚspecial_image_maskÚaudio_featuresÚspecial_audio_maskr   r!  r4  s   &&&&&&&&&&&&,       rd   rû   ÚInklingModel.forward™  sF  € ðd ˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZàÒ Ø ×5Ñ5Ô7¸	ÓBˆMà×+Ñ+×6Ñ6°}ÓEˆð Ò#Ø!×4Ñ4°\ÓB×PÑPˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ!%×!:Ñ!:Ø¨.¿+¹+×:TÑ:Tó"Ðð *×8Ñ8Ð9KÓ\ˆMð ˆØÒ&Ø!×4Ñ4°_Ó[×mÑmˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ!%×!:Ñ!:Ø¨.¿+¹+×:TÑ:Tó"Ðð *×8Ñ8Ð9KÈ^Ó\ˆMô °Ð?Ð-Ä×FÒFà˜$Ÿ+™+×5Ñ5Ó7Ø Ø  .Ø! ?Ø ðˆKð !Ô"4Ñ"C°{Ñ"CØ#Ô%FÑ%UÈÑ%UØ"Ô$CÑ$RÀkÑ$Rð#Ðð ×%Ò%ñ 
Ø.ð
à%ð
ð ,ð
ð (ð	
ð
  ð
ð ñ
ˆô *Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø2>Ò2J ô
ð 	
ð
 QUô
ð 	
rn   )r¬  r«  r­  rq   rÞ   )NNNNNNNNNNN)r–   r—   r˜   r™   Úaccepts_loss_kwargsrß   r   r   r²  r·  rÄ  rû   r£   r¤   r¥   r¦   s   @@rd   r¥  r¥  R  s˜   ù‡ € ð  Ð÷ó ð ÙÐ!rÔs÷Fó tó ðFð
 ÙÐ!fÔg÷1ñ 1ó hó ð1÷$ð ð6 Ø÷j
ñ j
ó ó ÷j
ð j
rn   r¥  c                   ó¤   a a€ ] tR tRt o/ tRR/tRtV3R lV 3R llt]V3R lR l4       t	]
]RV3R	 lR
 ll4       4       tRV 3R lltRtVtV ;t# )ÚInklingForConditionalGenerationi  r(  r)  Fc                ó    <€ V ^8„  d   QhRS[ /# r\  r§  )r’   r“   s   "€rd   r”   Ú,InklingForConditionalGeneration.__annotate__  r©  rn   c                óò   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  P                  VP                  P                  R R7      V n	        V P                  4        R# rS  )r^   rß   r¥  rå  rà   r)  r,   rs   rq   r(  r  rd  s   &&€rd   rß   Ú(InklingForConditionalGeneration.__init__  sU   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
ô —y’y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÔkˆŒØ�‰Örn   c                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# )rp   rž  ra   )râ   r  r   r   )r’   r“   s   "€rd   r”   rÕ    s+   ø€ ÷ Eñ E©u×/@Ñ/@ð EÉFÑSeÕLfñ Ern   c                ó<   € V P                   P                  ! V3/ VB # rÞ   )rå  r²  r±  s   &&,rd   r²  Ú2InklingForConditionalGeneration.get_image_features  s   € à�z‰z×,Ò,¨\ÑD¸VÑDÐDrn   c                óØ  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          R	S[ P                  R,          R
S[ P                  R,          RS[R,          RS[S[ P                  ,          RS[S[,          RS[	S[
,          /# )rp   r3   Nrž  r7   r  r:  rC  rµ  r4   r,  r  r-  ra   rê   )râ   r  r  rì   r   r‘   r�   r   r   r;  rÕ   )r’   r“   s   "€rd   r”   rÕ  #  s  ø€ ÷ a
ñ a
á×#Ñ# dÕ*ða
ñ ×'Ñ'¨$Õ.ða
ñ Ÿ™ tÕ+ð	a
ñ
 ×&Ñ&¨Õ-ða
ñ  �ða
ñ ×)Ñ)¨DÕ0ða
ñ $Ÿl™l¨TÕ1ða
ñ ×(Ñ(¨4Õ/ða
ñ × Ñ  4Õ'ða
ñ ˜$•;ða
ñ ™eŸl™lÕ*ða
ñ Ñ+Õ,ða
ñ 
Ñ.Õ	.ña
rn   c                óŒ  € V P                   ! RRVRVRVRVRVRVRVRVR	V
R
V	/
VB pV^ ,          V P                  P                  P                  ,          p\	        V\
        4      '       d   \        V) R4      MTpV P                  VRVR3,          4      pV P                  P                  P                  pVe&   VVP                  R,          8  d   VRRV13,          pRpV	e*   V P                  ! RRVR
V	RVP                  R,          /VB p\        VVVP                  VP                  VP                  VP                  R7      # )a<  
audio_input_ids (`torch.LongTensor` of shape `(num_audios, max_num_frames, n_mel_bins)`, *optional*):
    Batch of (padded) discretized dMel bin tokens produced by [`InklingProcessor`].
audio_input_ids_mask (`torch.Tensor` of shape `(num_audios, max_num_frames)`, *optional*):
    Mask marking valid (non-padding) audio frames in `audio_input_ids`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import httpx
>>> from io import BytesIO
>>> from transformers import AutoProcessor, InklingForConditionalGeneration

>>> model = InklingForConditionalGeneration.from_pretrained("thinkingmachines/Inkling-NVFP4")
>>> processor = AutoProcessor.from_pretrained("thinkingmachines/Inkling-NVFP4")

>>> messages = [
...     {
...         "role": "system",
...         "content": [
...             {"type": "text", "text": "You are a helpful assistant."}
...         ]
...     },
...     {
...         "role": "user", "content": [
...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
...             {"type": "text", "text": "Where is the cat standing?"},
...         ]
...     },
... ]

>>> inputs = processor.apply_chat_template(
...     messages,
...     tokenize=True,
...     return_dict=True,
...     return_tensors="pt",
...     add_generation_prompt=True
... )
>>> # Generate
>>> generate_ids = model.generate(**inputs)
>>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
```
r3   rž  rC  rµ  r7   r  r:  r4   r  r,  Nrï   .r0  rq   )r1  r0  r:  r6   ræ  rÊ  rS   rð   )rå  r  r,   r<   rÅ   r�   r2  r(  rr   rô   r3  rÕ   r:  r6   ræ  rÊ  )r`   r3   rž  r7   r  r:  rC  rµ  r4   r,  r  r-  ra   r4  r6   r5  r0  rr   r1  s   &&&&&&&&&&&&,      rd   rû   Ú'InklingForConditionalGeneration.forward!  sˆ  € ðD —*’*ñ 
Øð
à%ð
ð ,ð
ð "6ð	
ð
 *ð
ð &ð
ð ,ð
ð (ð
ð  ð
ð ð
ð ñ
ˆð   �
 T§[¡[×%<Ñ%<×%XÑ%XÕXˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆØ"Ÿk™k×5Ñ5×IÑIÐØÒ*Ð/BÀVÇ\Á\ÐRTÕEUÔ/UØ˜CÐ!5Ð"5Ð!5Ð5Õ6ˆFàˆØÒØ×%Ò%Ñj¨VÐj¸FÐjÈvÏ|É|Ð\^ÕO_ÐjÐciÑjˆDä,ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ô
ð 	
rn   c                ó~   <€ \         SV `  ! V3R VRVRVRVRV	RV
RV/VB pV'       g	   V	'       g   W^R&   W~R&   WŽR	&   V# )
r:  r4   r7   r  r  r-  Úis_first_iterationrž  rC  rµ  )r^   Úprepare_inputs_for_generation)r`   r3   r:  r4   r  rž  r7   rC  rµ  r  r-  r,  rß  ra   Úmodel_inputsrc   s   &&&&&&&&&&&&&, €rd   rà  Ú=InklingForConditionalGeneration.prepare_inputs_for_generation†  sŒ   ø€ ô" ‘wÒ<Øñ

à+ð

ð (ð

ð *ð	

ð
 &ð

ð  ð

ð *ð

ð  2ð

ð ñ

ˆ÷ §YØ+7˜Ñ(Ø.=Ð*Ñ+Ø3GÐ/Ñ0àÐrn   )r(  rå  )NNNNNNNNNNr   )NNNNNNNTr   NF)r–   r—   r˜   r™   r7  r8  rÑ  rß   r   r²  r   rû   rà  r£   r¤   r¥   r¦   s   @@rd   rÓ  rÓ    so   ù‡ € ð ÐØÐ0Ð1€Hð  Ð÷ó ð ÷Eó ðEð Ø÷a
ñ a
ó ó ða
÷F"÷ "rn   rÓ  )rJ   N)r  )rÀ   r*   r¨   rµ   rä  r  r'  r<  r‘  r¥  rÓ  )drl  Úcollections.abcr   râ   Útorch.nnrà   Útorch.nn.functionalr  rˆ  Úhuggingface_hub.dataclassesr   Ú r   rì  Úactivationsr   Úcache_utilsr   r   Úconfiguration_utilsr	   Ú
generationr
   Úintegrationsr   Úintegrations.accelerater   Úmasking_utilsr   r   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úgemma3.modeling_gemma3r   r   r   r    Ú&higgs_audio_v2.modeling_higgs_audio_v2r!   Úllama.modeling_llamar"   r#   Úmixtral.modeling_mixtralr$   Úqwen3_next.modeling_qwen3_nextr%   r&   r'   Ú
get_loggerr–   Úloggerr*   r¨   rµ   rÀ   rÑ   rÕ   r×   r  rÙ   r  r  rZ  ro  rw  rž  r¬  r/  rÔ  rä  r  r'  rñ  r<  rJ  ro  r�  r‘  r¥  rÓ  Ú__all__rS   rn   rd   Ú<module>rþ     s  ðó  Ý $ã Ý ß Ð Ý .å &Ý !ß .Ý 3Ý )Ý /Ý =ß sÑ sÝ 9ß Sß FÝ &÷õ õ 8Ý 5÷ó õ Lß :Ý 5ß qÑ qð 
×	Ò	˜HÓ	%€ñ Ð5Ô6ØôfÐ(ó fó ó 7ðfñR Ð5Ô6Øô$Ð)ó $ó ó 7ð$ñ2 Ð5Ô6Øô$Ð*ó $ó ó 7ð$ñ* Ø)ØXôð ô0(Ð$ó 0(ó ó	ð
0(ôf	Ð!:ô 	ô	Ð$@ô 	ô	�\ô 	ô^˜BŸI™Iô ^÷4%ô8a)�r—y‘yô a)ôH1�ô 1ô=�^ô =ô#H˜Ÿ	™	ô #HôL%˜2Ÿ9™9ô %ô6�—‘ô ñ( Ð*Ð,<Ð=Ó>ô;˜bŸi™ió ;ó ?ð;ô|.Ð4ô .ðb ô.˜_ó .ó ð.ðb ôN
Ð-ó N
ó ðN
ôbA
Ð*ô A
ôH ?Ð"8Ô >ð ô
Ð.ó 
ó ð
ô(" §	¡	ô "õJ÷";ð| ô,
Ð/ó ,
ó ð,
ñ^ ðôô
n
Ð)ó n
óð
n
ñb ðôô
[Ð&<¸oó [óð
[ò|‚rn   