+
    œ°³jƒ+  ã                   ó  € ^ RI Ht ^RIHt ^RIHt ^RIHtHt ]P                  ! ]
4      t]! RR7      ] ! R R]4      4       4       t]! RR7      ] ! R	 R
]4      4       4       t]! RR7      ] ! R R]4      4       4       t. ROtR# )é    )Ústrict)ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringÚloggingz meta-llama/Llama-4-Scout-17B-16E)Ú
checkpointc                   ó¤   a € ] tR t^t o RtRRRRRRRRRRR	RR
R/tRtRtRtRt	^"t
^t^tRtRtRt^tRtRtRtRtRtRtRtRtRtRtV 3R ltRtV tR# )ÚLlama4VisionConfigaG  
vision_output_dim (`int`, *optional*, defaults to 7680):
    Dimensionality of the vision model output. Includes output of transformer
    encoder with intermediate layers and global transformer encoder.
pixel_shuffle_ratio (`float`, *optional*, defaults to 0.5):
    Pixel-shuffle ratio for downsampling patch tokens. Smaller values produce fewer tokens (more downsampling).
projector_input_dim (`int`, *optional*, defaults to 4096):
    Width of the vision adapter MLP before pixel shuffle. Larger value increases capacity and compute.
projector_output_dim (`int`, *optional*, defaults to 4096):
    Output width of the vision adapter. Larger value yields higher-dimensional image features.
projector_dropout (`float`, *optional*, defaults to 0.0):
    Dropout rate inside the vision adapter MLP. Higher value adds more regularization.
zmodel.layers.*.self_attn.q_projÚcolwisezmodel.layers.*.self_attn.k_projzmodel.layers.*.self_attn.v_projzmodel.layers.*.self_attn.o_projÚrowwisezvision_adapter.mlp.fc1zvision_adapter.mlp.fc2zpatch_embedding.linearÚcolwise_gather_outputÚllama4_vision_modelÚvision_configi   Úgelui   i   iÀ  çñhãˆµøä>Údefaultç{®Gáz”?g      à?i   Fç        Nc                óÈ  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ S[S[ ,          ,          S[S[ S[ 3,          ,          ;R&   S[ S[S[ ,          ,          S[S[ S[ 3,          ,          ;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[ ;R&   S[ ;R&   S[;R&   S[S[ ,          ;R&   S[S[ ,          ;R&   S[S[,          R,          ;R&   # )é   Úhidden_sizeÚ
hidden_actÚnum_hidden_layersÚnum_attention_headsÚnum_channelsÚintermediate_sizeÚvision_output_dimÚ
image_sizeÚ
patch_sizeÚnorm_epsÚvision_feature_select_strategyÚinitializer_rangeÚpixel_shuffle_ratioÚprojector_input_dimÚprojector_output_dimÚmulti_modal_projector_biasÚprojector_dropoutÚattention_dropoutNÚrope_parameters)ÚintÚstrÚlistÚtupleÚfloatÚboolr   Údict)ÚformatÚ__classdict__s   "€Úp/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/llama4/configuration_llama4.pyÚ__annotate__ÚLlama4VisionConfig.__annotate__   sX  ø‡ ‚ ñ: Ññ; ñ< Ññ= ñ> Ññ? ñ@ Ñ!ñA ñB ÑñC ñD Ñ!ñE ñF Ñ!ñG ñH ‘d™3•i•¡%©©S¨¥/Õ1Ñ7ñI ñJ ‘d™3•i•¡%©©S¨¥/Õ1Ñ6ñK ñL ÑñM ñN %(Ñ3ñO ñP Ñ#ñQ ñR Ñ$ñS ñT Ñ#ñU ñV Ñ$ñW ñX !%Ñ,ñY ñZ ™s•{Ñ(ñ[ ñ\ ™s•{Ñ(ñ] ñ^ $¡dÕ*¨TÕ1Ñ8ò_ ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Úbase_model_tp_planÚ
model_typeÚbase_config_keyr   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r2   s   @r3   r
   r
      s·   ø‡ € ñð 	*¨9Ø)¨9Ø)¨9Ø)¨9Ø  )Ø  )Ø Ð"9ðÐð '€JØ%€Oà€KØ€JØÐØ!ÐØ€LØ!ÐØ!ÐØ47€JØ46€JØ€HØ*3Ð"Ø#ÐØ!$ÐØ#ÐØ $ÐØ',ÐØ%(ÐØ%(ÐØ48€O÷_ ƒ r6   r
   c                   ó@  a a€ ] tR t^Mt oRtRtR.tRtRRRRRRR	R
RRRRRR
RRRRRRRRRRRR
/tRRRRRRR	R
RRRRRRRRRR
RR/
t	Rt
RtRtRt^0t^(t^t^€tRtR)tRtRtR tR!t^t^tR"tR#t^t^tR!t^tR t R"t!R$t"R#t#R!t$R!t%^t&Rt'R!t(R t)Rt*R%t+R"t,V 3R& lt-V3R' lt.R(t/Vt0V ;t1# )*ÚLlama4TextConfigaç  
intermediate_size_mlp (`int`, *optional*, defaults to 16384):
    Intermediate size of dense MLP layers. Larger value increases FFN capacity and compute.
moe_layers (`list[int]`, *optional*):
    List of layer indices that use MoE. Overrides `interleave_moe_layer_step` when set.
interleave_moe_layer_step (`int`, *optional*, defaults to 1):
    Spacing between MoE layers when `moe_layers` is `None`. Larger value means fewer MoE layers.
use_qk_norm (`bool`, *optional*, defaults to `True`):
    Whether to L2-normalize queries/keys on RoPE layers. Can stabilize attention when enabled.
no_rope_layers (`list[int]`, *optional*):
    List with at least the same length as the number of layers in the model.
    A `1` at an index position indicates that the corresponding layer will use RoPE,
    while a `0` indicates that it's a NoPE layer.
no_rope_layer_interval (`int`, *optional*, defaults to 4):
    If `no_rope_layers` is `None`, it will be created using a NoPE layer every
    `no_rope_layer_interval` layers.
attention_chunk_size (`int`, *optional*, defaults to 8192):
    Chunk size for the attention computation. Smaller value enforces more local attention and lowers memory.
attn_temperature_tuning (`bool`, *optional*, defaults to `True`):
    Whether to dynamically scale the attention temperature for each query token based on sequence length.
    Recommended for long sequences (e.g., >32k tokens) to maintain stable output results.
floor_scale (`int`, *optional*, defaults to 8192):
    Base scale (in tokens) for attention temperature tuning. Larger value delays scaling to longer positions.
attn_scale (`float`, *optional*, defaults to 0.1):
    Strength of attention temperature tuning. Larger value increases scaling at long positions.

Example:
Úllama4_textÚpast_key_valuesg    €„Azlayers.*.self_attn.q_projr   zlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projr   z-layers.*.feed_forward.shared_expert.gate_projz+layers.*.feed_forward.shared_expert.up_projz-layers.*.feed_forward.shared_expert.down_projz*layers.*.feed_forward.experts.gate_up_projÚpacked_rowwisez'layers.*.feed_forward.experts.down_projzlayers.*.feed_forward.expertsÚmoe_tp_expertszlayers.*.feed_forward.gate_projzlayers.*.feed_forward.up_projzlayers.*.feed_forward.down_projÚgrouped_gemmzlayers.*.feed_forward.routerÚ	ep_routeri@ i   i    i @  Úsilur   r   TNFr   gü©ñÒMbP?gš™™™™™¹?c                óð  <€ V P                   f   V P                  V n         \        V P                  4       Uu. uF)  p\	        V^,           V P
                  ,          ^ 8g  4      NK+  	  ppV P                  '       d   V P                  MTV n        V P                  e   V P                  MV P                  V P                  ,          V n        V P                  e   V P                  M:\        \        V P                  ^,
          V P                  V P                  4      4      V n	        V P                  f+   V P                   Uu. uF  qD'       d   RMRNK  	  upV n        \        SV `8  ! R/ VB  R # u upi u upi )NÚchunked_attentionÚfull_attentionr7   )Únum_key_value_headsr   Úranger   r*   Úno_rope_layer_intervalÚno_rope_layersÚhead_dimr   Ú
moe_layersr,   Úinterleave_moe_layer_stepÚlayer_typesÚsuperÚ__post_init__)ÚselfÚkwargsÚ	layer_idxÚdefault_no_rope_layersÚno_ropeÚ	__class__s   &,   €r3   rX   ÚLlama4TextConfig.__post_init__°   sK  ø€ Ø×#Ñ#Ò+Ø'+×'?Ñ'?ˆDÔ$ô V[Ð[_×[qÑ[qÔUró"
ÙUrÈ	ŒC�˜Q• $×"=Ñ"=Õ=ÀÑBÖCÑUrð 	ð "
ð 6:×5H×5HÐ5H˜d×1Ò1ÐNdˆÔØ)-¯©Ò)B˜ŸšÈ×HXÑHXÐ\`×\tÑ\tÕHtˆŒð �‰Ò*ð �OŠOäÜØ×2Ñ2°QÕ6Ø×*Ñ*Ø×2Ñ2óóð 	Œð ×ÑÒ#àTX×TgÒTgó ÙTgÈ¦wÑ#Ð4DÒDÑTgñ ˆDÔô 	‰ÒÑ' Ô'ùò/"
ùò& s   ¸/E.Å E3c                óž  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[ ;R&   S[;R	&   S[ ;R
&   S[;R&   S[;R&   S[;R&   S[ R,          ;R&   S[ R,          ;R&   S[ S[S[ ,          ,          R,          ;R&   S[;R&   S[S[ ,          ;R&   S[ ;R&   S[ ;R&   S[S[ ,          R,          ;R&   S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          R,          ;R&   S[S[ ,          R,          ;R&   S[ ;R&   S[ R,          ;R&   S[S[,          R,          ;R &   S[;R!&   S[ ;R"&   S[;R#&   S[;R$&   # )%r   Ú
vocab_sizer   r   Úintermediate_size_mlpr   r   rO   rS   r   Úmax_position_embeddingsr"   Úrms_norm_epsÚ	use_cacheNÚpad_token_idÚbos_token_idÚeos_token_idÚtie_word_embeddingsr(   Únum_experts_per_tokÚnum_local_expertsrT   rU   Úuse_qk_normÚoutput_router_logitsÚrouter_aux_loss_coefÚrouter_jitter_noiser)   rR   rQ   Úattention_chunk_sizerV   Úattn_temperature_tuningÚfloor_scaleÚ
attn_scaleÚattention_bias)r*   r+   r.   r/   r,   r   r0   )r1   r2   s   "€r3   r4   ÚLlama4TextConfig.__annotate__M   sT  ø‡ ‚ ñ~ Ññ ñ@ ÑñA ñB Ñ!ñC ñD Ñ&ñE ñF ÑñG ñH Ñ!ñI ñJ Ñ ñK ñL ÑñM ñN ÑñO ñP !Ñ,ñQ ñR Ñ#ñS ñT ÑñU ñV ÑñW ñX ˜•*Ñ#ñY ñZ ˜•*Ñ ñ[ ñ\ ™™S�	•/ DÕ(Ñ,ñ] ñ^ Ñ%ñ_ ñ` ™s•{Ñ(ña ñb Ñ ñc ñd Ññe ñf ‘S•	˜DÕ Ñ'ñg ñh  #Ñ&ñi ñj Ññk ñl Ñ&ñm ñn  Ñ'ño ñp Ñ$ñq ñr $¡dÕ*¨TÕ1Ñ8ñs ñt ™•I Õ$Ñ+ñu ñv  Ñ#ñw ñx  �*Ñ+ñy ñz ‘c•˜TÕ!Ñ(ñ{ ñ| "Ñ(ñ} ñ~ Ññ ñ@ ÑñA ñB Ñ òC r6   )rS   rV   rT   rR   rO   i   )2r8   r9   r:   r;   r<   r>   Úkeys_to_ignore_at_inferenceÚdefault_thetar=   Úbase_model_ep_planra   r   r   rb   r   r   rO   rS   r   rc   r"   rd   re   rf   rg   rh   ri   r(   rj   rk   rT   rU   rl   rm   rn   ro   r)   rR   rQ   rp   rV   rq   rr   rs   rt   rX   r@   rA   rB   Ú__classcell__©r^   r2   s   @@r3   rD   rD   M   sv  ù‡ € ñð: €JØ#4Ð"5ÐØ€Mà# YØ# YØ# YØ# YØ7¸Ø5°yØ7¸Ø4Ð6FØ1°9Ø'Ð)9Ø)¨9Ø'¨Ø)¨9ðÐð  	$ YØ# YØ# YØ# YØ4°nØ1°>Ø)¨9Ø'¨Ø)¨9Ø&¨ðÐð €JØ€KØ!ÐØ!&ÐØÐØ!ÐØ ÐØ€HØ€JØ#,ÐØ#ÐØ€LØ€IØ#€LØ €LØ+,€LØ %ÐØ%(ÐØ ÐØÐØ#'€JØ%&ÐØ€KØ!&ÐØ"'ÐØ!$ÐØ48€OØ'+€NØ"#ÐØ'+ÐØ$(€KØ$(ÐØ€KØ€JØ €Nõ(÷G … r6   rD   c                   ó‚   a a€ ] tR t^Ît oRtRtRRRRRR/tR	]R
]/t	RR/t
RtRtRtRtRtRtV 3R ltV3R ltRtVtV ;t# )ÚLlama4Configa<  
boi_token_index (`int`, *optional*, defaults to 200080):
    The begin-of-image token index to wrap the image prompt.
eoi_token_index (`int`, *optional*, defaults to 200081):
    The end-of-image token index to wrap the image prompt.

```python
>>> from transformers import Llama4Model, Llama4Config

>>> # Initializing a Llama4 7B style configuration
>>> configuration = Llama4Config()

>>> # Initializing a model from the Llama4 7B style configuration
>>> model = Llama4Model(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```
Úllama4Úimage_token_idÚimage_token_indexÚboi_token_idÚboi_token_indexÚeoi_token_idÚeoi_token_indexÚtext_configr   zmulti_modal_projector.linear_1r   Ni� i‘ iœ Fc                óÜ  <€ V P                   f&   \        4       V n         \        P                  R4       M:\	        V P                   \
        4      '       d   \        R/ V P                   B V n         V P                  f&   \        4       V n        \        P                  R4       M:\	        V P                  \
        4      '       d   \        R/ V P                  B V n        \        SV `$  ! R/ VB  R # )Nz9vision_config is None, using default llama4 vision configz5text_config is None, using default llama4 text configr7   )
r   r
   ÚloggerÚinfoÚ
isinstancer0   r„   rD   rW   rX   )rY   rZ   r^   s   &,€r3   rX   ÚLlama4Config.__post_init__÷   s­   ø€ Ø×ÑÒ%Ü!3Ó!5ˆDÔÜ�K‰KÐSÕTÜ˜×*Ñ*¬D×1Ò1Ü!3Ñ!I°d×6HÑ6HÑ!IˆDÔà×ÑÒ#Ü/Ó1ˆDÔÜ�K‰KÐOÕPÜ˜×(Ñ(¬$×/Ò/Ü/ÑC°$×2BÑ2BÑCˆDÔÜ‰ÒÑ' Ô'r6   c                óž   <€ V ^8„  d   Qh/ S[ S[,          R,          ;R&   S[ S[,          R,          ;R&   S[;R&   S[;R&   S[;R&   S[;R&   # )r   Nr   r„   r�   rƒ   r   ri   )r0   r   r*   r/   )r1   r2   s   "€r3   r4   ÚLlama4Config.__annotate__Î   su   ø‡ ‚ ñD Ñ*Õ*¨TÕ1Ñ8ñE ñF Ñ(Õ(¨4Õ/Ñ6ñG ñH Ñ!ñI ñJ Ñ!ñK ñL Ñ#ñM ñN Ñ%òO r6   )r„   r   )r8   r9   r:   r;   r<   r>   Úattribute_maprD   r
   Úsub_configsr=   r   r„   r�   rƒ   r   ri   rX   r@   rA   rB   ry   rz   s   @@r3   r|   r|   Î   sy   ù‡ € ñð( €JàÐ-ØÐ)ØÐ)ð€Mð
 !Ð"2°OÐEWÐX€Kà(Ð*AðÐð 59€MØ26€KØ!€OØ!€OØ#ÐØ %Ðõ(÷S … r6   r|   N)r|   rD   r
   )Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r   Ú
get_loggerr8   r†   r
   rD   r|   Ú__all__r7   r6   r3   Ú<module>r”      s±   ðõ" /å 3Ý 1ß ,ð 
×	Ò	˜HÓ	%€ñ Ð=Ô>Øô-9Ð)ó -9ó ó ?ð-9ñ` Ð=Ô>Øô|(Ð'ó |(ó ó ?ð|(ñ~ Ð=Ô>Øô3(Ð#ó 3(ó ó ?ð3(òl E‚r6   