+
    œ°³j?¢  ã                   óÒ  € R t ^ RIt^ RIt^ RIHt ^ RIHtHtHtHt ^ RIH	t
 ^RIHtHtHt ^RIHt ^RIHt ^RIHt ^R	IHtHtHtHtHt ^R
IHt ^RIHtHt ^RI H!t! ]PD                  ! ]#4      t$R R lt%R R lt&R R lt'R R lt( ! R R]PR                  PT                  4      t+ ! R R]PX                  4      t- ! R R]PX                  4      t. ! R R]PX                  4      t/ ! R R]4      t0] ! R R ]4      4       t1] ! R! R"]14      4       t2]! R#R$7       ! R% R&]1]4      4       t3]! R'R$7       ! R( R)]14      4       t4] ! R* R+]14      4       t5] ! R, R-]14      4       t6. R.Ot7R# )/zPyTorch BLOOM model.N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚ	LayerNormÚMSELoss)Ú
functional)ÚCacheÚDynamicCacheÚStaticCache)ÚGenerationMixin)Úcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚQuestionAnsweringModelOutputÚ SequenceClassifierOutputWithPastÚTokenClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚlogging©ÚBloomConfigc                ó„   € V ^8„  d   QhR\         P                  R\        R\         P                  R\         P                  /# ©é   Úattention_maskÚ	num_headsÚdtypeÚreturn©ÚtorchÚTensorÚintr   )Úformats   "Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/models/bloom/modeling_bloom.pyÚ__annotate__r%   -   s=   € ÷ ,Jñ ,J¤u§|¡|ð ,JÄð ,JÌEÏKÉKð ,JÔ\a×\hÑ\hñ ,Jó    c           	     ól  € V P                   w  r4^\        P                  ! \        P                  ! V4      4      ,          p\        P
                  ! R^^\        P                  ! V4      ^,
          ) ,          ) ,          V P                  \        P                  R7      p\        P                  ! ^^V,           V P                  \        P                  R7      p\        P                  ! Wg4      pWQ8w  dæ   \        P
                  ! R^^\        P                  ! ^V,          4      ^,
          ) ,          ) ,          V P                  \        P                  R7      p	\        WQV,
          4      p
\        P                  ! ^^^V
,          ,           ^V P                  \        P                  R7      p\        P                  ! V\        P                  ! W›4      .^ R7      pV P                  RR7      ^,
          V ,          R,          pVR,          V,          pVP                  W1,          ^V4      P                  V4      # )aV  
Link to paper: https://huggingface.co/papers/2108.12409 Alibi tensor is not causal as the original paper mentions, it
relies on a translation invariance of softmax for quick implementation: with l being a tensor, and a fixed value
`softmax(l+a) = softmax(l)`. Based on
https://github.com/ofirpress/attention_with_linear_biases/blob/a35aaca144e0eb6b789dfcb46784c4b8e31b7983/fairseq/models/transformer.py#L742
TODO @thomasw21 this doesn't work as nicely due to the masking strategy, and so masking varies slightly.

Args:
Returns tensor shaped (batch_size * num_heads, 1, max_seq_len)
    attention_mask (`torch.Tensor`):
        Token-wise attention mask, this should be of shape (batch_size, max_seq_len).
    num_heads (`int`):
        number of heads
    dtype (`torch.dtype`, *optional*, default=`torch.bfloat16`):
        dtype of the output tensor
©Údevicer   ©Údim© éÿÿÿÿ)ºNNNNr.   ).N)ÚshapeÚmathÚfloorÚlog2r    Úfullr)   Úfloat32ÚarangeÚint32ÚpowÚminÚcatÚcumsumÚreshapeÚto)r   r   r   Ú
batch_sizeÚ
seq_lengthÚclosest_power_of_2ÚbaseÚpowersÚslopesÚ
extra_baseÚnum_remaining_headsÚextra_powersÚarange_tensorÚalibis   &&&           r$   Úbuild_alibi_tensorrH   -   s«  € ð" ,×1Ñ1Ñ€JØœdŸjšj¬¯ª°9Ó)=Ó>Õ>ÐÜ�:Š:Ø
ˆA�Aœ4Ÿ9š9Ð%7Ó8¸1Õ<Ð=Õ=Ð>Õ?È×H]ÑH]Ôej×erÑerô€Dô �\Š\˜!˜QÐ!3Õ3¸N×<QÑ<QÔY^×YdÑYdÔe€FÜ�YŠY�tÓ$€FàÔ&Ü—Z’ZØØ�Aœ4Ÿ9š9 QÐ);Õ%;Ó<¸qÕ@ÐAÕAÐBÕCØ!×(Ñ(Ü—-‘-ô	
ˆ
ô "Ð"4ÐBTÕ6TÓUÐÜ—|’| A q¨1Ð/BÕ+BÕ'BÀAÈn×NcÑNcÔkp×kvÑkvÔwˆÜ—’˜F¤E§I¢I¨jÓ$GÐHÈaÔPˆð %×+Ñ+°Ð+Ó3°aÕ7¸>ÕIÈ:ÕV€MØ�9Õ Õ-€EØ�=‰=˜Õ/°°JÓ?×BÑBÀ5ÓIÐIr&   c          
      ó�   € V ^8„  d   QhR\         P                  R\         P                  R\        R\        R\         P                  /# )r   ÚxÚresidualÚprobÚtrainingr   )r    r!   ÚfloatÚbool)r#   s   "r$   r%   r%   \   s?   € ÷ ñ ”5—<‘<ð ¬5¯<©<ð ¼uð ÔPTð ÔY^×YeÑYeñ r&   c                óF   € \         P                  ! WVR7      pW,           pV# )zÞ
Dropout add function

Args:
    x (`torch.tensor`):
        input tensor
    residual (`torch.tensor`):
        residual tensor
    prob (`float`):
        dropout probability
    training (`bool`):
        training mode
)ÚprM   )ÚFÚdropout)rJ   rK   rL   rM   Úouts   &&&& r$   Údropout_addrU   \   s    € ô �)Š)�A¨Ô
1€CØ
�.€CØ€Jr&   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# ©r   rJ   r   ©r    r!   )r#   s   "r$   r%   r%   o   s&   € ÷ 	Qñ 	Qœ%Ÿ,™,ð 	Q¬5¯<©<ñ 	Qr&   c                óž   € V R,          R\         P                  ! RV ,          ^RV ,          V ,          ,           ,          4      ,           ,          # )zÈ
Custom bias GELU function. Adapted from Megatron-DeepSpeed code. Here we use a simple implementation (inference) to
make the model jitable.

Args:
    x (`torch.tensor`):
        input hidden states
ç      à?ç      ð?ç Þe3Eˆé?ç÷Hmâä¦?©r    Útanh)rJ   s   &r$   Úbloom_gelu_forwardr`   o   s8   € ð ˆs�7�cœEŸJšJ z°A¥~¸¸XÈ½\ÈAÕ=MÕ9MÕ'NÓOÕOÕPÐPr&   c                óx   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  /# )r   ÚgrJ   r   rX   )r#   s   "r$   r%   r%   {   s-   € ÷ ñ ”u—|‘|ð ¬¯©ð ¼¿¹ñ r&   c                ó2  € V^ ,          p\         P                  ! RV,          ^RV,          V,          ,           ,          4      pRV,          ^W",          ,
          RRV,          V,          ,           ,          ,          R^V,           ,          ,           pW0,          # )a   
gradient of tanh approximation of gelu gradient of actual gelu is: 0.5 * (1. + torch.erf(x * 0.70710678)) +
0.3989423 * x * torch.exp(-0.5 * x * x)

Args:
    g (`torch.tensor`):
        gradient output tensor
    x (`torch.tensor`):
        input tensor
r\   r]   rZ   g6”ü¾vf»?r^   )rb   rJ   Útanh_outÚffs   &&  r$   Úbloom_gelu_backrf   {   sv   € ð 	
ˆ!�€AÜ�zŠz˜* q�.¨A°¸1µ¸qÕ0@Õ,@ÕAÓB€Hà	ˆq��Q˜Õ,Õ,°¸lÈQÕ>NÐQRÕ>RÕ1RÕSÕ	TÐWZÐ^_ÐbjÕ^jÕWkÕ	k€BØ�6€Mr&   c                   óX   a € ] tR t^�t o ]V 3R lR l4       t]V 3R lR l4       tRtV tR# )ÚGeLUFunctionc                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Úinputr   rX   )r#   Ú__classdict__s   "€r$   r%   ÚGeLUFunction.__annotate__�   s#   ø€ ÷ )ñ )™EŸL™Lð )©U¯\©\ñ )r&   c                ó:   € V P                  V4       \        V4      # ©N)Úsave_for_backwardr`   )Úctxrj   s   &&r$   ÚforwardÚGeLUFunction.forwardŽ   s   € à×Ñ˜eÔ$Ü! %Ó(Ð(r&   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Úgrad_outputr   rX   )r#   rk   s   "€r$   r%   rl   ”   s#   ø€ ÷ ñ ¡5§<¡<ð ±E·L±Lñ r&   c                ó4   € V P                   p\        W4      pV# rn   )Úsaved_tensorsrf   )rp   rt   rj   Útmps   &&  r$   ÚbackwardÚGeLUFunction.backward“   s   € à×!Ñ!ˆÜ˜kÓ1ˆØˆ
r&   r,   N)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ústaticmethodrq   rx   Ú__static_attributes__Ú__classdictcell__©rk   s   @r$   rh   rh   �   s+   ø‡ € Ø÷)ó ð)ð ÷ó ör&   rh   c                   óH   a a€ ] tR t^št oRtV 3R ltV3R lR ltRtVtV ;t	# )Ú	BloomGeluzF
Partly copied from Megatron-DeepSpeed code and adapted for our needs
c                ó$   <€ \         SV `  4        R # rn   )ÚsuperÚ__init__)ÚselfÚ	__class__s   &€r$   r†   ÚBloomGelu.__init__Ÿ   s   ø€ Ü‰ÑÖr&   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rW   rX   )r#   rk   s   "€r$   r%   ÚBloomGelu.__annotate__¢   s#   ø€ ÷ %ñ %™Ÿ™ð %©%¯,©,ñ %r&   c                ó,   € \         P                  V4      # rn   )rh   Úapply)r‡   rJ   s   &&r$   rq   ÚBloomGelu.forward¢   s   € Ü×!Ñ! !Ó$Ð$r&   r,   )
rz   r{   r|   r}   Ú__doc__r†   rq   r   r€   Ú__classcell__©rˆ   rk   s   @@r$   rƒ   rƒ   š   s   ù‡ € ñõ÷%÷ %ð %r&   rƒ   c                   ó|   a a€ ] tR t^¦t oR
V3R lV 3R llltV3R lR ltV3R lR ltRV3R lR lltR	tVt	V ;t
# )ÚBloomAttentionc                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©r   ÚconfigÚ	layer_idxN©r   r"   )r#   rk   s   "€r$   r%   ÚBloomAttention.__annotate__§   s#   ø€ ÷ Fñ F™{ð F±s¸Tµzñ Fr&   c                óÈ  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP
                  V n        V P                  V P                  ,          V n        V P                  V n        VP                  V n	        V P                  V P                  ,          V P                  8w  d'   \        R V P                   RV P                   R24      hR\        P                  ! V P                  4      ,          V n        RV n        W n        Vf.   \         P#                  RV P$                  P&                   R24       \(        P*                  ! V P                  ^V P                  ,          RR7      V n        \(        P*                  ! V P                  V P                  4      V n        \(        P0                  ! VP2                  4      V n        R# )	zA`hidden_size` must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).r[   NzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.T©Úbias)r…   r†   Úpretraining_tpÚslow_but_exactÚhidden_sizeÚn_headr   Úhead_dimÚ
split_sizeÚhidden_dropoutÚ
ValueErrorr0   ÚsqrtÚinv_norm_factorÚbetar—   ÚloggerÚwarning_oncerˆ   rz   r   ÚLinearÚquery_key_valueÚdenseÚDropoutÚattention_dropout)r‡   r–   r—   rˆ   s   &&&€r$   r†   ÚBloomAttention.__init__§   sx  ø€ Ü‰ÑÔà$×3Ñ3ˆÔØ$×3Ñ3ˆÔà!×-Ñ-ˆÔØŸ™ˆŒØ×(Ñ(¨D¯N©NÕ:ˆŒØ×*Ñ*ˆŒØ$×3Ñ3ˆÔà�=‰=˜4Ÿ>™>Õ)¨T×-=Ñ-=Ô=ÜØSÐTX×TdÑTdÐSeð fØ—N‘NÐ# 2ð'óð ð  #¤T§Y¢Y¨t¯}©}Ó%=Õ=ˆÔØˆŒ	Ø"ŒØÒÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôô  "Ÿyšy¨×)9Ñ)9¸1¸t×?OÑ?OÕ;OÐVZÔ[ˆÔÜ—Y’Y˜t×/Ñ/°×1AÑ1AÓBˆŒ
Ü!#§¢¨F×,DÑ,DÓ!EˆÖr&   c                ó�   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  S[ P                  S[ P                  3,          /# )r   Ú	fused_qkvr   )r    r!   Útuple)r#   rk   s   "€r$   r%   r™   È   s:   ø€ ÷ 3ñ 3¡%§,¡,ð 3±5¹¿¹ÁuÇ|Á|ÑUZ×UaÑUaÐ9aÕ3bñ 3r&   c                ó  € VP                   w  r#pVP                  W#V P                  ^V P                  4      pVR,          P	                  ^^4      pVR,          P	                  ^^4      pVR,          P	                  ^^4      pWVV3# )aº  
Split the last dimension into (num_heads, head_dim) and reshapes to (bs, heads, len, dim) shape
without making any copies, results share same memory storage as `fused_qkv`

Args:
    fused_qkv (`torch.tensor`): [batch_size, seq_length, num_heads * 3 * head_dim]

Returns:
    query: [batch_size, num_heads, seq_length, head_dim]
    key: [batch_size, num_heads, seq_length, head_dim]
    value: [batch_size, num_heads, seq_length, head_dim]
).é    r.   ).é   r.   ).r   r.   )r/   Úviewr   r¡   Ú	transpose)r‡   r±   r=   r>   Úthree_times_hidden_sizeÚquery_layerÚ	key_layerÚvalue_layers   &&      r$   Ú_reshapeÚBloomAttention._reshapeÈ   s€   € ð ;D¿/¹/Ñ7ˆ
Ð 7Ø—N‘N :¸4¿>¹>È1ÈdÏmÉmÓ\ˆ	Ø 	Õ*×4Ñ4°Q¸Ó:ˆØ˜iÕ(×2Ñ2°1°aÓ8ˆ	Ø 	Õ*×4Ñ4°Q¸Ó:ˆØ {Ð2Ð2r&   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rW   rX   )r#   rk   s   "€r$   r%   r™   Ü   s'   ø€ ÷ Qñ Q™eŸl™lð Q©u¯|©|ñ Qr&   c                ó  € VP                   w  r#pW P                  ,          pVP                  WPP                  W0P                  4      pVP	                  ^ ^^^4      pVP                  WSV P                  V P                  ,          4      # )zÇ
Merge heads together over the last dimension

Args:
    x (`torch.tensor`): [batch_size * num_heads, seq_length, head_dim]

Returns:
    torch.tensor: [batch_size, seq_length, num_heads * head_dim]
)r/   r   r¶   r¡   Úpermuter;   )r‡   rJ   Úbatch_size_and_num_headsr>   Ú_r=   s   &&    r$   Ú_merge_headsÚBloomAttention._merge_headsÜ   so   € ð 34·'±'Ñ/Ð ¨aØ-·±Õ?ˆ
ð �F‰F�:Ÿ~™~¨z¿=¹=ÓIˆð �I‰I�a˜˜A˜qÓ!ˆð �y‰y˜°·±À$Ç-Á-Õ1OÓPÐPr&   c                ó¢   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[R,          RS[RS[/# )	r   Úhidden_statesrK   rG   r   Ú
layer_pastNÚ	use_cacheÚoutput_attentions©r    r!   r   rO   )r#   rk   s   "€r$   r%   r™   õ   sn   ø€ ÷ A.ñ A.á—|‘|ðA.ñ —,‘,ðA.ñ �|‰|ð	A.ñ
 Ÿ™ðA.ñ ˜D•LðA.ñ ðA.ñ  ñA.r&   c                ó:  € VP                   w  ršpV P                  V4      pV P                  V4      w  rÞpVe   VP                  WïV P                  4      w  rïVP                  W�P                  ,          RV P                  4      pVP                  W�P                  ,          RV P                  4      P                  RR4      pVP                  W�P                  ,          RV P                  4      pVP                  VVV P                  V P                  R7      pVP                  W�P                  V
R4      pVe
   VV,           p\        P                  ! VR\        P                   R7      P#                  VP$                  4      pV P'                  V4      pVP                  W�P                  ,          V
R4      p\        P(                  ! VV4      pV P+                  V4      pV P,                  ^8”  dú   V P.                  '       dè   V P0                  V P,                  ,          p\        P2                  ! V4      p\5        V P,                  4       Fš  pV\        P6                  ! VRR\9        VV,          4      \9        V^,           V,          4      13,          V P:                  P<                  R\9        VV,          4      \9        V^,           V,          4      13,          4      ,           pKœ  	  MV P;                  V4      p\?        VW P@                  V PB                  4      pVV3# )N)Úbatch1Úbatch2r§   Úalpha)r+   r   r.   r-   éþÿÿÿ)"r/   r«   r¼   Úupdater—   r;   r   r¡   r·   Úbaddbmmr§   r¦   r¶   rR   Úsoftmaxr    r4   r<   r   r®   ÚbmmrÃ   r�   rž   rŸ   Ú
zeros_likeÚrangeÚlinearr"   r¬   ÚweightrU   r£   rM   )r‡   rÆ   rK   rG   r   rÇ   rÈ   rÉ   Úkwargsr=   Úq_lengthrÂ   r±   r¹   rº   r»   Úattention_scoresÚattn_weightsÚattention_probsÚattention_probs_reshapedÚcontext_layerÚslicesÚoutput_tensorÚis   &&&&&&&&,               r$   rq   ÚBloomAttention.forwardõ   s™  € ð #0×"5Ñ"5Ñˆ
˜aØ×(Ñ(¨Ó7ˆ	à.2¯m©m¸IÓ.FÑ+ˆ àÒ!Ø%/×%6Ñ%6°yÈtÏ~É~Ó%^Ñ"ˆIð "×)Ñ)¨*·~±~Õ*EÀrÈ4Ï=É=ÓYˆØ×%Ñ% j·>±>Õ&AÀ2ÀtÇ}Á}ÓU×_Ñ_Ð`bÐdfÓgˆ	Ø!×)Ñ)¨*·~±~Õ*EÀrÈ4Ï=É=ÓYˆð !Ÿ=™=ØØØ—‘Ø×&Ñ&ð	 )ó 
Ðð (×,Ñ,¨Z¿¹ÈÐSUÓVˆØÒ%Ø'¨.Õ8ˆLô Ÿ)š) L°bÄÇÁÔN×QÑQÐR]×RcÑRcÓdˆð ×0Ñ0°ÓAˆð $3×#7Ñ#7¸
Ç^Á^Õ8SÐU]Ð_aÓ#bÐ ô Ÿ	š	Ð":¸KÓHˆð ×)Ñ)¨-Ó8ˆð ×Ñ Ô" t×':×':Ð':Ø×%Ñ%¨×(;Ñ(;Õ;ˆFÜ!×,Ò,¨]Ó;ˆMÜ˜4×.Ñ.Ö/�Ø -´·²Ø! ! Q¬¨A°­J«¼#¸qÀ1½uÈÕ>NÓ:OÐ(OÐ"OÕPØ—J‘J×%Ñ% a¬¨Q°­Z«¼3ÀÀAÅÈÕ?OÓ;PÐ)PÐ&PÕQó1õ !’ò 0ð !ŸJ™J }Ó5ˆMä# M°8×=PÑ=PÐRV×R_ÑR_Ó`ˆØ˜oÐ-Ð-r&   )r®   r§   r¬   r¡   r£   rŸ   r¦   r—   r   r�   r«   rž   r¢   rn   ©NFF)rz   r{   r|   r}   r†   r¼   rÃ   rq   r   r€   r�   r‘   s   @@r$   r“   r“   ¦   s:   ù‡ € ÷Fõ F÷B3ð 3÷(Qð Q÷2A.÷ A.ò A.r&   r“   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚBloomMLPi9  c                ó    <€ V ^8„  d   QhRS[ /# ©r   r–   r   )r#   rk   s   "€r$   r%   ÚBloomMLP.__annotate__:  s   ø€ ÷ 	4ñ 	4™{ñ 	4r&   c                óL  <€ \         SV `  4        VP                  pVP                  V n        VP                  V n        \
        P                  ! V^V,          4      V n        \        4       V n	        \
        P                  ! ^V,          V4      V n
        VP                  V n        R# )é   N)r…   r†   rŸ   r�   rž   r   rª   Údense_h_to_4hrƒ   Ú	gelu_implÚdense_4h_to_hr£   )r‡   r–   rŸ   rˆ   s   && €r$   r†   ÚBloomMLP.__init__:  sz   ø€ Ü‰ÑÔØ×(Ñ(ˆà$×3Ñ3ˆÔØ$×3Ñ3ˆÔÜŸYšY {°A¸µOÓDˆÔÜ"›ˆŒÜŸYšY q¨;¥¸ÓDˆÔØ$×3Ñ3ˆÖr&   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   rÆ   rK   r   rX   )r#   rk   s   "€r$   r%   rè   E  s.   ø€ ÷ ñ ¡U§\¡\ð ¹U¿\¹\ð ÉeÏlÉlñ r&   c                óô  € V P                  V P                  V4      4      pV P                  ^8”  Ed   V P                  '       Ed   \        P
                  ! V4      pV P                  P                  P                  R,          V P                  ,          p\        V P                  4       F˜  pV\        P                  ! VRR\        WT,          4      \        V^,           V,          4      13,          V P                  P                  R\        WT,          4      \        V^,           V,          4      13,          4      ,           pKš  	  MV P                  V4      p\        W2V P                  V P                  4      pV# )rµ   r.   r-   )rì   rë   r�   rž   r    rÔ   rí   r×   r/   rÕ   rR   rÖ   r"   rU   r£   rM   )r‡   rÆ   rK   Úintermediate_outputrß   rá   Úoutputs   &&&    r$   rq   ÚBloomMLP.forwardE  s&  € ØŸ™ t×'9Ñ'9¸-Ó'HÓIˆà×Ñ Õ" t×':×':Ñ':Ü"'×"2Ò"2°8Ó"<ÐØ×'Ñ'×.Ñ.×4Ñ4°RÕ8¸4×;NÑ;NÕNˆFÜ˜4×.Ñ.Ö/�Ø&9¼A¿HºHØ! ! Q¬¨A­J«¼#¸qÀ1½uÈÕ>NÓ:OÐ(OÐ"OÕPØ×&Ñ&×-Ñ-¨a´°QµZ³Ä3ÈÈAÍÐQWÕGWÓCXÐ1XÐ.XÕYó=õ 'Ò#ò 0ð #'×"4Ñ"4°]Ó"CÐäÐ0¸D×<OÑ<OÐQU×Q^ÑQ^Ó_ˆàˆr&   )rí   rë   rì   r£   r�   rž   ©	rz   r{   r|   r}   r†   rq   r   r€   r�   r‘   s   @@r$   rå   rå   9  s   ù‡ € ÷	4ó 	4÷÷ ð r&   rå   c                   óX   a a€ ] tR tRt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )	Ú
BloomBlockiX  c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# r•   r˜   )r#   rk   s   "€r$   r%   ÚBloomBlock.__annotate__Y  s   ø€ ÷ 4ñ 4™{ð 4±s¸Tµzñ 4r&   c                óN  <€ \         SV `  4        VP                  p\        W1P                  R 7      V n        VP                  V n        \        W4      V n	        \        W1P                  R 7      V n
        \        V4      V n        VP                  V n        VP                  V n        R# )©ÚepsN)r…   r†   rŸ   r   Úlayer_norm_epsilonÚinput_layernormr    r   r“   Úself_attentionÚpost_attention_layernormrå   ÚmlpÚ(apply_residual_connection_post_layernormr£   )r‡   r–   r—   rŸ   rˆ   s   &&& €r$   r†   ÚBloomBlock.__init__Y  s   ø€ Ü‰ÑÔØ×(Ñ(ˆä(¨×:SÑ:SÔTˆÔØŸ™ˆŒÜ,¨VÓ?ˆÔÜ(1°+×C\ÑC\Ô(]ˆÔ%ä˜FÓ#ˆŒà8>×8gÑ8gˆÔ5Ø$×3Ñ3ˆÖr&   c                óˆ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[R,          RS[RS[/# )r   rÆ   rG   r   rÇ   NrÈ   rÉ   rÊ   )r#   rk   s   "€r$   r%   rø   g  sW   ø€ ÷ +$ñ +$á—|‘|ð+$ñ �|‰|ð+$ñ Ÿ™ð	+$ñ
 ˜D•Lð+$ñ ð+$ñ  ñ+$r&   c           
     óþ   € V P                  V4      pV P                  '       d   Tp	MTp	V P                  VV	VVVVVR 7      w  r«V P                  V
4      pV P                  '       d   Tp	MT
p	V P	                  W‰4      pWË3# ))rÇ   r   rG   rÈ   rÉ   )rý   r  rþ   rÿ   r   )r‡   rÆ   rG   r   rÇ   rÈ   rÉ   rØ   Úlayernorm_outputrK   Úattention_outputrÛ   rò   s   &&&&&&&,     r$   rq   ÚBloomBlock.forwardg  s£   € ð  ×/Ñ/°Ó>Ðð ×8×8Ð8Ø'‰Hà$ˆHð *.×)<Ñ)<ØØØ!Ø)ØØØ/ð *=ó *
Ñ&Ðð  ×8Ñ8Ð9IÓJÐð ×8×8Ð8Ø'‰Hà'ˆHð —‘Ð*Ó5ˆàÐ#Ð#r&   )r  r£   rý   r   r   rÿ   rþ   rn   rã   rô   r‘   s   @@r$   rö   rö   X  s   ù‡ € ÷4õ 4÷+$÷ +$ò +$r&   rö   c                   óD   a € ] tR tRt o RtRtR.tR.tRtV 3R lt	Rt
V tR# )	ÚBloomPreTrainedModeli•  ÚtransformerTrö   Úpast_key_valuesc                ó&   <€ V ^8„  d   Qh/ S[ ;R&   # rç   r   )r#   rk   s   "€r$   r%   Ú!BloomPreTrainedModel.__annotate__•  s   ø‡ ‚ áÑò r&   r,   N)rz   r{   r|   r}   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_can_compile_fullgraphÚ__annotate_func__r   r€   r�   s   @r$   r	  r	  •  s1   ø‡ € ð &ÐØ&*Ð#Ø%˜ÐØ#4Ð"5ÐØ!Ð÷ ƒ r&   r	  c                   óˆ   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltR tV3R lR lt]RV3R	 lR
 ll4       t	Rt
VtV ;t# )Ú
BloomModeliŸ  c                ó    <€ V ^8„  d   QhRS[ /# rç   r   )r#   rk   s   "€r$   r%   ÚBloomModel.__annotate__¡  s   ø€ ÷ ñ ™{ñ r&   c           
     ó*  <€ \         SV `  V4       VP                  V n        VP                  V n        \        P                  ! VP                  V P                  4      V n	        \        V P                  VP                  R 7      V n        \        P                  ! \        VP                  4       Uu. uF  p\!        WR7      NK  	  up4      V n        \        V P                  VP                  R 7      V n        RV n        V P)                  4        R# u upi )rú   )r—   FN)r…   r†   rŸ   Ú	embed_dimr    r   r   Ú	EmbeddingÚ
vocab_sizeÚword_embeddingsr   rü   Úword_embeddings_layernormÚ
ModuleListrÕ   Únum_hidden_layersrö   ÚhÚln_fÚgradient_checkpointingÚ	post_init)r‡   r–   rá   rˆ   s   && €r$   r†   ÚBloomModel.__init__¡  sÇ   ø€ Ü‰Ñ˜Ô à×+Ñ+ˆŒØŸ™ˆŒô  "Ÿ|š|¨F×,=Ñ,=¸t¿~¹~ÓNˆÔÜ)2°4·>±>Àv×G`ÑG`Ô)aˆÔ&ô —’ÌÈv×OgÑOgÔIhÓiÑIhÀA¤
¨6× ?ÑIhÑiÓjˆŒô ˜dŸn™n°&×2KÑ2KÔLˆŒ	à&+ˆÔ#ð 	�‰Öùò  js   Â0Dc                ón   <€ V ^8„  d   QhRS[ P                  RS[RS[ P                  RS[ P                  /# r   r   )r#   rk   s   "€r$   r%   r  ¶  sA   ø€ ÷ Dñ D±·±ð DÉ#ð DÑV[×VaÑVað DÑfk×frÑfrñ Dr&   c                ó   € \        WV4      # rn   )rH   )r‡   r   r   r   s   &&&&r$   rH   ÚBloomModel.build_alibi_tensor¶  s   € Ü! .¸UÓCÐCr&   c                ó   € V P                   # rn   ©r  )r‡   s   &r$   Úget_input_embeddingsÚBloomModel.get_input_embeddings¹  s   € Ø×#Ñ#Ð#r&   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   Únew_embeddingsrX   )r#   rk   s   "€r$   r%   r  ¼  s   ø€ ÷ .ñ .±5·<±<ñ .r&   c                ó   € Wn         R # rn   r)  ©r‡   r.  s   &&r$   Úset_input_embeddingsÚBloomModel.set_input_embeddings¼  s   € Ø-Ör&   c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          RS[R,          R	S[R,          R
S[S[ P                  R3,          S[,          /	# )r   Ú	input_idsNr  r   Úinputs_embedsrÈ   rÉ   Úoutput_hidden_statesÚreturn_dictr   .)r    Ú
LongTensorr   r!   rO   r²   r   )r#   rk   s   "€r$   r%   r  À  s·   ø€ ÷ g
ñ g
á×#Ñ# dÕ*ðg
ñ  �ðg
ñ Ÿ™ tÕ+ð	g
ñ
 ×'Ñ'¨$Õ.ðg
ñ ˜$•;ðg
ñ   $�;ðg
ñ # T�kðg
ñ ˜D•[ðg
ñ 
‰u�|‰|˜SÐ Õ	!Ñ$MÕ	Mñg
r&   c	           
     ób  € Ve   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pVe   TMV P                   P                  pVRJ VRJ,          '       d   \        R4      hV P                  '       d2   V P                  '       d    V'       d   \        P                  R4       RpVf   V P                  V4      pV'       d   Vf   \        V P                   R7      pVP                  w  r«pVe   VP                  4       M^ pW½,           pV P                  V4      pV'       d   RMRpV'       d   RMRpVf%   \        P                   ! W®3VP"                  R7      pMVP%                  VP"                  4      pV P'                  W0P(                  VP*                  R7      p\-        V P                   VVVR7      p\/        V P0                  4       FJ  w  ppV'       d   VV3,           pV! VVVVVVR	7      pV^ ,          pV'       g   K9  VV^,          3,           pKL  	  V P3                  V4      pV'       d   VV3,           pV'       g3   \4        ;QJ d    . R
 WòVV3 4       F  NK  	  5# ! R
 WòVV3 4       4      # \7        VVVVR7      # )áj  
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
    `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
    (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

    If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
    `input_ids`.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)r–   ©r)   )r   )r–   r5  r   r  )rÇ   r   rÈ   rÉ   rG   c              3   ó0   "  € T F  qf   K  Vx € K  	  R # 5irn   r,   )Ú.0Úvs   & r$   Ú	<genexpr>Ú%BloomModel.forward.<locals>.<genexpr>  s   é € ð Ùc�a—’Ócùs   ‚Œ
)Úlast_hidden_stater  rÆ   Ú
attentionsr,   )r–   rÉ   r6  rÈ   r7  r¤   r"  rM   r¨   r©   r  r	   r/   Úget_seq_lengthr  r    Úonesr)   r<   rH   r   r   r   Ú	enumerater   r!  r²   r   )r‡   r4  r  r   r5  rÈ   rÉ   r6  r7  rØ   r=   r>   rÂ   Úpast_lengthÚseq_length_with_pastrÆ   Úall_self_attentionsÚall_hidden_statesrG   Úcausal_maskrá   ÚblockÚoutputss   &&&&&&&&&,             r$   rq   ÚBloomModel.forward¿  s’  € ð4 2CÒ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ò$DÑ È$Ï+É+×JjÑJjð 	ð "+Ò!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ò%<‘kÀ$Ç+Á+×BYÑBYˆà˜Ð -°tÐ";×<Ô<ÜÐYÓZÐZà×&×&Ð&¨4¯=¯=¨=¿YÜ×ÑØlôð ˆIàÒ Ø ×0Ñ0°Ó;ˆMç˜Ò0Ü*°$·+±+Ô>ˆOà$1×$7Ñ$7Ñ!ˆ
 Ø:IÒ:U�o×4Ñ4Ô6Ð[\ˆØ)Õ7Ðà×6Ñ6°}ÓEˆç$5™b¸4Ðß"6™B¸DÐð Ò!Ü"ŸZšZ¨Ð(JÐS`×SgÑSgÔh‰Nà+×.Ñ.¨}×/CÑ/CÓDˆNà×'Ñ'¨¿¹Èm×NaÑNaÐ'ÓbˆÜ(Ø—;‘;Ø'Ø)Ø+ô	
ˆô " $§&¡&Ö)‰HˆAˆuß#Ø$5¸Ð8HÕ$HÐ!áØØ*Ø*Ø#Ø"3ØôˆGð $ A�JˆMß Ñ Ø&9¸WÀQ½Z¸MÕ&IÒ#ñ *ð$ Ÿ	™	 -Ó0ˆçØ 1°]Ð4DÕ DÐçß”5ñ Ø)Ð<MÐObÑcó—5ð �5ñ Ø)Ð<MÐObÑcóó ð ô 9Ø+Ø+Ø+Ø*ô	
ð 	
r&   )r  r"  r   r!  r   r  r  ©NNNNNNNN)rz   r{   r|   r}   r†   rH   r*  r1  r   rq   r   r€   r�   r‘   s   @@r$   r  r  Ÿ  sL   ù‡ € ÷ó ÷*Dð Dò$÷.ð .ð ÷g
ñ g
ó ÷g
ð g
r&   r  zˆ
    The Bloom Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    )Úcustom_introc                   óˆ   a a€ ] tR tRt oRR/tV3R lV 3R lltV3R lR ltRV 3R llt]RV3R	 lR
 ll4       t	Rt
VtV ;t# )ÚBloomForCausalLMi*  zlm_head.weightz"transformer.word_embeddings.weightc                ó    <€ V ^8„  d   QhRS[ /# rç   r   )r#   rk   s   "€r$   r%   ÚBloomForCausalLM.__annotate__3  s   ø€ ÷ ñ ™{ñ r&   c                óÊ   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  VP                  R R7      V n        V P                  4        R# ©Fr›   N)
r…   r†   r  r
  r   rª   rŸ   r  Úlm_headr#  ©r‡   r–   rˆ   s   &&€r$   r†   ÚBloomForCausalLM.__init__3  sI   ø€ Ü‰Ñ˜Ô Ü% fÓ-ˆÔÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Ör&   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r-  rX   )r#   rk   s   "€r$   r%   rS  ;  s   ø€ ÷ &ñ &±E·L±Lñ &r&   c                ó   € Wn         R # rn   )rV  r0  s   &&r$   Úset_output_embeddingsÚ&BloomForCausalLM.set_output_embeddings;  s   € Ø%Žr&   c                óR  <€ \         SV `  ! V3R VRVRVRVRV/VB p\        V\        4      '       du   Veq   VP	                  4       p	VP
                  w  r«W›,
          p\        P                  ! W¬VP                  VP                  R7      p\        P                  ! W=.RR7      pW8R&   V# )r  r   r5  rÈ   Úis_first_iterationr(   r*   r-   )r…   Úprepare_inputs_for_generationÚ
isinstancer
   Úget_max_lengthr/   r    Úzerosr)   r   r9   )r‡   r4  r  r   r5  rÈ   r^  rØ   Úmodel_inputsÚtarget_lengthr=   r>   ÚdiffÚnew_attn_maskrˆ   s   &&&&&&&,      €r$   r_  Ú.BloomForCausalLM.prepare_inputs_for_generation>  sÊ   ø€ ô ‘wÒ<Øñ
à+ð
ð *ð
ð (ð	
ð
  ð
ð  2ð
ð ñ
ˆô �o¤{×3Ò3¸Ò8RØ+×:Ñ:Ó<ˆMØ%3×%9Ñ%9Ñ"ˆJØ Õ-ˆDä!ŸKšK¨
À×AVÑAVÐ^l×^rÑ^rÔsˆMÜ"ŸYšY¨Ð'FÈBÔOˆNØ-;Ð)Ñ*àÐr&   c                ó‚  <€ V ^8„  d   QhRS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[R,          RS[S[ P                  ,          RS[S[ P                  ,          S[,          /# )r   r4  Nr  r   r5  ÚlabelsrÈ   rÉ   r6  r7  Úlogits_to_keepr   )r    r8  r   r!   rO   r"   r²   r   )r#   rk   s   "€r$   r%   rS  b  sØ   ø€ ÷ D
ñ D
á×#Ñ# dÕ*ðD
ñ  �ðD
ñ Ÿ™ tÕ+ð	D
ñ
 —|‘| dÕ*ðD
ñ —‘˜tÕ#ðD
ñ ˜$•;ðD
ñ   $�;ðD
ñ # T�kðD
ñ ˜D•[ðD
ñ ™eŸl™lÕ*ðD
ñ 
‰u�|‰|Õ	Ñ@Õ	@ñD
r&   c                ó  € V	e   T	MV P                   P                  p	V P                  VVVVVVVV	R7      pV^ ,          p\        V
\        4      '       d   \        V
) R4      MT
pV P                  VRVR3,          4      pRpVe+   V P                  ! VV3RV P                   P                  /VB pV	'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  VP                  R7      # )aô  
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
    `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
    (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

    If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
    `input_ids`.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
    `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
    are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
N©r  r   r5  rÈ   rÉ   r6  r7  r.   r  ºrµ   NN©ÚlossÚlogitsr  rÆ   rB  )r–   r7  r
  r`  r"   ÚslicerV  Úloss_functionr  r   r  rÆ   rB  )r‡   r4  r  r   r5  ri  rÈ   rÉ   r6  r7  rj  rØ   Útransformer_outputsrÆ   Úslice_indicesrp  ro  rò   s   &&&&&&&&&&&,      r$   rq   ÚBloomForCausalLM.forwarda  s(  € ð@ &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ó 	
Ðð ,¨AÕ.ˆä8BÀ>ÔSV×8WÒ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜m¨A¨}¸aÐ,?Õ@ÓAˆàˆØÒØ×%Ò%ØØñð  Ÿ;™;×1Ñ1ðð ñ	ˆD÷ Ø�YÐ!4°RÕ!8Õ8ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä0ØØØ/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r&   )rV  r
  )NNNTF)
NNNNNNNNNr´   )rz   r{   r|   r}   Ú_tied_weights_keysr†   r[  r_  r   rq   r   r€   r�   r‘   s   @@r$   rQ  rQ  *  sO   ù‡ € ð +Ð,PÐQÐ÷ó ÷&ð &÷!ðF ÷D
ñ D
ó ÷D
ð D
r&   rQ  aÖ  
    The Bloom Model transformer with a sequence classification head on top (linear layer).

    [`BloomForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-1) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   ó^   a a€ ] tR tRt oV3R lV 3R llt]RV3R lR ll4       tRtVtV ;t	# )ÚBloomForSequenceClassificationi©  c                ó    <€ V ^8„  d   QhRS[ /# rç   r   )r#   rk   s   "€r$   r%   Ú+BloomForSequenceClassification.__annotate__¸  s   ø€ ÷ ñ ™{ñ r&   c                óì   <€ \         SV `  V4       VP                  V n        \        V4      V n        \
        P                  ! VP                  VP                  R R7      V n        V P                  4        R# rU  )
r…   r†   Ú
num_labelsr  r
  r   rª   rŸ   Úscorer#  rW  s   &&€r$   r†   Ú'BloomForSequenceClassification.__init__¸  sV   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ% fÓ-ˆÔÜ—Y’Y˜v×1Ñ1°6×3DÑ3DÈ5ÔQˆŒ
ð 	�‰Ör&   c                óX  <€ V ^8„  d   QhRS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[R,          RS[S[ P                  ,          S[,          /
# ©r   r4  Nr  r   r5  ri  rÈ   rÉ   r6  r7  r   )r    r8  r   r!   rO   r²   r   )r#   rk   s   "€r$   r%   rz  Â  sÄ   ø€ ÷ e
ñ e
á×#Ñ# dÕ*ðe
ñ  �ðe
ñ Ÿ™ tÕ+ð	e
ñ
 —|‘| dÕ*ðe
ñ —‘˜tÕ#ðe
ñ ˜$•;ðe
ñ   $�;ðe
ñ # T�kðe
ñ ˜D•[ðe
ñ 
‰u�|‰|Õ	Ñ?Õ	?ñe
r&   c
                ó
  € V	e   T	MV P                   P                  p	V P                  VVVVVVVV	R7      pV^ ,          pV P                  V4      pVe   VP                  ^ ,          pMVP                  ^ ,          pV P                   P
                  f   V^8w  d   \        R4      hV P                   P
                  f   RpMÍVeœ   WP                   P
                  8g  P                  VP                  \        P                  4      p\        P                  ! VP                  R,          VP                  \        P                  R7      pVV,          P                  R4      pM.Rp\        P                  V P                  P                    R24       V\        P                  ! WíP                  R7      V3,          pRpVEeq   V P                   P"                  f–   V P$                  ^8X  d   RV P                   n        MrV P$                  ^8”  dQ   VP&                  \        P(                  8X  g    VP&                  \        P*                  8X  d   RV P                   n        MR	V P                   n        V P                   P"                  R8X  dL   \-        4       pV P$                  ^8X  d'   V! VP/                  4       VP/                  4       4      pMgV! VV4      pM]V P                   P"                  R8X  d   \1        4       pV! VV4      pM.V P                   P"                  R	8X  d   \3        4       pV! VV4      pV	'       g"   V3VR
,          ,           pVe   V3V,           # T# \5        VVVP6                  VP8                  VP:                  R7      # )áÎ  
input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
    `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
    (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

    If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
    `input_ids`.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Nrl  z=Cannot handle batch sizes > 1 if no padding token is defined.r(   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r;  Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrm  rn  r-   )r–   r7  r
  r}  r/   Úpad_token_idr¤   r<   r)   r    r6   r5   Úargmaxr¨   r©   rˆ   rz   Úproblem_typer|  r   Úlongr"   r   Úsqueezer   r   r   r  rÆ   rB  )r‡   r4  r  r   r5  ri  rÈ   rÉ   r6  r7  rØ   rs  rÆ   rp  r=   Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsro  Úloss_fctrò   s   &&&&&&&&&&,           r$   rq   Ú&BloomForSequenceClassification.forwardÁ  sç  € ð> &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ó 	
Ðð ,¨AÕ.ˆØ—‘˜MÓ*ˆàÒ Ø"Ÿ™¨Õ+‰Jà&×,Ñ,¨QÕ/ˆJà�;‰;×#Ñ#Ò+°
¸a´ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ò+Ø!#ÑØÒ"à%¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸLšL¨¯©¸Õ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Õ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|š|¨J¿}¹}ÔMÐOaÐaÕbˆàˆØÓØ�{‰{×'Ñ'Ò/Ø—?‘? aÔ'Ø/;�D—K‘KÕ,Ø—_‘_ qÔ(¨f¯l©l¼e¿j¹jÔ.HÈFÏLÉLÔ\a×\eÑ\eÔLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ô7Ü"›9�Ø—?‘? aÔ'Ù# M×$9Ñ$9Ó$;¸V¿^¹^Ó=MÓN‘Dá# M°6Ó:‘DØ—‘×)Ñ)Ð-JÔJÜ+Ó-�Ù ¨vÓ6‘Ø—‘×)Ñ)Ð-IÔIÜ,Ó.�Ù ¨vÓ6�ßØ#Ð%Ð(;¸BÕ(?Õ?ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä/ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r&   )r|  r}  r
  ©	NNNNNNNNN©
rz   r{   r|   r}   r†   r   rq   r   r€   r�   r‘   s   @@r$   rx  rx  ©  s1   ù‡ € ÷ó ð ÷e
ñ e
ó ÷e
ð e
r&   rx  c                   ó^   a a€ ] tR tRt oV3R lV 3R llt]RV3R lR ll4       tRtVtV ;t	# )ÚBloomForTokenClassificationi*  c                ó    <€ V ^8„  d   QhRS[ /# rç   r   )r#   rk   s   "€r$   r%   Ú(BloomForTokenClassification.__annotate__,  s   ø€ ÷ ñ ™{ñ r&   c                óÖ  <€ \         SV `  V4       VP                  V n        \        V4      V n        \        VR 4      '       d   VP                  e   VP                  pM/\        VR4      '       d   VP                  e   VP                  pMRp\        P                  ! V4      V n
        \        P                  ! VP                  VP                  4      V n        V P                  4        R# )Úclassifier_dropoutNr£   gš™™™™™¹?)r…   r†   r|  r  r
  Úhasattrr˜  r£   r   r­   rS   rª   rŸ   Ú
classifierr#  )r‡   r–   r˜  rˆ   s   && €r$   r†   Ú$BloomForTokenClassification.__init__,  sµ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä% fÓ-ˆÔÜ�6Ð/×0Ò0°V×5NÑ5NÒ5ZØ!'×!:Ñ!:ÑÜ�VÐ-×.Ò.°6×3HÑ3HÒ3TØ!'×!6Ñ!6Ñà!$ÐÜ—z’zÐ"4Ó5ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Ör&   c                óX  <€ V ^8„  d   QhRS[ P                  R,          RS[R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[R,          RS[S[ P                  ,          S[,          /
# r€  )r    r8  r   r!   rO   r²   r   )r#   rk   s   "€r$   r%   r–  >  sÄ   ø€ ÷ B
ñ B
á×#Ñ# dÕ*ðB
ñ  �ðB
ñ Ÿ™ tÕ+ð	B
ñ
 —|‘| dÕ*ðB
ñ —‘˜tÕ#ðB
ñ ˜$•;ðB
ñ   $�;ðB
ñ # T�kðB
ñ ˜D•[ðB
ñ 
‰u�|‰|Õ	Ñ4Õ	4ñB
r&   c
                óN  € V	e   T	MV P                   P                  p	V P                  VVVVVVVV	R7      pV^ ,          pV P                  V4      pV P	                  V4      pRpVeu   VP                  VP                  4      pVP                  w  pp\        4       pV! VP                  VV,          V P                  4      VP                  VV,          4      4      pV	'       g"   V3VR,          ,           pVe   V3V,           # T# \        VVVP                  VP                  R7      # )r‚  Nrl  ºr   NN)ro  rp  rÆ   rB  )r–   r7  r
  rS   rš  r<   r)   r/   r   r¶   r|  r   rÆ   rB  )r‡   r4  r  r   r5  ri  rÈ   rÉ   r6  r7  rØ   rs  rÆ   rp  ro  r=   r>   r�  rò   s   &&&&&&&&&&,        r$   rq   Ú#BloomForTokenClassification.forward=  s)  € ð> &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ó 	
Ðð ,¨AÕ.ˆØŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÒà—Y‘Y˜vŸ}™}Ó-ˆFØ%+§\¡\Ñ"ˆJ˜
Ü'Ó)ˆHÙØ—‘˜J¨Õ3°T·_±_ÓEÀvÇ{Á{ÐS]Ð`jÕSjÓGkóˆD÷ Ø�YÐ!4°RÕ!8Õ8ˆFØ)-Ò)9�T�G˜fÕ$ÐE¸vÐEä$ØØØ-×;Ñ;Ø*×5Ñ5ô	
ð 	
r&   )rš  rS   r|  r
  r‘  r’  r‘   s   @@r$   r”  r”  *  s1   ù‡ € ÷ó ð" ÷B
ñ B
ó ÷B
ð B
r&   r”  c                   óR   a a€ ] tR tRt oV 3R lt]RV3R lR ll4       tRtVtV ;t	# )ÚBloomForQuestionAnsweringiƒ  c                ó²   <€ \         SV `  V4       \        V4      V n        \        P
                  ! VP                  ^4      V n        V P                  4        R# )r   N)	r…   r†   r  r
  r   rª   rŸ   Ú
qa_outputsr#  rW  s   &&€r$   r†   Ú"BloomForQuestionAnswering.__init__…  sA   ø€ Ü‰Ñ˜Ô Ü% fÓ-ˆÔÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰Ör&   c                ó4  <€ V ^8„  d   QhRS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[ P                  R,          RS[R,          RS[R,          R	S[R,          R
S[S[,          /	# )r   r4  Nr   r5  Ústart_positionsÚend_positionsrÉ   r6  r7  r   )r    r8  ÚFloatTensorrO   r²   r   )r#   rk   s   "€r$   r%   Ú&BloomForQuestionAnswering.__annotate__Ž  s¹   ø€ ÷ F
ñ F
á×#Ñ# dÕ*ðF
ñ ×)Ñ)¨DÕ0ðF
ñ ×(Ñ(¨4Õ/ð	F
ñ
 ×)Ñ)¨DÕ0ðF
ñ ×'Ñ'¨$Õ.ðF
ñ   $�;ðF
ñ # T�kðF
ñ ˜D•[ðF
ñ 
Ñ-Õ	-ñF
r&   c	           	     ón  € Ve   TMV P                   P                  pV P                  VVVVVVR7      p
V
^ ,          pV P                  V4      pVP	                  ^RR7      w  rÞVP                  R4      P                  4       pVP                  R4      P                  4       pRpVeÄ   VeÀ   \        VP                  4       4      ^8”  d   VP                  R4      p\        VP                  4       4      ^8”  d   VP                  R4      pVP                  ^4      pVP                  ^ V4      pVP                  ^ V4      p\        VR7      pV! WÔ4      pV! Wå4      pVV,           ^,          pV'       g"   WÞ3V
R,          ,           pVe   V3V,           # T# \        VVVV
P                  V
P                  R7      # )r:  N)r   r5  rÉ   r6  r7  r*   )Úignore_indexrž  )ro  Ústart_logitsÚ
end_logitsrÆ   rB  r-   )r–   r7  r
  r£  ÚsplitrŠ  Ú
contiguousÚlenÚsizeÚclampr   r   rÆ   rB  )r‡   r4  r   r5  r¦  r§  rÉ   r6  r7  rØ   rL  Úsequence_outputrp  r¬  r­  Ú
total_lossÚignored_indexr�  Ú
start_lossÚend_lossrò   s   &&&&&&&&&,           r$   rq   Ú!BloomForQuestionAnswering.forward�  s±  € ð4 &1Ò%<‘kÀ$Ç+Á+×BYÑBYˆà×"Ñ"ØØ)Ø'Ø/Ø!5Ø#ð #ó 
ˆð " !�*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÒ&¨=Ò+Dä�?×'Ñ'Ó)Ó*¨QÔ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ô,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,Ó@ˆJÙ 
Ó:ˆHØ$ xÕ/°1Õ4ˆJçØ"Ð/°'¸"µ+Õ=ˆFØ/9Ò/E�Z�M FÕ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r&   )r£  r
  rN  r’  r‘   s   @@r$   r¡  r¡  ƒ  s,   ù‡ € õð ÷F
ñ F
ó ÷F
ð F
r&   r¡  )rQ  r  r	  rx  r”  r¡  )8r�   r0   r    r   Útorch.nnr   r   r   r   r   rR   Úcache_utilsr   r	   r
   Ú
generationr   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_bloomr   Ú
get_loggerrz   r¨   rH   rU   r`   rf   ÚautogradÚFunctionrh   ÚModulerƒ   r“   rå   rö   r	  r  rQ  rx  r”  r¡  Ú__all__r,   r&   r$   Ú<module>rÇ     sœ  ðñ ã ã Ý ß LÓ LÝ $ç ;Ñ ;Ý )Ý /Ý 9÷õ õ .÷õ -ð 
×	Ò	˜HÓ	%€õ,Jõ^õ&	Qõô$
�5—>‘>×*Ñ*ô 
ô	%�—	‘	ô 	%ôP.�R—Y‘Yô P.ôfˆr�y‰yô ô>:$Ð+ô :$ðz ô"˜?ó "ó ð"ð ôG
Ð%ó G
ó ðG
ñT ðôôv
Ð+¨_ó v
óðv
ñr ðôôp
Ð%9ó p
óðp
ðf ôU
Ð"6ó U
ó ðU
ðp ôP
Ð 4ó P
ó ðP
òf‚r&   