+
    "Ý³jÎ  ã                   ó  € ^ RI t ^ RIt^ RIHt ^RIHt ^RIHt ^RIH	t	 ^RI
HtHtHt ]! 4       tR R ltR	 tR
 R ltR tR tR R ltR R ltR R ltR R ltR tR tR R ltR t]P8                  ! 4       RR R ll4       tR# )é    N)ÚPreTrainedModel)Úsetup_logger)Úget_module_by_name_prefix)Útorch_empty_cache)Úapply_exact_had_to_linearÚget_hadKÚrandom_hadamard_matrixc                ó®   € V ^8„  d   QhR\         P                  P                  R\        P                  \         P                  P
                  ,          RR/# )é   Ú	layernormÚlinear_layersÚreturnN)ÚtorchÚnnÚModuleÚtypingÚIterableÚLinear)Úformats   "Úi/home/luca/screen-checking/.venv/lib/python3.14/site-packages/gptqmodel/quantization/rotation/rotation.pyÚ__annotate__r      sB   € ÷ Añ AÜ—8‘8—?‘?ðAÜ39·?±?Ä5Ç8Á8Ç?Á?Õ3SðAà	ñAó    c                ó(  € V EFŠ  pVP                   P                  pVP                   P                  P                  4       pW@P                   P                  4       ,          P	                  V4      VP                   n        \        V R4      '       g   K�  VP                  fS   \        P                  P                  \        P                  ! VP                  \        P                  R7      4      Vn        VP                  P                  P                  4       \        P                  ! W@P                  P                  4       4      ,           VP                  n        VP                  P                  P	                  V4      VP                  n        EK�  	  R# )zJ
fuse the linear operations in Layernorm into the adjacent linear blocks.
ÚbiasN©Údtype)Úweightr   ÚdataÚdoubleÚtoÚhasattrr   r   r   Ú	ParameterÚzerosÚout_featuresÚfloat64Úmatmul)r   r   ÚlinearÚlinear_dtypeÚW_s   &&   r   Úfuse_ln_linearr*      s	  € ô  ˆØ—}‘}×*Ñ*ˆð �]‰]×Ñ×&Ñ&Ó(ˆØ ×#3Ñ#3×#:Ñ#:Ó#<Õ<×@Ñ@ÀÓNˆ�‰Ôä�9˜f×%Ô%Ø�{‰{Ò"Ü#Ÿh™h×0Ñ0Ü—K’K × 3Ñ 3¼5¿=¹=ÔIó�”ð  &Ÿ{™{×/Ñ/×6Ñ6Ó8¼5¿<º<Ø—N‘N×)Ñ)Ó+ó<õ  ˆF�K‰KÔð  &Ÿ{™{×/Ñ/×2Ñ2°<Ó@ˆF�K‰K×ó  r   c                 ó|   € V P                   P                  p\        P                  ! V4      V P                   n        R # ©N)r   r   r   Ú	ones_like)ÚlnÚW_norms   & r   Úreset_lnr0   -   s#   € Ø�Y‰Y�^‰^€FÜ—_’_ VÓ,€B‡I�I†Nr   c                óH   € V ^8„  d   QhR\         R\        R\        R\        /# )r   ÚmodelÚpre_lm_head_norm_module_nameÚlayers_nodeÚlm_head_name)r   Ústr)r   s   "r   r   r   2   s*   € ÷ ñ œOð Ì3ð Ô]`ð Ôpsñ r   c                 ó  € \        W4      w  rEV FÂ  p\        VP                  VP                  P                  VP                  P
                  .4       \        VP                  VP                  P                  VP                  P                  VP                  P                  .4       \        VP                  4       \        VP                  4       KÄ  	  \        W4      w  ru\        W4      w  r…\        Wx.4       \        V4       V # r,   )r   r*   Úpost_attention_layernormÚmlpÚup_projÚ	gate_projÚinput_layernormÚ	self_attnÚq_projÚk_projÚv_projr0   )	r2   r3   r4   r5   ÚlayersÚ_ÚlayerÚpre_head_layernormÚlm_heads	   &&&&     r   Úfuse_layer_normsrF   2   sØ   € Ü*¨5Ó>�J€Fó ˆäØ×*Ñ*¨U¯Y©Y×->Ñ->ÀÇ	Á	×@SÑ@SÐ,Tô	
ô 	Ø×!Ñ!à—‘×&Ñ&Ø—‘×&Ñ&Ø—‘×&Ñ&ðô	
ô 	�×/Ñ/Ô0Ü�×&Ñ&Ö'ñ ô  6°eÓZÑÐÜ*¨5Ó?�J€GäÐ% yÔ1ÜÐÔ Ø€Lr   c                ón  € \         P                  P                  4        \         P                  ! W \         P                  R7      P                  V4      p\         P                  P                  V4      w  r4V\         P                  ! \         P                  ! V4      4      P                  ^ 4      ,          pV# )a�  
Generate a random orthogonal matrix of the specified size.
First, we generate a random matrix with entries from a standard distribution.
Then, we use QR decomposition to obtain an orthogonal matrix.
Finally, we multiply by a diagonal matrix with diag r to adjust the signs.

Args:
size (int): The size of the matrix (size x size).

Returns:
torch.Tensor: An orthogonal matrix of the specified size.
r   )r   ÚcudaÚempty_cacheÚrandnr%   r    ÚlinalgÚqrÚsignÚdiagÚ	unsqueeze)ÚsizeÚdeviceÚrandom_matrixÚqÚrs   &&   r   Úrandom_orthogonal_matrixrU   N   sr   € ô 
‡J�J×ÑÔÜ—K’K ´%·-±-Ô@×CÑCÀFÓK€MÜ�<‰<�?‰?˜=Ó)�D€AØŒ�Š”E—J’J˜q“MÓ	"×	,Ñ	,¨QÓ	/Õ/€AØ€Hr   c                 óf   € VR 8X  d   \        W4      # VR8X  d   \        W4      # \        RV 24      h)ÚrandomÚhadamardzUnknown mode )rU   r	   Ú
ValueError)rP   ÚmoderQ   s   &&&r   Úget_orthogonal_matrixr[   b   s:   € ØˆxÔÜ'¨Ó5Ð5Ø	�Ô	Ü% dÓ3Ð3ä˜=¨¨Ð/Ó0Ð0r   c                ób   € V ^8„  d   QhR\         \        P                  P                  ,          /# )r   r   )Úlistr   r   r   )r   s   "r   r   r   k   s    € ÷ &ñ &œT¤%§(¡(§/¡/Õ2ñ &r   c                 ó0   € V P                   P                  .# r,   )r2   Úembed_tokens)r2   s   &r   Úget_embeddingsr`   k   s   € Ø�K‰K×$Ñ$Ð%Ð%r   c                ó   € V ^8„  d   QhRR/# ©r   r   N© )r   s   "r   r   r   o   s   € ÷ Jñ J¨4ñ Jr   c                 ó@  € \        V 4       FŽ  pVP                  P                  P                  pVP                  P                  P	                  V\
        P                  R 7      p\
        P                  ! WQ4      P	                  RVR 7      VP                  n        K�  	  R# ©©rQ   r   ÚcpuN)r`   r   r   r   r    r   r%   r&   )r2   ÚQrQ   ÚWr   r)   s   &&&   r   Úrotate_embeddingsrj   o   si   € ä˜EÖ"ˆØ—‘—‘×#Ñ#ˆØ�X‰X�]‰]×Ñ V´5·=±=ÐÓAˆÜŸš RÓ+×.Ñ.°eÀ5Ð.ÓIˆ�‰Žó #r   c                ó   € V ^8„  d   QhRR/# rb   rc   )r   s   "r   r   r   w   s   € ÷ Jñ J°ñ Jr   c                 ó„  € V P                   P                  V P                   P                  V P                   P                  3 Fz  pVP                  P
                  pVP                  P                  V\        P                  R 7      p\        P                  ! WQ4      P                  RVR 7      VP                  n
        K|  	  R# re   )r=   r>   r?   r@   r   r   r    r   r%   r&   r   )rC   rh   rQ   ri   r   r)   s   &&&   r   Úrotate_attention_inputsrm   w   s}   € à�o‰o×$Ñ$ e§o¡o×&<Ñ&<¸e¿o¹o×>TÑ>TÓUˆØ—‘—‘ˆØ�X‰X�[‰[ ¬e¯m©mˆ[Ó<ˆÜŸš RÓ+×.Ñ.°eÀ5Ð.ÓIˆ�‰Žó Vr   c                ó   € V ^8„  d   QhRR/# rb   rc   )r   s   "r   r   r      s   € ÷ Iñ I°ñ Ir   c                 ój  € V P                   P                  pVP                  P                  P                  pVP                  P                  P                  V\        P                  R 7      p\        P                  ! VP                  V4      P                  RVR 7      VP                  n        VP                  ey   VP                  P                  P                  V\        P                  R 7      p\        P                  ! VP                  V4      P                  RVR 7      VP                  n        R# R# re   )r=   Úo_projr   r   r   r    r   r%   r&   ÚTr   )rC   rh   rQ   ri   r   r)   Úbs   &&&    r   Úrotate_attention_outputrs      sÆ   € à�‰×Ñ€AØ�H‰H�M‰M×Ñ€EØ	
�‰�‰×	Ñ	 ¬u¯}©}Ð	Ó	=€BÜ—L’L §¡ bÓ)×,Ñ,°EÀÐ,ÓG€A‡H�H„MØ‡v�vÒØ�F‰F�K‰K�N‰N &´·±ˆNÓ>ˆÜ—l’l 1§3¡3¨Ó*×-Ñ-°UÀ%Ð-ÓHˆ�‰Žñ r   c                 ór  € V P                   P                  V P                   P                  .pV F„  pVP                  P                  pVP                  P
                  P                  V\        P                  R 7      p\        P                  ! Wa4      P                  RVR 7      VP                  n        K†  	  R# re   )
r9   r:   r;   r   r   r   r    r   r%   r&   )rC   rh   rQ   Ú
mlp_inputsri   r   r)   s   &&&    r   Úrotate_mlp_inputrv   Š   s|   € à—)‘)×#Ñ# U§Y¡Y×%8Ñ%8Ð9€JÛˆØ—‘—‘ˆØ�X‰X�]‰]×Ñ V´5·=±=ÐÓAˆÜŸš RÓ+×.Ñ.°eÀ5Ð.ÓIˆ�‰Žó r   c                 óö  € V P                   P                  pVP                  P                  P                  pVP                  P                  P                  V\        P                  R 7      p\        P                  ! VP                  V4      P                  RVR 7      VP                  n        \        VRRR7       RVn        RVn        RVn        \        VP                  4      w  rgWcn        Wsn        VP$                  ey   VP$                  P                  P                  V\        P                  R 7      p\        P                  ! VP                  V4      P                  RVR 7      VP$                  n        R# R# )rf   rg   F©Úhad_dimÚoutputTNéÿÿÿÿ)r9   Ú	down_projr   r   r   r    r   r%   r&   rq   r   Úonline_full_hadÚonline_partial_hadry   r   Úin_featuresÚhad_KÚKr   )	rC   rh   rQ   ri   r   r)   r€   r�   rr   s	   &&&      r   Úrotate_mlp_outputr‚   “   s  € à�	‰	×Ñ€AØ�H‰H�M‰M×Ñ€EØ	
�‰�‰×	Ñ	 ¬u¯}©}Ð	Ó	=€BÜ—L’L §¡ bÓ)×,Ñ,°EÀÐ,ÓG€A‡H�H„MÜ˜a¨°EÕ:ð
 €AÔØ €AÔØ€A„IÜ˜Ÿ™Ó&�H€EØ„GØ
„Cà‡v�vÒØ�F‰F�K‰K�N‰N &´·±ˆNÓ>ˆÜ—l’l 1§3¡3¨Ó*×-Ñ-°UÀ%Ð-ÓHˆ�‰Žñ r   c                ó   € V ^8„  d   QhRR/# rb   rc   )r   s   "r   r   r   ª   s   € ÷ Fñ F°4ñ Fr   c                 ó6  € \        W4      w  rEVP                  P                  P                  pVP                  P                  P	                  V\
        P                  R 7      p\
        P                  ! Wq4      P	                  RVR 7      VP                  n        R# re   )r   r   r   r   r    r   r%   r&   )r2   rh   rQ   r5   ri   rB   r   r)   s   &&&&    r   Úrotate_headr…   ª   sf   € ä$ UÓ9�D€AØ�H‰H�M‰M×Ñ€EØ	
�‰�‰×	Ñ	 ¬u¯}©}Ð	Ó	=€BÜ—L’L Ó'×*Ñ*°%¸uÐ*ÓE€A‡H�H†Mr   c                 ó’   € V P                   P                  pV P                   P                  p\        W2R R7       \        WBRR7       R# )Trx   FN)r=   r@   rp   r   )rC   Úhead_numÚhead_dimr@   rp   s   &&&  r   Úrotate_ov_projr‰   ²   s4   € Ø�_‰_×#Ñ#€FØ�_‰_×#Ñ#€FÜ˜f¸tÕDä˜f¸u×Er   c          
      óh   € V ^8„  d   QhR\         R\        R\        P                  R\        R\        /# )r   r2   Úrotate_moderQ   r5   r4   )r   r6   r   rQ   )r   s   "r   r   r   »   s5   € ÷ ñ œð ´cð Ä5Ç<Á<ð Ô_bð Ü!ñr   c                 óî  € Vf!   \        V P                  P                  W4      MTpV P                  pVP                  pVP                  pW‡,          p	\	        WV4       \        WW#4       \        4        \        W4      w  r«\        \        P                  V
4      P                  R4      4       FA  w  rÍ\        WÕV4       \        WÕV4       \        WÕV4       \        WÕV4       \!        W×V	4       KC  	  W3# )NÚRotating)r[   ÚconfigÚhidden_sizeÚnum_attention_headsrj   r…   r   r   Ú	enumerateÚlogÚpbÚtitlerm   rs   rv   r‚   r‰   )r2   r‹   rQ   r5   r4   rh   rŽ   Ú	num_headsÚ	model_dimrˆ   rA   rB   ÚidxrC   s   &&&&&&        r   Úrotate_modelr˜   º   sÛ   € ð Š9ô 	Ø�L‰L×$Ñ$ kô	
ð ð ð �\‰\€FØ×*Ñ*€IØ×"Ñ"€IØÕ%€Hä�e Ô'Ü�˜&Ô/ÜÔÜ*¨5Ó>�J€FÜ¤§¡ v£× 4Ñ 4°ZÓ @ÖA‰
ˆÜ ¨&Ô1Ü ¨&Ô1Ü˜ 6Ô*Ü˜% FÔ+Ü�u¨Ö2ñ Bð ˆ8€Or   r,   )r   r   Útransformersr   Úutils.loggerr   Úutils.modelr   Úutils.torchr   Úhadamard_utilsr   r   r	   r’   r*   r0   rF   rU   r[   r`   rj   rm   rs   rv   r‚   r…   r‰   Úinference_moder˜   rc   r   r   Ú<module>rŸ      sŽ   ðó ã Ý (å (Ý 4Ý ,ß WÑ Wñ ƒn€õAò0-õ
ò8ò(1õ&õJõJõIòJòIõ.FòFð ×ÒÓöó òr   