+
    "Ý³jþ*  ã                   ó¼  € ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIt^ RIt^ RI	H
t
 ^ RIHt ^ RIHtHt ^RIHt . ROsRtRtR	t]! 4       t]P.                  ! R
4      t]P.                  ! R4      t]P.                  ! R]P4                  P6                  R7      tR tR tR tR t RR lt!R t"R t#R t$R t%R t&R t']PP                  ! 4       R 4       t)RR R llt*R# )é    N)ÚPath)Úload_dataset)Ú
DataLoader)ÚPreTrainedModelÚPreTrainedTokenizerBase)Úsetup_loggeri   i   z	Question:zanswer is \(?([A-J])\)?z.*[aA]nswer:\s*([A-J])z\b[A-J]\b(?!.*\b[A-J]\b))Úflagsc                  ól   € \        R 4      p V R,          V R,          r!\        V4      p\        V4      pW3# )zTIGER-Lab/MMLU-ProÚtestÚ
validation)r   Ú
preprocess)ÚdatasetÚtest_dfÚval_dfs      ÚX/home/luca/screen-checking/.venv/lib/python3.14/site-packages/gptqmodel/utils/mmlupro.pyÚload_mmlu_pror   #   s9   € ÜÐ/Ó0€GØ˜f•o w¨|Õ'<ˆVÜ˜Ó!€GÜ˜Ó€FØˆ?Ðó    c                 óž   € . pV  FD  p. pVR ,           F  pVR8X  d   K  VP                  V4       K  	  W2R &   VP                  V4       KF  	  V# )ÚoptionszN/A©Úappend)r   Úres_dfÚeachr   Úopts   &    r   r   r   +   sX   € Ø€FÛˆØˆØ˜	—?�?ˆCØ�eŒ|ÙØ�N‰N˜3Öñ #ð "ˆY‰Ø�‰�dÖñ ð €Mr   c                 ó    € R p\        V\        4      '       d   RP                  V4      pVP                  RR4      P                  RR4      pWV.# )ÚCoTÚ,Ú-Ú Ú_)Ú
isinstanceÚlistÚjoinÚreplace)Ú
model_nameÚselected_subjectsÚscoring_methodÚsubjectss   &&  r   Úargs_generate_pathr)   8   sO   € Ø€NÜÐ#¤T×*Ò*ØŸH™HÐ%6Ó7ÐØ ×(Ñ(¨¨cÓ2×:Ñ:¸3ÀÓD€HØ¨Ð1Ð1r   c                 ó^   € . pV  F$  pVR ,          V8X  g   K  VP                  V4       K&  	  V# )Úcategoryr   )ÚdfÚsubjectÚresr   s   &&  r   Úselect_by_categoryr/   @   s1   € Ø
€CÛˆØ�
Õ˜wÖ&Ø�J‰J�tÖñ ð €Jr   c                 óD  € R pV R,          pV R,          pW#R,           ,          pVR,          p\        V4       F)  w  rVVRP                  \        V,          V4      ,          pK+  	  V'       d+   V R,          P                  RR4      pW'R	,           ,          pV# VR,          pV# )
z
Question:
Úquestionr   Ú
z	Options:
z{}. {}
Úcot_contentzA: Let's think step by step.z!Answer: Let's think step by step.z

)Ú	enumerateÚformatÚchoicesr$   )ÚexampleÚincluding_answerÚpromptr1   r   Úir   r3   s   &&      r   Úformat_cot_exampler;   H   s¢   € Ø€FØ�zÕ"€HØ�iÕ €GØ
˜�oÕ€FØ
ˆlÕ€FÜ˜GÖ$‰ˆØ�*×#Ñ#¤G¨A¥J°Ó4Õ4Šñ %çØ˜mÕ,×4Ñ4Ð5SØ5XóZˆà Õ&Õ&ˆð €Mð 	Ð5Õ5ˆØ€Mr   c                 óÐ   € R pVR,          p\        W4      p V RV p VP                  RV4      R,           pV  F  pV\        VRR7      ,          pK  	  V\        VRR7      ,          pV# )z¶The following are multiple choice questions (with answers) about {$}. Think step by step and then finish your answer with 'the answer is (X)' where X is the correct letter choice.


r+   Nz{$}r2   T)r8   F)r/   r$   r;   )r   ÚcurrÚkr9   r-   r7   s   &&&   r   Úgenerate_cot_promptr?   Y   sq   € ð I€FØ�:Õ€GÜ Ó0€FØ�C�aˆ[€FØ�^‰^˜E 7Ó+¨dÕ2€FÛˆØÔ$ W¸tÔDÕDŠñ à
Ô  ¸Ô>Õ>€FØ€Mr   c                 ó¬   € \         P                  V 4      pV'       d   VP                  ^4      # \        P	                  RV ,           4       \        V 4      # )é   z1st answer extract failed
)Ú_ANSWER_IS_REÚsearchÚgroupÚlogÚinfoÚextract_again©ÚtextÚmatchs   & r   Úextract_answerrK   e   s@   € Ü× Ñ  Ó&€EßØ�{‰{˜1‹~Ðä�‰Ð.°Õ5Ô6Ü˜TÓ"Ð"r   c                 ót   € \         P                  V 4      pV'       d   VP                  ^4      # \        V 4      # )rA   )Ú_ANSWER_LINE_RErC   rD   Úextract_finalrH   s   & r   rG   rG   n   s.   € Ü×"Ñ" 4Ó(€EßØ�{‰{˜1‹~Ðä˜TÓ"Ð"r   c                 ób   € \         P                  V 4      pV'       d   VP                  ^ 4      # R# )r   N)Ú_FINAL_ANSWER_RErC   rD   rH   s   & r   rN   rN   v   s&   € Ü×#Ñ# DÓ)€EßØ�{‰{˜1‹~Ðár   c                 óâ  € . p. p\        W#R 7      p\        P                  V4      P                  R4      pV EF  pV! VRRR\        RR7      P                  V P                  4      p	\        P                  ! 4       ;_uu_ 4        V P                  V	R,          VRR\        \        .R	7      p
R
R
R
4       VP                  X
RV	R,          P                  ^,          R
13,          RR7      pV Uu. uF&  qÌP                  \        R4      P                  4       NK(  	  ppV F0  pVP!                  V4       \#        V4      pVP!                  V4       K2  	  EK  	  WT3#   + '       g   i     L¶; iu upi ))Ú
batch_sizezInference Progress:ÚptTÚleft)Úreturn_tensorsÚpaddingÚ
truncationÚ
max_lengthÚpadding_sideÚ	input_idsFç        )rZ   Ú	tokenizerÚ	do_sampleÚtemperatureÚmax_new_tokensÚstop_stringsN:NNN)Úskip_special_tokensÚ )r   rE   ÚpbÚtitleÚmax_model_lengthÚtoÚdeviceÚtorchÚinference_modeÚgenerater_   Ústop_stringÚbatch_decodeÚshaper$   Ústripr   rK   )Úmodelr\   Úinference_batchsrR   Úresponse_batchÚ
pred_batchÚ
dataloaderrc   ÚbatchÚinput_tensorÚoutputsÚgenerated_textsÚgenerated_textÚresultsÚpreds   &&&&           r   Úbatch_inferencer{   ~   sq  € Ø€NØ€JäÐ,ÔD€JÜ	�‰�
Ó	×	!Ñ	!Ð"7Ó	8€BäˆÙ  °tÀTÐVZÔgwð  GMô  N÷  Qñ  Qð  RW÷  R^ñ  R^ó  _ˆÜ×!Ò!×#Õ#Ø—n‘nØ& {Õ3Ø#ØØÜ-Ü)˜]ð %ó ˆG÷ $ð $×0Ñ0Ø�A�| KÕ0×6Ñ6°qÕ9Ñ:Ð:Õ;Ø $ð 1ó 
ˆñ
 ZiÓiÑYhÀ~×)Ñ)¬+°rÓ:×@Ñ@ÖBÑYhˆÐiã%ˆNØ×!Ñ! .Ô1Ü! .Ó1ˆDØ×Ñ˜dÖ#ô &ñ' ð. Ð%Ð%÷+ $×#üò js   Â(EÃ-,E,ÅE)c                 ó  € R R R rCp\        VR4      ;_uu_ 4       pVP                  \        P                  ! V 4      4       RRR4       V  FŒ  pVR,          '       gR   \        P
                  ! ^ \        VR,          4      ^,
          4      pWvR,          8X  d   V^,          pKX  V^,          pKc  VR,          VR,          8X  d   V^,          pKƒ  V^,          pKŽ  	  W4,           ^ 8X  d   R# W3V,           ,          pW#V3#   + '       g   i     LÄ; i)r[   ÚwNrz   r   Úanswer_indexÚanswer)r[   r[   r[   )ÚopenÚwriteÚjsonÚdumpsÚrandomÚrandintÚlen)r.   Úoutput_pathÚaccuÚcorrÚwrongÚfor   Úxs   &&      r   Úsave_resr�   Ÿ   s×   € Ø˜S #�€DÜ	ˆk˜3×	Ô	 2Ø
�‰”—’˜C“Ô!÷ 
 ãˆØ�F�|Œ|Ü—’˜q¤# d¨9¥oÓ"6¸Õ":Ó;ˆAØ˜Õ(Ô(Ø˜•	’à˜•
’Ø�&�\˜T (�^Ô+Ø�A�IŠDà�Q�JŠEñ ð …|�qÔØÐØ˜%•<Õ €DØ�uÐÐ÷! 
 ×	ús   š&C<Ã<D	c           	      óB  € \         P                  R V ,           4       . p\        \        V4      4       F¯  p	Tp
WI,          pRpRpV'       g†   \	        W;V
4      pV! VRR7      pVP                  4        UUu/ uF  w  ppVVP                  4       bK  	  ppp\        VR,          ^ ,          4      pV\        \        ,
          8  d   RpV
^,          p
K�  VP                  V4       K±  	  \        WW‡4      w  pp. p\        V4       F/  w  ppVV,          VR&   VV,          VR&   VP                  V4       K1  	  \        VV4      w  ppp\         P                  R	P                  \        V4      \        V4      \        V4      4      4       \        VV4      w  pppVVV3# u uppi )
zevaluating FNrS   )rU   rZ   Trz   Úmodel_outputsz,this batch accu is: {}, corr: {}, wrong: {}
)rE   rF   Úranger†   r?   ÚitemsÚcudare   r_   r   r{   r4   r�   r5   Ústr)r-   ro   r\   r   r   r‡   ÚntrainrR   Úinference_batchesr:   r>   r=   Úprompt_length_okr9   ÚinputsÚkeyÚvalueÚlengthrr   rq   r.   Újrˆ   r‰   rŠ   s   &&&&&&&&                 r   Úeval_cotrœ   ´   sƒ  € ô ‡H�Hˆ]˜WÕ$Ô%ØÐÜ”3�w“<Ö ˆØˆØ�zˆØ ÐØˆß"Ü(¨°qÓ9ˆFÙ˜v°dÔ;ˆFØ:@¿,¹,¼.ÔI¹.©J¨C°�c˜5Ÿ:™:›<Ò'¹.ˆFÑIÜ˜ Õ,¨QÕ/Ó0ˆFØÔ(¬>Õ9Ô9Ø#'Ð Ø��FŠAØ× Ñ  Ö(ñ !ô "1°ÐCTÓ!aÑ€J�Ø
€CÜ˜WÖ%‰ˆˆ4Ø! !•}ˆˆV‰Ø .¨qÕ 1ˆˆ_ÑØ�
‰
�4Öñ &ô !  kÓ2Ñ€Dˆ$�Ü‡H�HÐ<×CÑCÄCÈÃIÌsÐSWËyÔZ]Ð^cÓZdÓeÔfä   kÓ2Ñ€Dˆ$�Ø��uÐÐùó# Js   Á5Fc                ó’   € V ^8„  d   QhR\         R\        R\        R\        R\        R\        R\        R\        R	\        R
,          /	# )é   ro   r\   r”   r&   Úsave_dirÚglobal_record_filerR   ÚseedÚmax_samplesN)r   r   Úintr“   )r5   s   "r   Ú__annotate__r¤   Õ   sn   € ÷ bñ b”?ð bÜ.ðbäðbô  #ðbô ð	bô
 !$ðbô ðbô ðbô ˜t�ñbr   c	                 ó0  € \         P                  ! V4       VR 8X  d#   \        \        R4      R,          R ,          4      p\        P
                  ! VRR7       \        P                  P                  V P                  P                  4      p	\        P                  P                  V4      '       gT   \        P                  P                  V4      R8X  d0   \        R4      p
V
P                  RRR7       \        W¥,          4      p\        P                  P                  VRP                  \        W“4      4      4      pRP                  \        W“4      4      p\        P                  ! 4       p\        P                   ! R	\        P"                  ! V4      4      pV R
V R2p\        P                  P                  VRV4      p\        P
                  ! \        P                  P                  VR4      RR7       \        P
                  ! VRR7       \        P                  P%                  V4      '       g   \        P
                  ! V4       \'        4       w  pp. pV F+  pVR,          V9  g   K  VP)                  VR,          4       K-  	  VR8X  d   TpM_VP+                  R4      p. pV FF  pV F=  pVP-                  RR
4      VP-                  RR
4      9   g   K,  VP)                  V4       K?  	  KH  	  \.        P1                  RRP                  V4      ,           4       / p\3        V4      p\5        \        P                  P                  V4      R4      ;_uu_ 4       pVP7                  R4       RRR4       V EFF  pVV9  d   RRRRRR/VV&   \9        VV4      pVe   VRV p\9        VV4      p\        P                  P                  VRP;                  V4      4      pVR8X  d   ^p\.        P=                  R4       \?        VWVVVW&4      w  ppp\.        P1                  RV RV RV R V 24       VVV,          R&   VVV,          R&   VVV,          R&   \5        \        P                  P                  V4      R4      ;_uu_ 4       pVP7                  R!P;                  VV,          R,          V4      4       RRR4       EKI  	  RRp!p VPA                  4        F&  w  p"p#V V#R,          ,          p V!V#R,          ,          p!K(  	  V V V!,           R",           ,          p$RV RV!RV$/VR#&   \5        \        P                  P                  V4      R4      ;_uu_ 4       pVP7                  R$4       T$p%VP7                  R%P;                  V%4      4       RRR4       \5        VRRR&7      ;_uu_ 4       p&\B        PD                  ! V&4      p'\        W“4      VX%.,           p(V'PG                  V(4       RRR4       \5        \        P                  P                  V4      R'R(R)7      ;_uu_ 4       p&V&PI                  4       p)RRR4       V)#   + '       g   i     ELÓ; i  + '       g   i     EKæ  ; i  + '       g   i     Lá; i  + '       g   i     Lš; i  + '       g   i     X)# ; i)*ry   ÚlogsÚmmluproT)Úexist_okrb   )Úparentsr¨   Ú/r   z%m-%d_%H-%Mr    z_summary.txtÚsummaryr+   Úallr   r   zselected subjects:
r2   Úaz 
------category level sta------
Nr‰   r[   rŠ   rˆ   z{}.jsonÚautozrMMLU_PRO: batch size `auto` detected. Fixed to static: batch_size = 4. If you oom, reduce this value in mmlupro.pyz	subject: z, acc: z, corr_count: z, wrong_count: zAverage accuracy {:.4f} - {}
g�íµ ÷Æ°>Útotalz
------average acc sta------
zAverage accuracy: {:.4f}
)ÚnewlineÚrzutf-8)Úencoding)%r„   r¡   r“   r   ÚosÚmakedirsÚpathÚbasenameÚconfigÚname_or_pathÚisabsÚdirnameÚmkdirr#   r)   ÚtimeÚstrftimeÚ	localtimeÚexistsr   r   Úsplitr$   rE   rF   Úsortedr€   r�   r/   r5   Úwarnrœ   r‘   ÚcsvÚwriterÚwriterowÚread)*ro   r\   r”   r&   rŸ   r    rR   r¡   r¢   r%   Úlogs_dirÚsave_result_dirÚfile_prefixÚ	timestampÚtime_strÚ	file_nameÚsummary_pathÚfull_test_dfÚfull_val_dfÚall_subjectsr   Úargs_selectedÚsubÚsta_dictÚfr-   r   r   r‡   ÚaccÚ
corr_countÚwrong_countÚ
total_corrÚtotal_wrongr>   ÚvÚ
total_accuÚweighted_accÚfilerÄ   Úrecordr«   s*   &&&&&&&&&                                 r   r§   r§   Õ   s>  € ô ‡K‚K�ÔØ�9ÔÜ”t˜F“| iÕ/°)Õ;Ó<ˆÜ‡K‚K� 4Õ(Ü—‘×!Ñ! %§,¡,×";Ñ";Ó<€JÜ�7‰7�=‰=Ð+×,Ò,´·±·±ÐASÓ1TÐXZÔ1ZÜ˜“<ˆØ�‰˜t¨dˆÔ3Ü  Õ!>Ó?Ðä—g‘g—l‘lØ�#—(‘(Ô-¨jÓLÓMó€Oð —(‘(Ô-¨jÓLÓM€KÜ—	’	“€IÜ�}Š}˜]¬D¯NªN¸9Ó,EÓF€HØ�-˜q  
¨,Ð7€IÜ—7‘7—<‘< ¨)°YÓ?€LÜ‡K‚K”—‘—‘˜X yÓ1¸DÕAÜ‡K‚K�¨$Õ/ä�7‰7�>‰>˜/×*Ò*Ü
�Š�OÔ$Ü -£Ñ€L�+Ø€LÛˆØ�
Õ <Ö/Ø×Ñ  ZÕ 0Ö1ñ ð ˜EÔ!Ø(Ñà)×/Ñ/°Ó4ˆØÐÛˆCÛ%�Ø—<‘<  SÓ)¨S¯[©[¸¸cÓ-BÖBØ%×,Ñ,¨SÖ1ó &ñ  ô
 ‡H�HÐ# d§i¡iÐ0AÓ&BÕBÔCØ€HÜÐ0Ó1Ðä	Œb�g‰g�l‰l˜<Ó(¨#×	.Ô	.°!Ø	�‰Ð4Ô5÷ 
/ô %ˆØ˜(Ô"Ø!'¨¨g°s¸FÀCÐ HˆH�WÑÜ$ \°7Ó;ˆØÒ"Ø˜l˜{Ð+ˆGÜ# K°Ó9ˆÜ—g‘g—l‘l ?°I×4DÑ4DÀWÓ4MÓNˆð ˜ÔØˆJÜ�H‰Hð  Jô  Kä'/°¸È6ÐSZÐ\gÐioÓ'|Ñ$ˆˆZ˜ä�‰�9˜W˜I W¨S¨E°À
¸|È?Ð[fÐZgÐhÔià$.ˆ�Õ˜&Ñ!Ø%0ˆ�Õ˜'Ñ"Ø$'ˆ�Õ˜&Ñ!Ü”"—'‘'—,‘,˜|Ó,¨c×2Ô2°aØ�G‰GÐ4×;Ñ;¸HÀWÕ<MÈfÕ<UÐW^Ó_Ô`÷ 3Ò2ñ+ %ð0 " 3�€JØ—‘Ö ‰ˆˆ1Ø�a˜•iÕˆ
Ø�q˜•zÕ!Šñ !ð ˜z¨KÕ7¸(ÕBÕC€JØ ¨W°kÀ6È:ÐV€HˆWÑä	Œb�g‰g�l‰l˜<Ó(¨#×	.Ô	.°!Ø	�‰Ð1Ô2Ø!ˆØ	�‰Ð,×3Ñ3°LÓAÔB÷ 
/ô
 
Ð  #¨r×	2Õ	2°dÜ—’˜DÓ!ˆÜ# JÓBÀhÐP\ÐE]Õ]ˆØ�‰˜Ô÷ 
3ô
 
Œb�g‰g�l‰l˜<Ó(¨#¸×	@Õ	@ÀDØ—)‘)“+ˆ÷ 
Að €N÷_ 
/×	.Ð	.ú÷0 3×2Ñ2ú÷ 
/×	.ú÷
 
3×	2ú÷
 
AÖ	@ð €Nús<   ÍX5Ò0Y	Õ4YÖ#<Y1ØZØ5Y	Ù	YÙY.	Ù1Z	ÚZ	)ÚAÚBÚCÚDÚEÚFÚGÚHÚIÚJÚKÚLÚMÚNÚOÚP)T)é   r¬   ry   zeval_record_collection.csvrA   i90  N)+rÃ   r‚   r³   r„   r¼   Úpathlibr   Úpcrerh   Údatasetsr   Útorch.utils.datar   Útransformersr   r   Úutils.loggerr   r6   re   r_   rk   rE   ÚcompilerB   rM   ÚFlagÚDOTALLrP   r   r   r)   r/   r;   r?   rK   rG   rN   r{   r�   ri   rœ   r§   © r   r   Ú<module>rú      sé   ðó Û Û 	Û Û Ý ã Û Ý !Ý 'ß Aå 'ò [€ØÐ Ø€Ø€áƒn€Ø—’Ð7Ó8€Ø—,’,Ð8Ó9€Ø—<’<ØØ
�)‰)×
Ñ
ôÐ ò
ò
ò2òôò"	ò#ò#òò&òBð* ×ÒÓñó ð÷@bñ br   