+
    œ°³jõs  ã                   óæ   € R t ^ RIt^ RIt^ RIt^ RIt^ RIt^RIHt ^RIH	t	 ]	P                  ! ]4      tR tR tR tR tR tR	 tRR
 ltR tR tR tR tR tRR ltRR ltR tR tR tR tR# )a·  
Very heavily inspired by the official evaluation script for SQuAD version 2.0 which was modified by XLNet authors to
update `find_best_threshold` scripts for SQuAD V2.0

In addition to basic functionality, we also compute additional statistics and plot precision-recall curves if an
additional na_prob.json file is provided. This file is expected to map question ID's to the model's predicted
probability that a question is unanswerable.
N)ÚBasicTokenizer)Úloggingc           	     óN   € R pR pR pR pV! V! V! V! V 4      4      4      4      # )zALower text and remove punctuation, articles and extra whitespace.c                 ó|   € \         P                  ! R \         P                  4      p\         P                  ! VRV 4      # )z\b(a|an|the)\bÚ )ÚreÚcompileÚUNICODEÚsub)ÚtextÚregexs   & Úh/home/luca/screen-checking/.venv/lib/python3.14/site-packages/transformers/data/metrics/squad_metrics.pyÚremove_articlesÚ)normalize_answer.<locals>.remove_articles'   s*   € Ü—
’
Ð,¬b¯j©jÓ9ˆÜ�vŠv�e˜S $Ó'Ð'ó    c                 ó@   € R P                  V P                  4       4      # )r   )ÚjoinÚsplit©r   s   &r   Úwhite_space_fixÚ)normalize_answer.<locals>.white_space_fix+   s   € Ø�x‰x˜Ÿ
™
›Ó%Ð%r   c                 ól   a€ \        \        P                  4      oR P                  V3R lV  4       4      # )Ú c              3   ó8   <"  € T F  qS9  g   K  Vx € K  	  R # 5i©N© )Ú.0ÚchÚexcludes   & €r   Ú	<genexpr>Ú8normalize_answer.<locals>.remove_punc.<locals>.<genexpr>0   s   øé € Ð>¡D˜b°gÑ,=—r’r£Dùs   ƒ�
)ÚsetÚstringÚpunctuationr   )r   r   s   &@r   Úremove_puncÚ%normalize_answer.<locals>.remove_punc.   s(   ø€ Ü”f×(Ñ(Ó)ˆØ�w‰wÔ>¡DÓ>Ó>Ð>r   c                 ó"   € V P                  4       # r   )Úlowerr   s   &r   r'   Únormalize_answer.<locals>.lower2   s   € Ø�z‰z‹|Ðr   r   )Úsr   r   r$   r'   s   &    r   Únormalize_answerr*   $   s1   € ò(ò&ò?òñ ™?©;±u¸Q³xÓ+@ÓAÓBÐBr   c                 óH   € V '       g   . # \        V 4      P                  4       # r   )r*   r   )r)   s   &r   Ú
get_tokensr,   8   s   € ßØˆ	Ü˜AÓ×$Ñ$Ó&Ð&r   c                 óB   € \        \        V 4      \        V4      8H  4      # r   )Úintr*   )Úa_goldÚa_preds   &&r   Úcompute_exactr1   >   s   € ÜÔ Ó'Ô+;¸FÓ+CÑCÓDÐDr   c                 óÐ  € \        V 4      p\        V4      p\        P                  ! V4      \        P                  ! V4      ,          p\        VP	                  4       4      p\        V4      ^ 8X  g   \        V4      ^ 8X  d   \        W#8H  4      # V^ 8X  d   ^ # RV,          \        V4      ,          pRV,          \        V4      ,          p^V,          V,          Wg,           ,          pV# )é    ç      ð?)r,   ÚcollectionsÚCounterÚsumÚvaluesÚlenr.   )	r/   r0   Ú	gold_toksÚ	pred_toksÚcommonÚnum_sameÚ	precisionÚrecallÚf1s	   &&       r   Ú
compute_f1rA   B   s´   € Ü˜6Ó"€IÜ˜6Ó"€IÜ× Ò  Ó+¬k×.AÒ.AÀ)Ó.LÕL€FÜ�6—=‘=“?Ó#€HÜ
ˆ9ƒ~˜Ôœc )›n°Ô1ä�9Ñ)Ó*Ð*Ø�1„}ÙØ�h•¤ Y£Õ/€IØ�8�^œc )›nÕ,€FØ
ˆi�-˜&Õ
  YÕ%7Õ	8€BØ€Ir   c                ól  a€ / p/ pV  F¢  pVP                   pVP                   Uu. uF&  p\        VR,          4      '       g   K  VR,          NK(  	  ppV'       g   R.pWQ9  d   \        RV 24       Kl  W,          o\	        V3R lV 4       4      W%&   \	        V3R lV 4       4      W5&   K¤  	  W#3# u upi )zN
Computes the exact and f1 scores from the examples and the model predictions
r   r   zMissing prediction for c              3   ó<   <"  € T F  p\        VS4      x € K  	  R # 5ir   )r1   ©r   ÚaÚ
predictions   & €r   r   Ú!get_raw_scores.<locals>.<genexpr>f   s   øé € Ð"VÉÀA¤=°°J×#?Ð#?Ëùó   ƒc              3   ó<   <"  € T F  p\        VS4      x € K  	  R # 5ir   )rA   rD   s   & €r   r   rG   g   s   øé € ÐPÁ<¸a¤
¨1¨j× 9Ð 9Ã<ùrH   )Úqas_idÚanswersr*   ÚprintÚmax)	ÚexamplesÚpredsÚexact_scoresÚ	f1_scoresÚexamplerJ   ÚanswerÚgold_answersrF   s	   &&      @r   Úget_raw_scoresrU   R   s²   ø€ ð €LØ€IãˆØ—‘ˆØ5<·_²_Ói±_¨6ÔHXÐY_Ð`fÕYg×Hhœ˜˜vŸ˜±_ˆÐiçà˜4ˆLàÔÜÐ+¨F¨8Ð4Ô5Ùà•]ˆ
Ü"Ô"VÉÓ"VÓVˆÑÜÔPÁ<ÓPÓPˆ	Óñ ð  Ð"Ð"ùò js   ¥B1ÁB1c                 óž   € / pV P                  4        F6  w  rVW,          V8„  pV'       d   \        W%,          '       * 4      WE&   K2  WdV&   K8  	  V# r   )ÚitemsÚfloat)ÚscoresÚna_probsÚqid_to_has_ansÚna_prob_threshÚ
new_scoresÚqidr)   Úpred_nas   &&&&    r   Úapply_no_ans_thresholdr`   l   sI   € Ø€JØ—,‘,–.‰ˆØ•- .Ñ0ˆßÜ#¨Õ(;Ô$;Ó<ˆJ‹Oà�s‹Oñ !ð Ðr   c           	      óÖ  a a€ V'       gu   \        S 4      p\        P                  ! R R\        S P	                  4       4      ,          V,          3RR\        SP	                  4       4      ,          V,          3RV3.4      # \        V4      p\        P                  ! R R\        V 3R lV 4       4      ,          V,          3RR\        V3R lV 4       4      ,          V,          3RV3.4      # )Úexactç      Y@r@   Útotalc              3   ó6   <"  € T F  pSV,          x € K  	  R # 5ir   r   )r   ÚkrP   s   & €r   r   Ú!make_eval_dict.<locals>.<genexpr>…   s   øé € Ð%H¹x¸! l°1§o¢o»xùó   ƒc              3   ó6   <"  € T F  pSV,          x € K  	  R # 5ir   r   )r   rf   rQ   s   & €r   r   rg   †   s   øé € Ð"B¹°A 9¨Q§<¢<»ùrh   )r9   r5   ÚOrderedDictr7   r8   )rP   rQ   Úqid_listrd   s   ff& r   Úmake_eval_dictrl   w   sÕ   ù€ ßÜ�LÓ!ˆÜ×&Ò&à˜%¤# l×&9Ñ&9Ó&;Ó"<Õ<¸uÕDÐEØ�uœs 9×#3Ñ#3Ó#5Ó6Õ6¸Õ>Ð?Ø˜%Ð ðó
ð 	
ô �H“ˆÜ×&Ò&à˜%¤#Ô%H¹xÓ%HÓ"HÕHÈ5ÕPÐQØ�uœsÔ"B¹Ó"BÓBÕBÀUÕJÐKØ˜%Ð ðó
ð 	
r   c                 ó6   € V F  pW,          W R V 2&   K  	  R# )Ú_Nr   )Ú	main_evalÚnew_evalÚprefixrf   s   &&& r   Ú
merge_evalrr   Œ   s    € ÛˆØ%-¥[ˆ	�H˜A˜a˜S�/Ó"ó r   c                 óÚ  aa€ \        V3R  lS 4       4      pTpTpRp\        SV3R lR7      pV FQ  p	W‘9  d   K  SV	,          '       d
   W,          p
MW	,          '       d   Rp
M^ p
WZ,          pWV8”  g   KF  TpSV	,          pKS  	  ^ ^ rËV F4  p	SV	,          '       g   K  V^,          pW‘9  d   K%  W±V	,          ,          pK6  	  RV,          \        V4      ,          VRV,          V,          3# )c              3   óJ   <"  € T F  pSV,          '       d   K  ^x € K  	  R# 5i©é   Nr   ©r   rf   r[   s   & €r   r   Ú&find_best_thresh_v2.<locals>.<genexpr>’   ó   øé € ÐH¡˜1°nÀQ×6GÑ6G—Q’Q£ùó   ƒ#™
#ç        c                 ó   <€ SV ,          # r   r   ©rf   rZ   s   &€r   Ú<lambda>Ú%find_best_thresh_v2.<locals>.<lambda>–   ó	   ø€ ¨h°q®kr   ©Úkeyrc   r4   éÿÿÿÿ)r7   Úsortedr9   )rO   rY   rZ   r[   Ú
num_no_ansÚ	cur_scoreÚ
best_scoreÚbest_threshrk   r^   ÚdiffÚhas_ans_scoreÚhas_ans_cnts   &&ff         r   Úfind_best_thresh_v2rŒ   ‘   sé   ù€ ÜÔH¡ÓHÓH€JØ€IØ€JØ€KÜ�hÔ$9Ô:€HÛˆØÔÙØ˜#×ÔØ•;‰Dà�zŒzØ‘à�ØÕˆ	ØÖ!Ø"ˆJØ" 3�-ŠKñ ð "# A�;ÛˆØ˜c×"Ô"ÙØ�qÕˆàÔÙØ �Õ$Šñ ð �:Õ¤ F£Õ+¨[¸#ÀÕ:MÐP[Õ:[Ð[Ð[r   c                 ór   € \        WWE4      w  rgp\        WWE4      w  ršpW`R &   WpR&   W�R&   W R&   W€R&   W°R&   R# )Ú
best_exactÚbest_exact_threshÚbest_f1Úbest_f1_threshÚhas_ans_exactÚ
has_ans_f1N)rŒ   )ro   rO   Ú	exact_rawÚf1_rawrZ   r[   rŽ   Úexact_threshr’   r�   Ú	f1_threshr“   s   &&&&&&      r   Úfind_all_best_thresh_v2r˜   ³   sW   € Ü.AÀ%ÐT\Ó.mÑ+€J˜mÜ%8¸ÈÓ%aÑ"€G˜
Ø(ˆlÑØ%1Ð!Ñ"Ø"ˆiÑØ"+ÐÑØ!.ˆoÑØ(ˆlÓr   c                 óX  aa€ \        V3R  lS 4       4      pTpTpRp\        SV3R lR7      p\        V4       FS  w  ršW¡9  d   K  SV
,          '       d
   W,          pMW
,          '       d   RpM^ pW[,          pWV8”  g   KH  TpSV
,          pKU  	  RV,          \        V4      ,          V3# )c              3   óJ   <"  € T F  pSV,          '       d   K  ^x € K  	  R# 5iru   r   rw   s   & €r   r   Ú#find_best_thresh.<locals>.<genexpr>¿   ry   rz   r{   c                 ó   <€ SV ,          # r   r   r}   s   &€r   r~   Ú"find_best_thresh.<locals>.<lambda>Ã   r€   r   r�   rc   rƒ   )r7   r„   Ú	enumerater9   )rO   rY   rZ   r[   r…   r†   r‡   rˆ   rk   rn   r^   r‰   s   &&ff        r   Úfind_best_threshrŸ   ¾   s£   ù€ ÜÔH¡ÓHÓH€JØ€IØ€JØ€KÜ�hÔ$9Ô:€HÜ˜HÖ%‰ˆØÔÙØ˜#×ÔØ•;‰Dà�zŒzØ‘à�ØÕˆ	ØÖ!Ø"ˆJØ" 3�-ŠKñ &ð �:Õ¤ F£Õ+¨[Ð8Ð8r   c                 ó^   € \        WWE4      w  rg\        WWE4      w  r‰W`R &   WpR&   W€R&   W�R&   R# )rŽ   r�   r�   r‘   N)rŸ   )
ro   rO   r”   r•   rZ   r[   rŽ   r–   r�   r—   s
   &&&&&&    r   Úfind_all_best_threshr¡   Õ   sB   € Ü/°À(Ó[Ñ€JÜ)¨%¸ÓRÑ€Gà(ˆlÑØ%1Ð!Ñ"Ø"ˆiÑØ"+ÐÓr   c                 ó|  € V  Uu/ uF"  qDP                   \        VP                  4      bK$  	  ppVP                  4        UUu. uF  w  rgV'       g   K  VNK  	  pppVP                  4        UUu. uF  w  rgV'       d   K  VNK  	  p	ppVf   \        P                  VR4      p\        W4      w  r«\        W¢WS4      p\        W²WS4      p\        WÍ4      pV'       d   \        WÍVR7      p\        WïR4       V	'       d   \        WÍV	R7      p\        VVR4       V'       d   \        WáW«W%4       V# u upi u uppi u uppi )Nr{   )rk   ÚHasAnsÚNoAns)rJ   ÚboolrK   rW   ÚdictÚfromkeysrU   r`   rl   rr   r¡   )rN   rO   Úno_answer_probsÚno_answer_probability_thresholdrR   Úqas_id_to_has_answerrJ   Ú
has_answerÚhas_answer_qidsÚno_answer_qidsrb   r@   Úexact_thresholdÚf1_thresholdÚ
evaluationÚhas_ans_evalÚno_ans_evals   &&&&             r   Úsquad_evaluater³   ß   s  € ÙQYÓZÑQYÀgŸN™N¬D°·±Ó,AÒAÑQYÐÐZØ8L×8RÑ8RÔ8TÔcÑ8TÑ"4 &ÔXb—v�vÑ8T€OÑcØ7K×7QÑ7QÔ7SÔfÑ7SÑ!3 Ô[e—f�fÑ7S€NÑfàÒÜŸ-™-¨¨sÓ3ˆä˜xÓ/�I€Eä,ØÐ 4ó€Oô *¨"Ð?SÓu€Lä Ó>€JçÜ% oÈoÔ^ˆÜ�:¨XÔ6çÜ$ _È^Ô\ˆÜ�:˜{¨GÔ4çÜ˜Z°¸?ÔaàÐùò7 [ùÛcùÛfs   …(D-ÁD2ÁD2Á/D8Â D8c                ó  € R p\        VR7      pRP                  VP                  V4      4      pVP                  V 4      pVR8X  d'   V'       d   \        P                  RV  RV R24       V# V\        V 4      ,           ^,
          pV! V4      w  ršV! V4      w  r¼\        V	4      \        V4      8w  d'   V'       d   \        P                  RV	 RV R24       V# / pVP                  4        F	  w  rïWíV&   K  	  R	pW}9   d   W×,          pVV
9   d
   V
V,          pVf    V'       d   \        P                  R
4       V# R	pW�9   d   WØ,          pVV
9   d
   V
V,          pVf    V'       d   \        P                  R4       V# VVV^,            pV# )z;Project the tokenized prediction back to the original text.c                 óÎ   € . p\         P                  ! 4       p\        V 4       F,  w  r4VR 8X  d   K  W2\        V4      &   VP	                  V4       K.  	  RP                  V4      pWR3# )r   r   )r5   rj   rž   r9   Úappendr   )r   Úns_charsÚns_to_s_mapÚiÚcÚns_texts   &     r   Ú_strip_spacesÚ%get_final_text.<locals>._strip_spaces  s`   € ØˆÜ!×-Ò-Ó/ˆÜ˜d–O‰DˆAØ�CŒxÙØ)*œ˜H›Ñ&Ø�O‰O˜AÖñ	 $ð
 —'‘'˜(Ó#ˆØÐ%Ð%r   )Údo_lower_caser   zUnable to find text: 'z' in 'Ú'z*Length not equal after stripping spaces: 'z' vs 'NzCouldn't map start positionzCouldn't map end positionrƒ   )r   r   ÚtokenizeÚfindÚloggerÚinfor9   rW   )Ú	pred_textÚ	orig_textr¾   Úverbose_loggingr¼   Ú	tokenizerÚtok_textÚstart_positionÚend_positionÚorig_ns_textÚorig_ns_to_s_mapÚtok_ns_textÚtok_ns_to_s_mapÚtok_s_to_ns_mapr¹   Ú	tok_indexÚorig_start_positionÚns_start_positionÚorig_end_positionÚns_end_positionÚoutput_texts   &&&&                 r   Úget_final_textrÖ   þ   s˜  € ò8	&ô ¨]Ô;€Ià�x‰x˜	×*Ñ*¨9Ó5Ó6€Hà—]‘] 9Ó-€NØ˜ÔßÜ�K‰KÐ0°°¸6À)ÀÈAÐNÔOØÐØ!¤C¨	£NÕ2°QÕ6€Lá'4°YÓ'?Ñ$€\Ù%2°8Ó%<Ñ"€[ä
ˆ<ÓœC Ó,Ô,ßÜ�K‰KÐDÀ\ÀNÐRXÐYdÐXeÐefÐgÔhØÐð €OØ'×-Ñ-Ö/‰ˆØ%&˜	Ó"ñ 0ð ÐØÔ(Ø+Õ;ÐØÐ 0Ô0Ø"2Ð3DÕ"EÐàÒ"ßÜ�K‰KÐ5Ô6ØÐàÐØÔ&Ø)Õ7ˆØÐ.Ô.Ø 0°Õ AÐàÒ ßÜ�K‰KÐ3Ô4ØÐàÐ/Ð3DÀqÕ3HÐJ€KØÐr   c                ó¾   € \        \        V 4      R RR7      p. p\        \        V4      4       F*  pWA8¼  d    V# VP	                  W$,          ^ ,          4       K,  	  V# )z"Get the n-best logits from a list.c                 ó   € V ^,          # )rv   r   ©Úxs   &r   r~   Ú#_get_best_indexes.<locals>.<lambda>^  s   € ¸aÀ¾dr   T©r‚   Úreverse)r„   rž   Úranger9   r¶   )ÚlogitsÚn_best_sizeÚindex_and_scoreÚbest_indexesr¹   s   &&   r   Ú_get_best_indexesrã   \  s]   € äœY vÓ.±NÈDÔQ€Oà€LÜ”3�Ó'Ö(ˆØÔØàÐð 	×Ñ˜OÕ.¨qÕ1Ö2ñ )ð Ðr   c                ó  € V '       g   . # RpV  F  pVe	   W!8”  g   K  TpK  	  . pRpV  F8  p\         P                  ! W!,
          4      pVP                  V4       WE,          pK:  	  . pV F  pVP                  W$,          4       K  	  V# )z,Compute softmax probability over raw logits.Nr{   )ÚmathÚexpr¶   )rY   Ú	max_scoreÚscoreÚ
exp_scoresÚ	total_sumrÚ   Úprobss   &      r   Ú_compute_softmaxrì   h  sŠ   € çØˆ	à€IÛˆØÒ Ö 1ØŠIñ ð €JØ€IÛˆÜ�HŠH�UÕ&Ó'ˆØ×Ñ˜!ÔØ�Š	ñ ð
 €EÛˆØ�‰�UÕ&Ö'ñ à€Lr   c                ó  € V'       d   \         P                  RV 24       V'       d   \         P                  RV 24       V'       d!   V
'       d   \         P                  RV 24       \        P                  ! \        4      pV F$  pWÞP
                  ,          P                  V4       K&  	  / pV F  pVVVP                  &   K  	  \        P                  ! R. R	O4      p\        P                  ! 4       p\        P                  ! 4       p\        P                  ! 4       p\        V 4       EF”  w  ppVV,          p. pRp^ p^ p^ p\        V4       EF�  w  ppWþP                  ,          p\        VP                  V4      p\        VP                  V4      pV
'       d]   VP                  ^ ,          VP                  ^ ,          ,           p V V8  d+   T pTpVP                  ^ ,          pVP                  ^ ,          pV Fî  p!V Få  p"V!\        VP                  4      8¼  d   K  V"\        VP                  4      8¼  d   K;  V!VP                   9  d   KN  V"VP                   9  d   Ka  VP"                  P%                  V!R4      '       g   K…  V"V!8  d   KŽ  V"V!,
          ^,           p#V#V8”  d   K§  VP                  V! VV!V"VP                  V!,          VP                  V",          R	7      4       Kç  	  Kð  	  EK   	  V
'       d   VP                  V! V^ ^ VVR	7      4       \'        VR
 RR7      p\        P                  ! R. RO4      p$/ p%. p&V EFX  p'\        V&4      V8¼  d    EMFVV'P(                  ,          pV'P*                  ^ 8”  dé   VP                  V'P*                  V'P,                  ^,            p(VP                   V'P*                  ,          p)VP                   V'P,                  ,          p*VP.                  V)V*^,            p+VP1                  V(4      p,V,P3                  4       p,RP5                  V,P7                  4       4      p,RP5                  V+4      p-\9        V,V-WY4      p.V.V%9   d   EK  RV%V.&   MRp.RV%V.&   V&P                  V$! V.V'P:                  V'P<                  R7      4       EK[  	  V
'       dM   RV%9  d   V&P                  V$! RVVR7      4       \        V&4      ^8X  d   V&P?                  ^ V$! RRRR7      4       V&'       g   V&P                  V$! RRRR7      4       \        V&4      ^8  d   \A        R4      h. p/Rp0V& FO  p1V/P                  V1P:                  V1P<                  ,           4       V0'       d   K9  V1PB                  '       g   KM  T1p0KQ  	  \E        V/4      p2. p3\        V&4       Fe  w  p4p1\        P                  ! 4       p5V1PB                  V5R&   V2V4,          V5R&   V1P:                  V5R&   V1P<                  V5R&   V3P                  V54       Kg  	  \        V34      ^8  d   \A        R4      hV
'       g   V3^ ,          R,          VVPF                  &   McVV0P:                  ,
          V0P<                  ,
          p6V6VVPF                  &   V6V8”  d   RVVPF                  &   MV0PB                  VVPF                  &   V3VVPF                  &   EK—  	  V'       dL   \I        VR4      ;_uu_ 4       p7V7PK                  \L        PN                  ! V^R7      R,           4       RRR4       V'       dL   \I        VR4      ;_uu_ 4       p7V7PK                  \L        PN                  ! V^R7      R,           4       RRR4       V'       dV   V
'       dN   \I        VR4      ;_uu_ 4       p7V7PK                  \L        PN                  ! V^R7      R,           4       RRR4       V# V#   + '       g   i     LÂ; i  + '       g   i     L‚; i  + '       g   i     T# ; i)zHWrite final predictions to the json file and log-odds of null if needed.úWriting predictions to: zWriting nbest to: zWriting null_log_odds to: ÚPrelimPredictionÚstart_logitÚ	end_logité@B F)Úfeature_indexÚstart_indexÚ	end_indexrð   rñ   c                 ó<   € V P                   V P                  ,           # r   )rð   rñ   rÙ   s   &r   r~   Ú,compute_predictions_logits.<locals>.<lambda>â  s   € ÀqÇ}Á}ÐWX×WbÑWbÖGbr   TrÜ   ÚNbestPredictionr   r   r   )r   rð   rñ   Úemptyr{   úNo valid predictionsNÚprobabilityÚw©ÚindentÚ
)(rÂ   rÃ   r5   ÚdefaultdictÚlistÚexample_indexr¶   Ú	unique_idÚ
namedtuplerj   rž   rã   Ústart_logitsÚ
end_logitsr9   ÚtokensÚtoken_to_orig_mapÚtoken_is_max_contextÚgetr„   ró   rô   rõ   Ú
doc_tokensÚconvert_tokens_to_stringÚstripr   r   rÖ   rð   rñ   ÚinsertÚ
ValueErrorr   rì   rJ   ÚopenÚwriteÚjsonÚdumps)8Úall_examplesÚall_featuresÚall_resultsrà   Úmax_answer_lengthr¾   Úoutput_prediction_fileÚoutput_nbest_fileÚoutput_null_log_odds_filerÆ   Úversion_2_with_negativeÚnull_score_diff_thresholdrÇ   Úexample_index_to_featuresÚfeatureÚunique_id_to_resultÚresultÚ_PrelimPredictionÚall_predictionsÚall_nbest_jsonÚscores_diff_jsonr  rR   ÚfeaturesÚprelim_predictionsÚ
score_nullÚmin_null_feature_indexÚnull_start_logitÚnull_end_logitró   Ústart_indexesÚend_indexesÚfeature_null_scorerô   rõ   ÚlengthÚ_NbestPredictionÚseen_predictionsÚnbestÚpredÚ
tok_tokensÚorig_doc_startÚorig_doc_endÚorig_tokensrÈ   rÅ   Ú
final_textÚtotal_scoresÚbest_non_null_entryÚentryrë   Ú
nbest_jsonr¹   ÚoutputÚ
score_diffÚwriters8   &&&&&&&&&&&&&                                           r   Úcompute_predictions_logitsr?    sì  € ÷  Ü�‰Ð.Ð/EÐ.FÐGÔHßÜ�‰Ð(Ð):Ð(;Ð<Ô=ß ×%<Ü�‰Ð0Ð1JÐ0KÐLÔMä +× 7Ò 7¼Ó =ÐÛˆØ!×"7Ñ"7Õ8×?Ñ?ÀÖHñ  ð ÐÛˆØ06Ð˜F×,Ñ,Ó-ñ ô $×.Ò.ØÒeóÐô "×-Ò-Ó/€OÜ ×,Ò,Ó.€NÜ"×.Ò.Ó0Ðä"+¨L×"9Ñˆ�wØ,¨]Õ;ˆàÐàˆ
Ø!"ÐØÐØˆÜ&/°×&9Ñ"ˆM˜7Ø(×):Ñ):Õ;ˆFÜ-¨f×.AÑ.AÀ;ÓOˆMÜ+¨F×,=Ñ,=¸{ÓKˆKç&Ø%+×%8Ñ%8¸Õ%;¸f×>OÑ>OÐPQÕ>RÕ%RÐ"Ø%¨
Ô2Ø!3�JØ-:Ð*Ø'-×':Ñ':¸1Õ'=Ð$Ø%+×%6Ñ%6°qÕ%9�NÛ,�Û!,�Ið #¤c¨'¯.©.Ó&9Ô9Ù Ø ¤C¨¯©Ó$7Ô7Ù Ø"¨'×*CÑ*CÔCÙ Ø ¨×(AÑ(AÔAÙ Ø"×7Ñ7×;Ñ;¸KÈ×OÒOÙ Ø  ;Ô.Ù Ø&¨Õ4°qÕ8�FØÐ 1Ô1Ù Ø&×-Ñ-Ù)Ø*7Ø(3Ø&/Ø(.×(;Ñ(;¸KÕ(HØ&,×&7Ñ&7¸	Õ&Bôöó' "-ô  -ñ ':÷R #Ø×%Ñ%Ù!Ø"8Ø !ØØ 0Ø,ôôô $Ð$6Ñ<cÐmqÔrÐä&×1Ò1ØÒCó
Ðð ÐØˆÜ&ˆDÜ�5‹z˜[Ô(ÚØ˜t×1Ñ1Õ2ˆGØ×Ñ !Ô#Ø$Ÿ^™^¨D×,<Ñ,<ÀÇÁÐQRÕ@RÐT�
Ø!(×!:Ñ!:¸4×;KÑ;KÕ!L�Ø&×8Ñ8¸¿¹ÕH�Ø%×0Ñ0°À<ÐRSÕCSÐU�à$×=Ñ=¸jÓI�ð $Ÿ>™>Ó+�ØŸ8™8 H§N¡NÓ$4Ó5�ØŸH™H [Ó1�	ä+¨H°iÀÓ`�
ØÐ!1Ô1Úà/3Ð  Ò,à�
Ø/3Ð  Ñ,à�L‰LÑ)¨zÀt×GWÑGWÐcg×cqÑcqÔr×sñA '÷D #ØÐ)Ô)Ø—‘Ñ-°2ÐCSÐ_mÔnÔoô �5‹z˜QŒØ—‘˜QÑ 0°gÈ3ÐZ]Ô ^Ô_÷ Ø�L‰LÑ)¨wÀCÐSVÔWÔXäˆu‹:˜Œ>ÜÐ3Ó4Ð4àˆØ"ÐÛˆEØ×Ñ × 1Ñ 1°E·O±OÕ CÔDß&Ñ&Ø—:—:’:Ø*/Ò'ñ	 ô ! Ó.ˆàˆ
Ü! %Ö(‰HˆAˆuÜ ×,Ò,Ó.ˆFØ"ŸZ™ZˆF�6‰NØ$)¨!¥HˆF�=Ñ!Ø$)×$5Ñ$5ˆF�=Ñ!Ø"'§/¡/ˆF�;ÑØ×Ñ˜fÖ%ñ )ô ˆz‹?˜QÔÜÐ3Ó4Ð4ç&Ø.8¸­m¸FÕ.CˆO˜GŸN™NÒ+ð $Ð&9×&EÑ&EÕEÐI\×IfÑIfÕgˆJØ/9Ð˜WŸ^™^Ñ,ØÐ5Ô5Ø24� §¡Ò/à2E×2JÑ2J� §¡Ñ/Ø)3ˆ�w—~‘~Ô&ño #:÷r ÜÐ(¨#×.Ô.°&Ø�L‰LœŸš O¸AÔ>ÀÕEÔF÷ /÷ ÜÐ# S×)Ô)¨VØ�L‰LœŸš N¸1Ô=ÀÕDÔE÷ *÷ !×%<ÜÐ+¨S×1Ô1°VØ�L‰LœŸšÐ$4¸QÔ?À$ÕFÔG÷ 2ð Ðˆ?Ð÷ /×.ú÷ *×)ú÷ 2Ö1ð Ðús$   Û/_Ü2/_Þ/_.ß_	ß_+	ß._?	c                ó¼  € \         P                  ! R. R	O4      p\         P                  ! R. RO4      p\        P                  RV 24       \         P                  ! \
        4      pV F%  pVVP                  ,          P                  V4       K'  	  / pV F  pVVVP                  &   K  	  \         P                  ! 4       p\         P                  ! 4       p\         P                  ! 4       p\        V 4       EF>  w  ppVV,          p. pRp\        V4       EFJ  w  ppVVP                  ,          pVP                  p\        VV4      p\        V4       EF  p\        V	4       Fõ  pVP                  V,          pVP                  V,          p VV	,          V,           p!VP                   V!,          p"VP"                  V!,          p#V VP$                  ^,
          8¼  d   Ky  V#VP$                  ^,
          8¼  d   K“  VP&                  P)                  V R4      '       g   K·  V#V 8  d   KÀ  V#V ,
          ^,           p$V$V8”  d   KÙ  VP                  V! VV V#VV"R	7      4       K÷  	  EK  	  EKM  	  \+        VR
 RR7      p/ p%. p&V EFk  p'\-        V&4      V8¼  d    EMYVV'P.                  ,          pVP0                  V'P2                  V'P4                  ^,            p(VP6                  V'P2                  ,          p)VP6                  V'P4                  ,          p*VP8                  V)V*^,            p+VP;                  V(4      p,V,P=                  4       p,RP?                  V,PA                  4       4      p,RP?                  V+4      p-\C        VR4      '       d   VPD                  p.MVPF                  p.\I        V,V-V.V4      p/V/V%9   d   EK8  RV%V/&   V&P                  V! V/V'PJ                  V'PL                  R7      4       EKn  	  V&'       g   V&P                  V! RRRR7      4       . p0Rp1V& F;  p2V0P                  V2PJ                  V2PL                  ,           4       V1'       d   K9  T2p1K=  	  \O        V04      p3. p4\        V&4       Fe  w  pp2\         P                  ! 4       p5V2PP                  V5R&   V3V,          V5R&   V2PJ                  V5R&   V2PL                  V5R&   V4P                  V54       Kg  	  \-        V44      ^8  d   \S        R4      hV1f   \S        R4      hTp6V6VVPT                  &   V1PP                  VVPT                  &   V4VVPT                  &   EKA  	  \W        VR4      ;_uu_ 4       p7V7PY                  \Z        P\                  ! V^R7      R,           4       RRR4       \W        VR4      ;_uu_ 4       p7V7PY                  \Z        P\                  ! V^R7      R,           4       RRR4       V
'       dN   \W        VR4      ;_uu_ 4       p7V7PY                  \Z        P\                  ! V^R7      R,           4       RRR4       V# V#   + '       g   i     L²; i  + '       g   i     Lz; i  + '       g   i     T# ; i)z¥
XLNet write prediction logic (more complex than Bert's). Write final predictions to the json file and log-odds of
null if needed.

Requires utils_squad_evaluate.py
rï   Ústart_log_probÚend_log_probrø   r   rî   rò   F)ró   rô   rõ   rA  rB  c                 ó<   € V P                   V P                  ,           # r   )rA  rB  rÙ   s   &r   r~   Ú/compute_predictions_log_probs.<locals>.<lambda>­  s   € ¨q×/?Ñ/?À!Ç.Á.Ö/Pr   TrÜ   r   r¾   )r   rA  rB  r   Nrû   rú   rü   rý   rÿ   g    €„.Á)/r5   r  rÂ   rÃ   r   r  r  r¶   r  rj   rž   Ú
cls_logitsÚminrÞ   r  Ústart_top_indexr  Úend_top_indexÚparagraph_lenr	  r
  r„   r9   ró   r  rô   rõ   r  r  r  r  r   r   Úhasattrr¾   Údo_lowercase_and_remove_accentrÖ   rA  rB  rì   r   r  rJ   r  r  r  r  )8r  r  r  rà   r  r  r  r  Ústart_n_topÚ	end_n_topr  rÇ   rÆ   r!  r/  r  r  r  r   r"  r#  r$  r  rR   r%  r&  r'  ró   Úcur_null_scorer¹   ÚjrA  rô   Új_indexrB  rõ   r.  r0  r1  r2  r3  r4  r5  r6  rÈ   rÅ   r¾   r7  r8  r9  r:  rë   r;  r<  r=  r>  s8   &&&&&&&&&&&&&                                           r   Úcompute_predictions_log_probsrQ  N  s£  € ô* $×.Ò.ØÒkóÐô #×-Ò-ØÒEóÐô ‡K�KÐ*Ð+AÐ*BÐCÔDä +× 7Ò 7¼Ó =ÐÛˆØ! '×"7Ñ"7Õ8×?Ñ?ÀÖHñ  ð ÐÛˆØ06Ð˜F×,Ñ,Ó-ñ ô "×-Ò-Ó/€OÜ ×,Ò,Ó.€NÜ"×.Ò.Ó0Ðä"+¨L×"9Ñˆ�wØ,¨]Õ;ˆàÐàˆ
ä&/°×&9Ñ"ˆM˜7Ø(¨×):Ñ):Õ;ˆFà#×.Ñ.ˆNô ˜Z¨Ó8ˆJä˜;×'�Ü˜yÖ)�AØ%+×%8Ñ%8¸Õ%;�NØ"(×"8Ñ"8¸Õ";�Kà )�m¨aÕ/�Gà#)×#4Ñ#4°WÕ#=�LØ &× 4Ñ 4°WÕ =�Ið
 # g×&;Ñ&;¸aÕ&?Ô?Ù Ø  G×$9Ñ$9¸AÕ$=Ô=Ù à"×7Ñ7×;Ñ;¸KÈ×OÒOÙ Ø  ;Ô.Ù Ø&¨Õ4°qÕ8�FØÐ 1Ô1Ù à&×-Ñ-Ù)Ø*7Ø(3Ø&/Ø+9Ø)5ôöô3 *ô (ñ ':ôX $ØÑ$QÐ[_ô
Ðð ÐØˆÜ&ˆDÜ�5‹z˜[Ô(ÚØ˜t×1Ñ1Õ2ˆGð !Ÿ™¨×(8Ñ(8¸D¿N¹NÈQÕ<NÐPˆJØ$×6Ñ6°t×7GÑ7GÕHˆNØ"×4Ñ4°T·^±^ÕDˆLØ!×,Ñ,¨^¸|ÈaÕ?OÐQˆKØ ×9Ñ9¸*ÓEˆHð  —~‘~Ó'ˆHØ—x‘x §¡Ó 0Ó1ˆHØŸ™ Ó-ˆIä�y /×2Ò2Ø )× 7Ñ 7‘à )× HÑ H�ä'¨°)¸]ÈOÓ\ˆJàÐ-Ô-Úà+/Ð˜ZÑ(à�L‰LÙ  jÀ×ATÑATÐcg×ctÑctÔu÷ñO '÷Z Ø�L‰LÑ)¨rÀ$ÐUYÔZÔ[àˆØ"ÐÛˆEØ×Ñ × 4Ñ 4°u×7IÑ7IÕ IÔJß&Ñ&Ø&+Ò#ñ ô
 ! Ó.ˆàˆ
Ü! %Ö(‰HˆAˆuÜ ×,Ò,Ó.ˆFØ"ŸZ™ZˆF�6‰NØ$)¨!¥HˆF�=Ñ!Ø',×';Ñ';ˆFÐ#Ñ$Ø%*×%7Ñ%7ˆF�>Ñ"Ø×Ñ˜fÖ%ñ )ô ˆz‹?˜QÔÜÐ3Ó4Ð4ØÒ&ÜÐ3Ó4Ð4àˆ
Ø+5Ð˜Ÿ™Ñ(ð +>×*BÑ*Bˆ˜Ÿ™Ñ'à)3ˆ�w—~‘~Ô&ñM #:ôP 
Ð$ c×	*Ô	*¨fØ�‰”T—Z’Z ¸Ô:¸TÕAÔB÷ 
+ô 
Ð ×	%Ô	%¨Ø�‰”T—Z’Z °qÔ9¸DÕ@ÔA÷ 
&÷ ÜÐ+¨S×1Ô1°VØ�L‰LœŸšÐ$4¸QÔ?À$ÕFÔG÷ 2ð Ðˆ?Ð÷ 
+×	*ú÷ 
&×	%ú÷ 2Ö1ð Ðús$   Õ/X$Ö/X7×)/Y
Ø$X4	Ø7Y	Ù
Y	r   )Nr4   )F)Ú__doc__r5   r  rå   r   r"   Úmodels.bertr   Úutilsr   Ú
get_loggerÚ__name__rÂ   r*   r,   r1   rA   rU   r`   rl   rr   rŒ   r˜   rŸ   r¡   r³   rÖ   rã   rì   r?  rQ  r   r   r   Ú<module>rW     s–   ðñó Û Û Û 	Û å )Ý ð 
×	Ò	˜HÓ	%€òCò('òEòò #ò4ô
ò*1ò
\òD)ò9ò.,ôô>[ò|	òò.Lô^}r   