ó
    †ñ:iÒT  ã                   óâ   • S SK r S SKrS SKrSSKJrJr  SSKJr  SSK	J
r
JrJr  SSKJr   " S S	\5      r " S
 S5      rS rSS0rSS0rSS/r/ SQr " S S5      r " S S5      rS rS rS rg)é    Né   )ÚDeserializerÚ
Serializer)Úsafe_isinstance)ÚSENTENCEPIECE_TOKENIZERSÚgetattr_silentÚ!parse_prefix_suffix_for_tokenizeré   )ÚMaskerc                   ó†   ^ • \ rS rSrSrSS jrS rS rS rS r	S r
S	 rS
 rS rS rU 4S jr\SU 4S jj5       rSrU =r$ )ÚTexté   z„This masks out tokens according to the given tokenizer.

The masked variables are

output_type : "string" (default) or "token_ids"

c                 ó†  • Uc  [        5       U l        O([        U5      (       a  Xl        O [        U5      U l        X@l        X0l        X l        X l        [        U[        5      (       a  UOSU l
        [        U R                  5      nUS   U l        US   U l        SU l        Ucm  [        U R                  S5      bN  U R                  R                  U l        [        U R                  S5      U l
        U R
                  S:X  a  S	U l        OS
U l        OX l        U R                  c0  U R                  U R                  5      S   U R                     U l
        U R
                  S:X  a  SU l        U R                  SL U l        SU l        SU l        SU l        SU l        SU l        SU l        g! [         a    [        S5      ef = f)aµ  Build a new Text masker given an optional passed tokenizer.

Parameters
----------
tokenizer : callable or None
    The tokenizer used to break apart strings during masking. The passed tokenizer must support a minimal
    subset of the HuggingFace Transformers PreTrainedTokenizerBase API. This minimal subset means the
    tokenizer must return a dictionary with 'input_ids' and then either include
    an 'offset_mapping' entry in the same dictionary or provide a .convert_ids_to_tokens or .decode method.

mask_token : string, int, or None
    The sub-string or integer token id used to mask out portions of a string. If None it will use the
    tokenizer's .mask_token attribute, if defined, or "..." if the tokenizer does not have a .mask_token
    attribute.

collapse_mask_token : True, False, or "auto"
    If True, when several consecutive tokens are masked only one mask token is used to replace the entire
    series of original tokens.

Nz�The passed tokenizer cannot be wrapped as a masker because it does not have a __call__ method, not can it be interpreted as a splitting regexp!Úkeep_prefixÚkeep_suffixTÚ
mask_tokenÚmask_token_idÚautoFz...Ú	input_idsé   )ÚSimpleTokenizerÚ	tokenizerÚcallableÚ	ExceptionÚoutput_typeÚcollapse_mask_tokenÚinput_mask_tokenr   Ú
isinstanceÚintr   r	   r   r   Ú	text_datar   Úfixed_backgroundÚdefault_batch_sizeÚ_sÚ_tokenized_s_fullÚ_tokenized_sÚ_segments_sÚimmutable_outputs)Úselfr   r   r   r   Úparsed_tokenizer_dicts         ÚU/srv/projetos/modelo_ml_acdoc/venv/lib/python3.13/site-packages/shap/maskers/_text.pyÚ__init__ÚText.__init__   s¨  € ð* ÑÜ,Ó.ˆD�NÜ�i× Ñ Ø&�NðÜ!0°Ó!;�”ð 'ÔØ#6Ô Ø *ÔØ$ŒÜ+5°jÄ#×+FÑ+F™ZÈDˆÔÜ AÀ$Ç.Á.Ó QÐà0°Ñ?ˆÔØ0°Ñ?ˆÔð
 ˆŒàÑÜ˜dŸn™n¨lÓ;ÑGØ"&§.¡.×";Ñ";�”Ü%3°D·N±NÀOÓ%T�Ô"Ø×+Ñ+¨vÓ5Ø/4�DÔ,øà"'�•à(ŒOà×ÑÑ%Ø!%§¡°·±Ó!@ÀÑ!MÈd×N^ÑN^Ñ!_ˆDÔà×#Ñ# vÓ-Ø'+ˆDÔ$ð !%× 2Ñ 2°dÐ :ˆÔà"#ˆÔð ˆŒØ!%ˆÔØ ˆÔØˆÔð "&ˆÕøôq ó ÜðOóð ðús   ¬F* Æ*G c                 óŒ  • U R                  X5      nU R                  U5        U R                  S:”  a  UR                  5       nSUS U R                  & U R                  S:”  a  UR                  5       nSXR                  * S & U R
                  S:X  Ga  / nSn[        U R                  S5      n[        U5       H�  u  pgU(       d  XPR                  U   :X  a"  UR                  U R                  U   5        SnM@  U R                  (       a  U R                  (       d  Md  U(       a  Mm  UR                  SU R                  -   5        SnM�     SR                  U5      n[        U R                  [        5      (       a  UR!                  SS5      n["        R$                  " S	SU5      R'                  5       nOxU R(                  c  U R*                  U   nO[[,        R.                  " [1        [3        U5      5       Vs/ s H'  oaU   (       a  U R*                  U   OU R(                  PM)     sn5      n[,        R.                  " U/5      4$ s  snf )
Nr   TÚstringFÚ	sep_tokenÚ Ú õ   â–�z[\s]+)Ú_standardize_maskÚ_update_s_cacher   Úcopyr   r   r   r   Ú	enumerater&   Úappendr   r   Újoinr   r   ÚreplaceÚreÚsubÚstripr   r%   ÚnpÚarrayÚrangeÚlen)	r(   ÚmaskÚsÚ	out_partsÚ%is_previous_appended_token_mask_tokenr/   ÚiÚvÚouts	            r*   Ú__call__ÚText.__call__o   sõ  € Ø×%Ñ% dÓ.ˆØ×Ñ˜QÔð ×Ñ˜aÓØ—9‘9“;ˆDØ'+ˆDÐ#�4×#Ñ#Ð$Ø×Ñ˜aÓØ—9‘9“;ˆDØ(,ˆD×"Ñ"Ð"Ð$Ð%à×Ñ˜xÔ'ð
 ˆIØ49Ð1Ü& t§~¡~°{ÓCˆIÜ! $ž‘�æ˜	×%5Ñ%5°aÑ%8Ó8Ø×$Ñ$ T×%5Ñ%5°aÑ%8Ô9Ø<AÒ9à×3×3Ø×0×0Ñ0×9^Ð9^à!×(Ñ(¨¨t¯©Ñ)>Ô?Ø@DÒ=ñ (ð —'‘'˜)Ó$ˆCô
 ˜tŸ~™~Ô/G×HÑHØ—k‘k %¨Ó-�ô —&’&Ø˜#˜sóç‰e‹gñ ð
 ×!Ñ!Ñ)Ø×'Ñ'¨Ñ-‘ä—h’hÔafÔgjÐkoÓgpÔaqÓrÒaqÐ\]¸Q¿ × 1Ñ 1°!Ò 4ÀT×EWÑEWÒ WÑaqÑrÓs�ô —’˜#˜“Ð!Ð!ùò  ss   Ç4.Ic                 ó,   • U R                  U5      S   4$ )zfCalled by explainers to allow us to convert data to better match masking (here this means tokenizing).r   )Útoken_segments©r(   rB   s     r*   Údata_transformÚText.data_transform­   s   € à×#Ñ# AÓ& qÑ)Ð+Ð+ó    c                 ó$  •  U R                  USS9nUS   nU Vs/ s H	  oDc  SOUPM     nn[        [        U5      S-
  5       Vs/ s H#  oQX5   S   [        X5   S   X5S-      S   5       PM%     nnUR	                  X[        U5      S-
     S   U[        U5      S-
     S    5        XbS   4$ s  snf s  snf ! [
        [        4 GaÅ    U R                  U5      S   n[        U R                   S	5      (       a  U R                   R                  U5      nO1U V	s/ s H  o�R                   R                  U	/5      PM      Os  sn	f nn	[        U R                   S
5      (       a‚  U R                   R                  USS9n
[        U R                   S5      [        U R                   S5      /n[        U
5       H-  u  p\US:X  d  M  X…   U;  d  US-   [        U
5      :X  d  M)  SX…'   M/     [        U R                   [        5      (       a9  [        U5       H)  u  p\UR                  S5      (       d  M  SX…   SS -   X…'   M+     ON[        U5       H?  u  p\UR                  S5      (       a  X…   SS X…'   M&  US:w  d  M.  US:w  d  M6  SX…   -   X…'   MA     X‡4s $ f = f)zFReturns the substrings associated with each token in the given string.T)Úreturn_offsets_mappingÚoffset_mappingN)r   r   r
   r   r   Úconvert_ids_to_tokensÚget_special_tokens_mask)Úalready_has_special_tokensr/   r   r1   Ú_r0   ú##r   )r   r?   r@   Úmaxr7   ÚNotImplementedErrorÚ	TypeErrorÚhasattrrS   ÚdecoderT   r   r6   r   r   Ú
startswith)r(   rB   Ú
token_dataÚoffsetsÚorE   ÚpartsÚ	token_idsÚtokensÚidÚspecial_tokens_maskÚspecial_keeprF   s                r*   rK   ÚText.token_segments±   s|  € ð$	%ØŸ™¨À$˜ÐGˆJØ Ð!1Ñ2ˆGÙ;BÓCº7°a¡‘v°Ò1¹7ˆGÐCÜW\Ô]`ÐahÓ]iÐlmÑ]mÔWnÓoÒWnÐRS�w‘z !‘}¤s¨7©:°a©=¸'ÀaÁ%¹.ÈÑ:KÓ'LÓMÑWnˆEÐoØ�L‰L˜¤3 w£<°!Ñ#3Ñ4°QÑ7¸'Ä#ÀgÃ,ÐQRÑBRÑ:SÐTUÑ:VÐWÔXØ [Ñ1Ð1Ð1ùò DùÚoøô $¤YÐ/ô 	%ØŸ™ qÓ)¨+Ñ6ˆIÜ�t—~‘~Ð'>×?Ñ?ØŸ™×=Ñ=¸iÓH‘á@IÓJÂ	¸"Ÿ.™.×/Ñ/°°Ö5Â	ùÒJ�ÐJÜ�t—~‘~Ð'@×AÑAØ&*§n¡n×&LÑ&LÈYÐswÐ&LÐ&xÐ#ô # 4§>¡>°;Ó?Ü" 4§>¡>°<Ó@ð �ô &Ð&9Ö:‘D�AØ˜A•v 6¡9°LÓ#@ÀAÈÁEÌSÐQdÓMeÕDeØ$&˜›	ñ ;ô
 ˜tŸ~™~Ô/G×HÑHÜ% fÖ-‘D�AØ—|‘| C×(Ó(Ø$'¨&©)°A°B¨-Ñ$7˜›	ò .ô & fÖ-‘D�AØ—|‘| D×)Ñ)Ø$*¡I¨a¨b M˜›	Ø˜b� Q¨!¥VØ$'¨&©)¡O˜›	ñ	 .ð Ð$Ò$ð;	%ús[   ‚B9 ›B/«B9 Á*B4Á0>B9 Â/
B9 Â9A JÄ%D?Ä>A=JÆ?JÇAJÈ*AJÉ4JÉ<JÊJc                 ó”  • U R                  U5        / n[        U R                  S5      nUc  / nOU/n/ n[        R                  " S5      n[        R                  " S5      n[        U R                  5       H�  u  pxUS:”  ac  UR                  U R                  US-
     5      cA  UR                  U5      b/  XGS-
     S:w  a$  UR                  SUR                  5       -   5        Mn  UR                  UR                  5       5        M�     [        XB5      n	U	SS2S	4   U	SS2S
4'   U	SS2S
4==   U	SS2S
4   R                  5       -  ss'   U	$ )z6Compute the clustering of tokens for the given string.r/   Nz^.*\W$z	^[A-Za-z]r   r
   r1   rW   é   r   )r4   r   r   r:   Úcompiler6   r&   Úmatchr7   r<   Úpartition_treerX   )
r(   rB   Úspecial_tokensr/   rc   Ú	space_endÚletter_startrE   rF   Úpts
             r*   Ú
clusteringÚText.clusteringÙ   s&  € à×Ñ˜QÔØˆÜ" 4§>¡>°;Ó?ˆ	ØÑØ‰Nà'˜[ˆNð ˆÜ—J’J˜yÓ)ˆ	Ü—z’z ,Ó/ˆÜ˜d×.Ñ.Ö/‰DˆAà�A“Ø—O‘O D×$4Ñ$4°Q¸±UÑ$;Ó<ÑDØ ×&Ñ& qÓ)Ñ5Ø˜q™5‘M RÓ'à—‘˜d Q§W¡W£YÑ.Ö/à—‘˜aŸg™g›iÖ(ñ 0ô ˜FÓ3ˆð ’a˜�d‘8ˆŠ1ˆaˆ4‰Ø
Š1ˆaˆ4‹�B’q˜!�t‘H—L‘L“NÑ"‹àˆ	rO   c                 óÄ   • U R                   U:w  aP  Xl         U R                  U5      u  p#[        R                  " U5      U l        [        R                  " U5      U l        g g ©N)r#   rK   r=   r>   r%   r&   )r(   rB   rc   rb   s       r*   r4   ÚText._update_s_cache   sL   € Ø�7‰7�a‹<ØŒGØ $× 3Ñ 3°AÓ 6ÑˆFÜ "§¢¨Ó 3ˆDÔÜ!Ÿxšx¨Ó/ˆDÕð	 rO   c                 óR   • U R                  U5        S[        U R                  5      4$ )zuThe shape of what we return as a masker.

Note we only return a single sample, so there is no expectation averaging.
r
   ©r4   r@   r%   rL   s     r*   ÚshapeÚ
Text.shape'  s(   € ð
 	×Ñ˜QÔØ”3�t×(Ñ(Ó)Ð*Ð*rO   c                 óR   • U R                  U5        [        U R                  5      4/$ )z!The shape of the masks we expect.rw   rL   s     r*   Úmask_shapesÚText.mask_shapes/  s'   € à×Ñ˜QÔÜ�T×&Ñ&Ó'Ð)Ð*Ð*rO   c                 ó˜  • U R                  U5        [        R                  " [        U R                  5      [
        S9nU R                  S:”  a  SUSU R                  & U R                  S:”  a  SX R                  * S& [        U R                  5       H%  u  p4U[        U R                  S5      :X  d  M!  SX#'   M'     UR                  SS5      $ )úLThe names of the features for each mask position for the given input string.)Údtyper   TNÚsep_token_idr
   éÿÿÿÿ)r4   r=   Úzerosr@   r%   Úboolr   r   r6   r   r   Úreshape)r(   rB   Ú
invariantsrE   rF   s        r*   r…   ÚText.invariants4  s´   € à×Ñ˜QÔä—X’Xœc $×"3Ñ"3Ó4¼DÑAˆ
Ø×Ñ˜aÓØ-1ˆJÐ)˜×)Ñ)Ð*Ø×Ñ˜aÓØ.2ˆJ×(Ñ(Ð(Ð*Ð+ä˜d×/Ñ/Ö0‰DˆAØ”N 4§>¡>°>ÓBÕBØ $�
“ñ 1ð ×!Ñ! ! RÓ(Ð(rO   c                 ó„   • U R                  U5        U R                   Vs/ s H  o"R                  5       PM     sn/$ s  snf )r~   )r4   r&   r<   )r(   rB   rF   s      r*   Úfeature_namesÚText.feature_namesC  s7   € à×Ñ˜QÔØ$(×$4Ò$4Ó5Ò$4˜q—‘–Ñ$4Ñ5Ð6Ð6ùÒ5s    =c                 óN  >• [         TU ]  U5        [        USSS9 nUR                  SU R                  5        UR                  SU R                  5        UR                  SU R
                  5        UR                  SU R                  5        SSS5        g! , (       d  f       g= f)	z$Save a Text masker to a file stream.úshap.maskers.Textr   )Úversionr   r   r   r   N)ÚsuperÚsaver   r   r   r   r   )r(   Úout_filerB   Ú	__class__s      €r*   rŽ   Ú	Text.saveH  sz   ø€ ä‰‰�XÔÜ˜Ð"5¸qÒAÀQØ�F‰F�; §¡Ô/Ø�F‰F�< ×!6Ñ!6Ô7Ø�F‰FÐ(¨$×*BÑ*BÔCØ�F‰F�= $×"2Ñ"2Ô3÷	 B×AÖAús   œA1BÂ
B$c                 óB  >• U(       a  U R                  U5      $ [        TU ]	  USS9n[        USSSS9 nUR                  S5      US'   UR                  S5      US'   UR                  S5      US'   UR                  S	5      US	'   S
S
S
5        U$ ! , (       d  f       U$ = f)z&Load a Text masker from a file stream.F)Úinstantiater‹   r   )Úmin_versionÚmax_versionr   r   r   r   N)Ú_instantiated_loadr�   Úloadr   )ÚclsÚin_filer“   ÚkwargsrB   r�   s        €r*   r—   Ú	Text.loadQ  sª   ø€ ö Ø×)Ñ)¨'Ó2Ð2ä‘‘˜g°5�Ð9ˆÜ˜'Ð#6ÀAÐSTÒUÐYZØ"#§&¡&¨Ó"5ˆF�;ÑØ#$§6¡6¨,Ó#7ˆF�<Ñ Ø,-¯F©FÐ3HÓ,IˆFÐ(Ñ)Ø$%§F¡F¨=Ó$9ˆF�=Ñ!÷	 Vð
 ˆ÷ VÔUð
 ˆús   ´ABÂ
B)r#   r&   r%   r$   r   r"   r!   r'   r   r   r   r   r   r   r    r   )NNr   r.   ©T)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r+   rH   rM   rK   rq   r4   rx   r{   r…   rˆ   rŽ   Úclassmethodr—   Ú__static_attributes__Ú__classcell__)r�   s   @r*   r   r      sX   ø† ñôT&òl<"ò|,ò&%òP òN0ò+ò+ò
)ò7õ
4ð öó örO   r   c                   ó,   • \ rS rSrSrSS jrSS jrSrg)	r   i`  z!A basic model agnostic tokenizer.c                 ó:   • [         R                  " U5      U l        g)z7Create a tokenizer based on a simple splitting pattern.N)r:   rj   Úsplit_pattern)r(   r§   s     r*   r+   ÚSimpleTokenizer.__init__c  s   € äŸZšZ¨Ó6ˆÕrO   c                 ót  • Sn/ n/ n[         R                  " U R                  U5       H=  nUR                  S5      u  pxUR	                  X745        UR	                  XU 5        UnM?     U[        U5      :w  a/  UR	                  U[        U5      45        UR	                  XS 5        0 n	XYS'   U(       a  XIS'   U	$ )zbTokenize the passed string, optionally returning the offsets of each token in the original string.r   Nr   rR   )r:   Úfinditerr§   Úspanr7   r@   )
r(   rB   rQ   ÚposÚoffset_rangesr   ÚmÚstartÚendrG   s
             r*   rH   ÚSimpleTokenizer.__call__g  sµ   € àˆØˆØˆ	Ü—’˜T×/Ñ/°Ö3ˆAØŸ™ ›‰JˆEØ× Ñ  # Ô.Ø×Ñ˜Q 5˜\Ô*ØŠCñ	 4ð
 ”#�a“&‹=Ø× Ñ  #¤s¨1£v Ô/Ø×Ñ˜Q˜t˜WÔ%àˆØ$ˆKÑÞ!Ø$1Ð Ñ!Øˆ
rO   )r§   N)z\W+rœ   )r�   rž   rŸ   r    r¡   r+   rH   r£   © rO   r*   r   r   `  s   † Ù+ô7÷rO   r   c                 ó*   • U R                  SS5      n U $ )zIReplaces whitespace encoded as '_' with ' ' for sentencepiece tokenizers.r2   r0   )r9   ©rB   s    r*   Ú+post_process_sentencepiece_tokenizer_outputrµ   |  s   € à	�	‰	�%˜Ó€AØ€HrO   Ú(Ú)Ú.Ú,)ÚbutÚandÚorc                   ó*   • \ rS rSrSrS rS rS rSrg)ÚTokeniˆ  z1A token representation used for token clustering.c                 óV   • Xl         U[        ;   d
  U[        ;   a  SU l        g SU l        g )NFT)rB   ÚopenersÚclosersÚbalanced)r(   Úvalues     r*   r+   ÚToken.__init__‹  s$   € ØŒØ”GÓ˜u¬Ó/Ø!ˆD�Mà ˆD�MrO   c                 ó   • U R                   $ rt   r´   ©r(   s    r*   Ú__str__ÚToken.__str__’  s   € Ø�v‰vˆrO   c                 óZ   • U R                   (       d  U R                  S-   $ U R                  $ )NÚ!©rÂ   rB   rÆ   s    r*   Ú__repr__ÚToken.__repr__•  s    € Ø�}�}Ø—6‘6˜C‘<ÐØ�v‰vˆrO   rË   N)	r�   rž   rŸ   r    r¡   r+   rÇ   rÌ   r£   r²   rO   r*   r¾   r¾   ˆ  s   † Ù;ò!òõrO   r¾   c                   ó:   • \ rS rSrSrS
S jrS rS rS rS r	S	r
g)Ú
TokenGroupi›  zCA token group (substring) representation used for token clustering.Nc                 ó   • Xl         X l        g rt   ©ÚgÚindex)r(   ÚgrouprÓ   s      r*   r+   ÚTokenGroup.__init__ž  s   € ØŒØ�
rO   c                 ó6   • U R                   R                  5       $ rt   )rÒ   rÌ   rÆ   s    r*   rÌ   ÚTokenGroup.__repr__¢  s   € Ø�v‰v�‰Ó Ð rO   c                 ó    • U R                   U   $ rt   )rÒ   )r(   rÓ   s     r*   Ú__getitem__ÚTokenGroup.__getitem__¥  s   € Ø�v‰v�e‰}ÐrO   c                 óF   • [        U R                  UR                  -   5      $ rt   )rÏ   rÒ   )r(   r`   s     r*   Ú__add__ÚTokenGroup.__add__¨  s   € Ü˜$Ÿ&™& 1§3¡3™,Ó'Ð'rO   c                 ó,   • [        U R                  5      $ rt   )r@   rÒ   rÆ   s    r*   Ú__len__ÚTokenGroup.__len__«  s   € Ü�4—6‘6‹{ÐrO   rÑ   rt   )r�   rž   rŸ   r    r¡   r+   rÌ   rÙ   rÜ   rß   r£   r²   rO   r*   rÏ   rÏ   ›  s   † ÙMôò!òò(õrO   rÏ   c                 óà  • Sn[        U5      S:”  a9  U S   R                  U;   a&  US   R                  U;   a  U[        R                  -  nUS   R                  R	                  S5      (       a  US-  nUS   R                  S:X  a6  [        U5      S:X  d"  [        U5      S:X  a  US   R                  S;   a  US	-  nU S   R                  S:X  a  US   R                  S;   a  US	-  nU S   R                  R	                  S
5      =(       a    U S   R                  R                  S5      nUS   R                  R	                  S
5      =(       a    US   R                  R                  S5      nU(       a  U(       a  U(       a  U(       d  US-  nUS   R                  [        ;   a  US   R                  (       d  US-  nU S   R                  [        ;   a  U S   R                  (       d  US-  nU S   R                  [        ;   a  US   [        ;  a  US-  nU S   R                  [        ;   d  US   R                  [        ;   a  US-  nU S   R                  S:X  a  US-  nUS   R                  S:X  a  [        U5      S:”  a  US-  nOUS-  nU S   R                  S;   a  US-  nUS   R                  S;   a  [        U5      S:”  a  US-  nOUS-  nU[        U 5      [        U5      -   -  nU$ )z~Compute the score of merging two token groups.

special_tokens: tokens (such as separator tokens) that should be grouped last
r   r�   rW   é   Ú'r
   r   )ÚtrB   é   Ú[Ú]iè  éd   r¹   é
   )r¸   Ú?rÊ   )
r@   rB   ÚmathÚinfr]   ÚendswithrÀ   rÂ   rÁ   Ú
connectors)Úgroup1Úgroup2rm   ÚscoreÚ
start_ctrlÚend_ctrls         r*   Úmerge_scorerô   ¯  s‰  € ð
 €Eä
ˆ>Ó˜QÓØ�"‰:�<‰<˜>Ó)¨f°Q©i¯k©k¸^Ó.KØ”T—X‘XÑˆEð ˆa�y‡{�{×Ñ˜d×#Ñ#Ø�‰ˆð ˆa�y‡{�{�cÓœs 6›{¨aÓ/´C¸³KÀ1Ó4DÈÐPQÉÏÉÐXbÓIbØ�‰ˆØˆb�z‡|�|�sÓ˜v a™yŸ{™{¨jÓ8Ø�‰ˆà˜‘—‘×'Ñ'¨Ó,×J°¸±·±×1EÑ1EÀcÓ1J€JØ�b‰z�|‰|×&Ñ& sÓ+×J°°r±
·±×0EÑ0EÀcÓ0J€Hæž8®¾*Ø�‰ˆØˆa�y‡{�{”gÓ f¨Q¡i×&8×&8Ø�‰ˆØˆb�z‡|�|”wÓ v¨b¡z×':×':Ø�‰ˆð ˆa�y‡{�{”gÓ &¨¡*´GÓ";Ø�‰
ˆð ˆb�z‡|�|”zÓ! V¨A¡Y§[¡[´JÓ%>Ø�‰
ˆð ˆb�z‡|�|�sÓØ�‰ˆØˆa�y‡{�{�cÓÜˆv‹;˜‹?Ø�R‰K‰Eà�Q‰JˆEð ˆb�z‡|�|�Ó&Ø�‰ˆØˆa�y‡{�{�oÓ%Üˆv‹;˜‹?Ø�R‰K‰Eà�Q‰JˆEà	ŒS�‹[œ3˜v›;Ñ&Ñ&€Eà€LrO   c           	      ó´  • [        [        U 5      S-
  5       Vs/ s H  n[        X   XS-      U5      PM     nn[        R                  " U5      nX   XS-      -   X'   X   S   R
                  [        ;   aI  XS-      S   R
                  [        X   S   R
                     :X  a  SX   S   l        SXS-      S   l        U R                  US-   5        gs  snf )zEFinds the two token groups with the best merge score and merges them.r
   r   r�   TN)	r?   r@   rô   r=   ÚargmaxrB   rÀ   rÂ   Úpop)Úgroupsrm   rE   ÚscoresÚinds        r*   Úmerge_closest_groupsrû   í  sÒ   € äMRÔSVÐW]ÓS^ÐabÑSbÔMcÓdÒMcÈŒk˜&™) V°©E¡]°NÖCÑMc€FÐdä
�)Š)�FÓ
€CØ‘+ ¨Q¡w¡Ñ/€F�Kà�{�1�~×Ñœ7Ó" v°A©g¡°rÑ':×'<Ñ'<ÄÈÉÐTUÉ×HXÑHXÑ@YÓ'YØ"&ˆ‰�A‰ÔØ'+ˆ�Q‰w‰˜ÑÔ$à
‡J�Jˆs�Q‰wÕùò es   šCc           
      óú  • [        U 5       VVs/ s H  u  p#[        [        U5      /U5      PM     nnn[        U 5      nUn[        R
                  " US-
  S45      n[        [        U5      S-
  5       GHe  n[        [        U5      S-
  5       Vs/ s H  n[        XB   XBS-      U5      PM     nn[        R                  " U5      n	XI   R                  n
XIS-      R                  nXI   R                  XvU-
  S4'   XIS-      R                  XvU-
  S4'   X‰   * XvU-
  S4'   X¥:¼  a	  XzU-
  S4   OSXµ:¼  a	  X{U-
  S4   OS-   XvU-
  S4'   XI   XIS-      -   XI'   XdU	   l	        XI   S   R                  [        ;   aI  XIS-      S   R                  [        XI   S   R                     :X  a  SXI   S   l        SXIS-      S   l        UR                  U	S-   5        US-  nGMh     USS2S4   S	-   USS2S4'   U$ s  snnf s  snf )
z±Build a heriarchial clustering of tokens that align with sentence structure.

Note that this is fast and heuristic right now.
TODO: Build this using a real constituency parser.
r
   é   r   r   ri   r�   TNré   )r6   rÏ   r¾   r@   r=   r‚   r?   rô   rö   rÓ   rB   rÀ   rÂ   r÷   )Údecoded_tokensrm   rE   rä   Útoken_groupsÚMÚ	new_indexÚclustmrù   rú   ÚlindÚrinds               r*   rl   rl   û  sA  € ô ;DÀNÔ:SÔTÒ:S±$°!”J¤ a£˜z¨1Ö-Ñ:S€LÑTäˆNÓ€AØ€IÜ�XŠX�q˜1‘u˜a�jÓ!€FÜ”3�|Ó$ qÑ(×)ˆäW\Ô]`ÐamÓ]nÐqrÑ]rÔWsó
ÚWsÐRSŒK˜™¨¸!±eÑ)<¸nÖMÑWsð 	ð 
ô �iŠi˜ÓˆàÑ ×&Ñ&ˆØ !™GÑ$×*Ñ*ˆØ#/Ñ#4×#:Ñ#:ˆ˜1‰}˜aÐÑ Ø#/°a±Ñ#8×#>Ñ#>ˆ˜1‰}˜aÐÑ Ø$*¡K <ˆ˜1‰}˜aÐÑ Ø;?»9 F°!©8°Q¨;Ò$7È!ÐgkÓgpÐPVÐ^_ÑW_ÐabÐWbÒPcÐvwÑ#xˆ˜1‰}˜aÐÑ à(Ñ-°À1¹WÑ0EÑEˆÑØ"+�SÑÔð Ñ˜QÑ×!Ñ!¤WÓ,°ÀA¹gÑ1FÀrÑ1J×1LÑ1LÔPWÐXdÑXiÐjkÑXl×XnÑXnÑPoÓ1oØ,0ˆLÑ˜aÑ Ô)Ø15ˆL˜q™Ñ! "Ñ%Ô.à×Ñ˜˜q™Ô!Ø�Q‰‹	ñ/ *ð6 š!˜Q˜$‘< "Ñ$€FŠ1ˆaˆ4�Là€MùóE Uùò
s   �"G2ÂG8)rë   r:   Únumpyr=   Ú_serializabler   r   Úutilsr   Úutils.transformersr   r   r	   Ú_maskerr   r   r   rµ   rÀ   rÁ   Úendersrî   r¾   rÏ   rô   rû   rl   r²   rO   r*   Ú<module>r     s‡   ðÛ Û 	ã ç 4Ý #÷ñ õ
 ôMˆ6ô M÷`
ñ ò8ð �ˆ*€Ø�ˆ*€Ø
ˆsˆ€Ú!€
÷ñ ÷&ñ ò(;ò|ó(rO   