o D2h@snddlmZddlZddlZddlZdddZd d d Zd!d"ddZ  d#d$ddZd%ddZ d!d&ddZ dS)') annotationsNblobpathstrreturnbytesc Cs|ds<|dsddtdD}dd|Dd}tdD]}||vr-|||td|<|d7}qt|dks6Jt||}dd|d dd D}dfdd ddt|D} t| }|D]\} } || | | <|d7}qaddl} | t||} fdd| D}| dd| dd| |ksJ| S)NcSs(g|]}t|rt|dkr|qS) )chr isprintable.0brrr ]s(z3data_gym_to_mergeable_bpe_ranks..cSsi|]}t||qSr)rGrIrrr _sz3data_gym_to_mergeable_bpe_ranks..rcSsg|]}t|qSr)tuplesplit)rJ merge_strrrrrLjs valuerrrcstfdd|DS)Nc3s|]}|VqdSrrrIdata_gym_byte_to_byterr mszKdata_gym_to_mergeable_bpe_ranks..decode_data_gym..r)rUrVrrdecode_data_gymlsz8data_gym_to_mergeable_bpe_ranks..decode_data_gymcSsi|] \}}t|g|qSrrY)rJirKrrrrNpscsi|] \}}||qSrr)rJkv)rZrrrN}ss <|endoftext|>s<|startoftext|>)rUrrr) rangeappendrGlenr@decoderQ enumeratejsonloadsitemspop)rArBrCrDrank_to_intbytenrKvocab_bpe_contents bpe_merges bpe_ranksfirstsecondrc encoder_jsonencoder_json_loadedr)rWrZrdata_gym_to_mergeable_bpe_ranksVs2       rprktiktoken_bpe_fileNonec Cszddl}Wnty}ztd|d}~ww||d*}t|dddD]\}}|t|dt| dq)WddS1sKwYdS) Nrrr'cSs|dS)NrOr)xrrrsz#dump_tiktoken_bpe..)key  ) r r r sortedrer8base64 b64encoderr/)rkrqr rrtokenrankrrrdump_tiktoken_bpes &"r}c Csvt||}i}|D]-}|sq z|\}}t||t|<Wq ty8}z td|d||d}~ww|S)NzError parsing line z in )r@ splitlinesrQintry b64decode Exceptionr4)rqrr>retliner{r|rrrrload_tiktoken_bpes   r)rrrr)rrrrrrr)rrrr!rr)NN) rArrBrrCr!rDr!rrE)rkrErqrrrr)rqrrr!rrE) __future__rryrr(rr r@rpr}rrrrrs    9 0