Module Tikitoken

val byte_pair_merge : Core.Bytes.t -> (Core.Bytes.t, int) Core.Hashtbl.t -> (int * int) array
val byte_pair_encode : Core.Bytes.t -> (Core.Bytes.t, int) Core.Hashtbl.t -> int list
val encode_ordinary : Pcre.regexp -> (Core.Bytes.t, int) Core.Hashtbl.t -> string -> int list
module Bytes_hashable : Core.Hashtbl.Key with type t = bytes
val parse_tiktoken_bpe : string -> (Bytes_hashable.t, int) Core.Hashtbl.t
val decode_native : ('a, Core.Bytes.t) Core.Hashtbl.t -> 'a Core.Hashtbl.key list -> bytes
val regex : Pcre.regexp
type codec = {
  1. encoder : (bytes, int) Core.Hashtbl.t;
  2. decoder : (int, bytes) Core.Hashtbl.t;
}
val create_codec : string -> codec
val decode : codec:codec -> encoded:int Core.Hashtbl.key list -> bytes
val encode : codec:codec -> text:string -> int list