diff --git a/README.md b/README.md index 432a413..68cb6d3 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,6 @@ typedef struct CChatCompletionRequestMessage { const struct CFunctionCall *function_call; } CChatCompletionRequestMessage; -void tiktoken_init_logger(void); - CoreBPE *tiktoken_r50k_base(void); CoreBPE *tiktoken_p50k_base(void); @@ -56,24 +54,28 @@ CoreBPE *tiktoken_get_bpe_from_model(const char *model); size_t tiktoken_get_completion_max_tokens(const char *model, const char *prompt); size_t tiktoken_num_tokens_from_messages(const char *model, - uint32_t num_messages, - const struct CChatCompletionRequestMessage *messages); + uint32_t num_messages, + const struct CChatCompletionRequestMessage *messages); size_t tiktoken_get_chat_completion_max_tokens(const char *model, - uint32_t num_messages, - const struct CChatCompletionRequestMessage *messages); + uint32_t num_messages, + const struct CChatCompletionRequestMessage *messages); size_t *tiktoken_corebpe_encode_ordinary(CoreBPE *ptr, const char *text, size_t *num_tokens); size_t *tiktoken_corebpe_encode(CoreBPE *ptr, - const char *text, - const char *const *allowed_special, - size_t allowed_special_len, - size_t *num_tokens); + const char *text, + const char *const *allowed_special, + size_t allowed_special_len, + size_t *num_tokens); -size_t *tiktoken_corebpe_encode_with_special_tokens(CoreBPE *ptr, const char *text, size_t *num_tokens); +size_t *tiktoken_corebpe_encode_with_special_tokens(CoreBPE *ptr, + const char *text, + size_t *num_tokens); char *tiktoken_corebpe_decode(CoreBPE *ptr, const size_t *tokens, size_t num_tokens); + +const char *tiktoken_c_version(void); ``` ## Language Bindings