17 lines
437 B
Python
Raw Normal View History

2025-02-17 19:44:17 +05:30
import tiktoken
from fastapi import FastAPI
ENCODING_NAME = "cl100k_base" # gpt-4, gpt-3.5-turbo, text-embedding-ada-002
def init_tokenizer(app: FastAPI) -> None: # pragma: no cover
"""
Initialize tokenizer.
TikToken downloads the encoding on start. It is then
stored in the state of the application.
:param app: current application.
"""
app.state.token_encoding = tiktoken.get_encoding(ENCODING_NAME)