mirror of
https://github.com/trushildhokiya/allininx-2.git
synced 2025-03-15 18:58:39 +00:00
17 lines
437 B
Python
17 lines
437 B
Python
|
import tiktoken
|
||
|
|
from fastapi import FastAPI
|
||
|
|
|
||
|
|
ENCODING_NAME = "cl100k_base" # gpt-4, gpt-3.5-turbo, text-embedding-ada-002
|
||
|
|
|
||
|
|
|
||
|
|
def init_tokenizer(app: FastAPI) -> None: # pragma: no cover
|
||
|
|
"""
|
||
|
|
Initialize tokenizer.
|
||
|
|
|
||
|
|
TikToken downloads the encoding on start. It is then
|
||
|
|
stored in the state of the application.
|
||
|
|
|
||
|
|
:param app: current application.
|
||
|
|
"""
|
||
|
|
app.state.token_encoding = tiktoken.get_encoding(ENCODING_NAME)
|