>>101603185
Now install mistral_common like the README says and see what this outputs:
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
from mistral_common.protocol.instruct.messages import (UserMessage,
AssistantMessage,
SystemMessage)
from mistral_common.protocol.instruct.request import ChatCompletionRequest
tokenizer = MistralTokenizer.from_file(f"./tekken.json")
completion_request = ChatCompletionRequest(messages=[
SystemMessage(content="system message"),
UserMessage(content="user message"),
AssistantMessage(content="assistant message"),
UserMessage(content="user message 2"),
])
chat = tokenizer.encode_chat_completion(completion_request)
print(f"text = |{chat.text}|")