diff --git a/src/tokenizers.js b/src/tokenizers.js index ce60f7d..8cf1325 100644 --- a/src/tokenizers.js +++ b/src/tokenizers.js @@ -2986,7 +2986,7 @@ export class PreTrainedTokenizer extends Callable { * ```javascript * import { AutoTokenizer } from "@xenova/transformers"; * - * const tokenizer = await AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1"); + * const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1"); * * const chat = [ * { "role": "user", "content": "Hello, how are you?" }, diff --git a/tests/generate_tests.py b/tests/generate_tests.py index c449d34..b363879 100644 --- a/tests/generate_tests.py +++ b/tests/generate_tests.py @@ -229,7 +229,7 @@ TOKENIZERS_WITH_CHAT_TEMPLATES = { 'basic', ], - 'mistralai/Mistral-7B-Instruct-v0.1': [ + 'Xenova/mistral-tokenizer-v1': [ 'basic', ], diff --git a/tests/tokenizers.test.js b/tests/tokenizers.test.js index 8b92c67..cfbe7c6 100644 --- a/tests/tokenizers.test.js +++ b/tests/tokenizers.test.js @@ -334,7 +334,7 @@ describe('Extra decoding tests', () => { describe('Chat templates', () => { it('should generate a chat template', async () => { - const tokenizer = await AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1"); + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1"); const chat = [ { "role": "user", "content": "Hello, how are you?" },