Mintplex-Labs/transformersjs-electron
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/transformersjs-electron.git synced 2026-07-19 21:26:04 -04:00
Code Issues Packages Projects Releases Wiki Activity

Fix typo in tokenizers.js (#518)

Browse Source
This commit is contained in:
Joshua Lochner
2024-01-14 20:39:48 +02:00
committed by GitHub
parent 935274fd3f
commit b07336d8f7
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tokenizers.js
+1 -1
View File
@@ -2463,7 +2463,7 @@ export class PreTrainedTokenizer extends Callable {
this.sep_token = this.getToken('sep_token');
this.sep_token_id = this.model.tokens_to_ids.get(this.sep_token);
this.unk_token = this.getToken(tokenizerConfig, 'unk_token');
this.unk_token = this.getToken('unk_token');
this.unk_token_id = this.model.tokens_to_ids.get(this.unk_token);
this.model_max_length = tokenizerConfig.model_max_length;