We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 9378e21 commit 1e2c6f7Copy full SHA for 1e2c6f7
include/pytorch/tokenizers/tokenizer.h
@@ -59,7 +59,7 @@ class Tokenizer {
59
protected:
60
bool initialized_ = false;
61
int32_t vocab_size_ = 0;
62
- uint64_t bos_tok_, eos_tok_ = 0;
+ uint64_t bos_tok_ = 0, eos_tok_ = 0;
63
};
64
65
} // namespace tokenizers
0 commit comments