Skip to content

Commit

Permalink
Fix build after merge
Browse files Browse the repository at this point in the history
  • Loading branch information
LaylBongers committed Jun 26, 2024
1 parent 09f9636 commit c04bf26
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions llama.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5092,7 +5092,7 @@ static void llm_load_vocab(
vocab.special_unk_id = -1;
vocab.special_sep_id = -1;
vocab.special_pad_id = -1;
vocab.add_space_prefix = false;
vocab.tokenizer_add_space_prefix = false;
} else {
throw std::runtime_error(format("unknown tokenizer: '%s'", tokenizer_model.c_str()));
}
Expand Down Expand Up @@ -14673,7 +14673,7 @@ struct llm_tokenizer_rwkv {
// Iterate through possible tokens backwards, starting with the largest
for (int32_t i = (int32_t)tokens.size() - 1; i >= 0; i--) {
// Skip tokens that aren't normal type, we can't match on those
if (vocab.id_to_token[i].type != LLAMA_TOKEN_TYPE_NORMAL) {
if (vocab.id_to_token[i].attr != LLAMA_TOKEN_ATTR_NORMAL) {
continue;
}

Expand Down

0 comments on commit c04bf26

Please sign in to comment.