Skip to content

Commit 281f69c

Browse files
dungquixote42dungquixote42
committed
add special rewind case: idx=0
1 parent e3b4a52 commit 281f69c

1 file changed

Lines changed: 65 additions & 25 deletions

File tree

‎common/sampling.cpp‎

Lines changed: 65 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -831,7 +831,6 @@ static void elb_sub(common_params_sampling& sparams, const common_params_samplin
831831
}
832832

833833
void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float* logits) {
834-
// printf("%s[%d]: \n", __func__, __LINE__);
835834
const auto& elb = ctx_sampling->elb_states[ctx_sampling->elb_idx];
836835

837836
auto index_first_inactive = [&elb](auto& tokens) {
@@ -850,6 +849,7 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float
850849
for (size_t j = 0; j < ifi; ++j) {
851850
const auto& [id, bias, _, cond] = elb.other_tokens[j];
852851
if (string_ends_with(window, cond)) {
852+
// LLAMA_LOG_DEBUG("%s[%d]: %d\n", __func__, __LINE__, id);
853853
logits[id] += bias;
854854
}
855855
}
@@ -860,20 +860,22 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float
860860
if (window.empty()) {
861861
// empty case here
862862
for (size_t j = 0; j < ifi; ++j) {
863+
// LLAMA_LOG_DEBUG("%s[%d]: %d\n", __func__, __LINE__, elb.first_tokens[j].id);
863864
logits[elb.first_tokens[j].id] += elb.first_tokens[j].bias;
864865
}
865866
} else {
866867
for (size_t j = 0; j < ifi; ++j) {
867868
const auto& [id, bias, _, cond] = elb.first_tokens[j];
868869
// no bias if seen (probably too late)
869870
if (!string_ends_with(window, cond)) {
871+
// LLAMA_LOG_DEBUG("%s[%d]: %d\n", __func__, __LINE__, id);
870872
logits[id] += bias;
871873
}
872874
}
873875
}
874876
}
875877

876-
// expiring sampler bias
878+
// expiring sparam bias
877879
for (auto& entry: ctx_sampling->params.elb_params[ctx_sampling->elb_idx].entries) {
878880
if (!entry.biases.empty()) {
879881
continue; // next entry
@@ -924,18 +926,17 @@ void common_expiring_logit_bias_apply(struct common_sampler* ctx_sampling, float
924926
}
925927
}
926928
}
927-
// printf("%s[%d]: \n", __func__, __LINE__);
928929
}
929930

930931
void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, struct llama_context * ctx_main) {
931-
// printf("%s[%d]: \n", __func__, __LINE__);
932932
const auto idx = ctx_sampling->elb_idx;
933933
if (idx >= ctx_sampling->elb_states.size()) {
934934
ctx_sampling->elb_idx++;
935935
return;
936936
}
937937

938938
auto& elb = ctx_sampling->elb_states[idx];
939+
// LLAMA_LOG_DEBUG("%s[%d]: idx = %d, countup = %zu, %s\n", __func__, __LINE__, idx, elb.countup, string_unescape(common_token_to_piece(ctx_main, ctx_sampling->prev.back(), true)).c_str());
939940
if ((elb.delay > ++(elb.countup)) || (elb.search_word_len == 0)) {
940941
return;
941942
}
@@ -947,16 +948,17 @@ void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, stru
947948
if (string_is_found(window, elb.jumpword, pos)) {
948949
LLAMA_LOG_DEBUG("%s: found %s in %s @ %zu\n", __func__, string_unescape(elb.jumpword).c_str(), string_unescape(window).c_str(), search_pos + pos);
949950
ctx_sampling->elb_idx = elb.jump_idx;
951+
search_pos += pos + elb.jumpword.length();
950952
} else if (string_is_found(window, elb.exitword, pos)) {
951953
LLAMA_LOG_DEBUG("%s: found %s in %s @ %zu\n", __func__, string_unescape(elb.exitword).c_str(), string_unescape(window).c_str(), search_pos + pos);
952954
ctx_sampling->elb_idx++;
955+
search_pos += pos + elb.exitword.length();
953956
} else {
954957
search_pos += std::max(0, int32_t(window.length()) - elb.search_word_len);
955958
return;
956959
}
957-
search_pos += pos;
958960

959-
// undo current sampler bias
961+
// undo current sparam bias
960962
for (auto& entry: ctx_sampling->params.elb_params[idx].entries) {
961963
for (const auto addflag: entry.addflags) {
962964
if (addflag) {
@@ -971,24 +973,18 @@ void common_expiring_logit_bias_accept(struct common_sampler* ctx_sampling, stru
971973
}
972974
}
973975

974-
// prepare next sampler bias
976+
// initialize next stage
977+
ctx_sampling->elb_states[ctx_sampling->elb_idx].init_pos = search_pos;
975978
for (auto& entry: ctx_sampling->params.elb_params[ctx_sampling->elb_idx].entries) {
976979
entry.search_posi.assign(entry.search_posi.size(), search_pos);
977980
}
978-
979-
// single character clearance for exitword
980-
search_pos++;
981-
982-
ctx_sampling->elb_states[ctx_sampling->elb_idx].init_pos = search_pos;
983-
// printf("%s[%d]: \n", __func__, __LINE__);
984981
}
985982

986983
void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) {
987-
printf("%s[%d]: \n", __func__, __LINE__);
988984
auto& idx = ctx_sampling->elb_idx;
989985
auto n_rewind = ctx_sampling->n_rewind;
990986

991-
LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind);
987+
// LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind);
992988

993989
if (idx >= ctx_sampling->elb_states.size()) {
994990
// entirely expired
@@ -997,12 +993,11 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) {
997993
if (n_rewind <= 0) {
998994
// not enough rewind to reanimate
999995
idx -= n_rewind;
1000-
LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind);
1001996
return;
1002997
}
1003998
}
1004999

1005-
LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind);
1000+
// LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind);
10061001

10071002
// consume n_rewind
10081003
while (n_rewind > 0) {
@@ -1020,14 +1015,62 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) {
10201015
}
10211016
LLAMA_LOG_DEBUG("%s[%d]: idx = %d, countup = %zu, n_rewind = %d\n", __func__, __LINE__, idx, countup, n_rewind);
10221017
}
1018+
LLAMA_LOG_DEBUG("%s[%d]: idx = %d, n_rewind = %d\n", __func__, __LINE__, idx, n_rewind);
1019+
1020+
if (idx == 0) {
1021+
// trivial case
1022+
const auto& decoded_text = ctx_sampling->decoded_text;
1023+
auto& elb = ctx_sampling->elb_states[idx];
1024+
std::string window;
1025+
for (auto& entry: ctx_sampling->params.elb_params[idx].entries) {
1026+
if (!entry.biases.empty()) {
1027+
// no need to rewind elb
1028+
continue;
1029+
}
1030+
1031+
for (size_t j = 0; j < entry.phrases.size(); ++j) {
1032+
const auto& phrase = entry.phrases[j];
1033+
if (phrase.empty() && (elb.countup < entry.duration) && !entry.addflags[j]) {
1034+
elb_add(ctx_sampling->params, entry);
1035+
entry.addflags[j] = true;
1036+
continue;
1037+
}
1038+
1039+
std::string_view decoded_tail;
1040+
if (phrase.length() > decoded_text.length()) {
1041+
decoded_tail = decoded_text;
1042+
entry.search_posi[j] = 0;
1043+
} else {
1044+
decoded_tail = decoded_text.substr(decoded_text.length() - phrase.length() + 1);
1045+
entry.search_posi[j] = decoded_text.length() - phrase.length() + 1;
1046+
}
1047+
window.clear();
1048+
window.append(decoded_tail);
1049+
window.append(ctx_sampling->rewinded_text);
1050+
1051+
// triggered within rewinded window?
1052+
size_t count = 0;
1053+
auto pos = window.find(phrase, entry.search_posi[j]);
1054+
while (pos != std::string::npos) {
1055+
++count;
1056+
pos = window.find(phrase, pos + phrase.length());
1057+
}
1058+
if (count % 2 == 1) {
1059+
(entry.addflags[j] ? elb_sub : elb_add)(ctx_sampling->params, entry);
1060+
entry.addflags[j] = !entry.addflags[j];
1061+
}
1062+
}
1063+
}
1064+
ctx_sampling->elb_search_pos = std::max(0, int32_t(decoded_text.length()) - elb.search_word_len + 1);
1065+
} else {
1066+
ctx_sampling->elb_search_pos = ctx_sampling->elb_states[idx].init_pos;
1067+
}
1068+
LLAMA_LOG_DEBUG("%s[%d]: elb_search_pos = %d\n", __func__, __LINE__, ctx_sampling->elb_search_pos);
10231069

1024-
// rewind sparam bias including new current
1025-
for (int32_t j = idx; j < ctx_sampling->elb_states.size(); ++j) {
1026-
printf("%s[%d]: \n", __func__, __LINE__);
1070+
// rewind sparam bias
1071+
for (int32_t j = std::max(1, idx); j < ctx_sampling->elb_states.size(); ++j) {
10271072
for (auto& entry: ctx_sampling->params.elb_params[j].entries) {
1028-
printf("%s[%d]: %zu\n", __func__, __LINE__, entry.addflags.size());
10291073
for (const auto addflag: entry.addflags) {
1030-
printf("%s[%d]: \n", __func__, __LINE__);
10311074
if (addflag) {
10321075
LLAMA_LOG_DEBUG("%s: before\n", __func__);
10331076
elb_print(ctx_sampling->params, entry);
@@ -1040,9 +1083,6 @@ void common_expiring_logit_bias_rewind(struct common_sampler* ctx_sampling) {
10401083
}
10411084
}
10421085
}
1043-
1044-
ctx_sampling->elb_search_pos = ctx_sampling->elb_states[idx].init_pos;
1045-
LLAMA_LOG_DEBUG("%s[%d]: elb_search_pos = %d\n", __func__, __LINE__, ctx_sampling->elb_search_pos);
10461086
}
10471087

10481088

0 commit comments

Comments
 (0)