Skip to content

Commit 438466f

Browse files
generallgenerall
committed
update heuristic to rescore from the are of the closest guess
1 parent bce13dd commit 438466f

2 files changed

Lines changed: 234 additions & 11 deletions

File tree

‎contexto/guess.py‎

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,39 @@ def add_result(self, word, order):
5757
self.results.append((word, order))
5858
self.word_to_distances[word] = self.get_distances(word)
5959

60+
def get_score(self, guesses, word_to_distances, min_gap=0.1, num_samples=50):
61+
scores = np.zeros(len(self.words))
62+
63+
for _ in range(0, num_samples):
64+
65+
word_a, order_a = random.choice(guesses)
66+
word_b, order_b = random.choice(guesses)
67+
68+
if order_a < order_b * (1.0 - min_gap):
69+
scores += (word_to_distances[word_a] - word_to_distances[word_b] < 0)
70+
71+
if order_a > order_b * (1.0 + min_gap):
72+
scores += (word_to_distances[word_a] - word_to_distances[word_b] > 0)
73+
74+
return scores
75+
76+
def best_scores(self, guesses, word_to_distances, top: int):
77+
""" Returns a mask of the top scores of the best guess
78+
"""
79+
best_guess_word, best_guess_rank = sorted(guesses, key=lambda x: x[1])[0]
80+
best_guess_distances = word_to_distances[best_guess_word]
81+
top_distances = np.argsort(best_guess_distances)[:top]
82+
top_distances_mask = np.zeros(len(self.words), dtype=bool)
83+
top_distances_mask[top_distances] = True
84+
return top_distances_mask
85+
86+
def already_guessed_mask(self, guesses):
87+
already_guessed_mask = np.zeros(len(self.words), dtype=bool)
88+
for word, _ in guesses:
89+
word_id = self.words.index(word)
90+
already_guessed_mask[word_id] = True
91+
return already_guessed_mask
92+
6093
def sample_score(self, min_gap=0.1, num_samples=50, guesses=None, word_to_distances=None):
6194
if guesses is None:
6295
guesses = self.results
@@ -70,18 +103,13 @@ def sample_score(self, min_gap=0.1, num_samples=50, guesses=None, word_to_distan
70103
if word in word_to_distances
71104
]
72105

73-
scores = np.zeros(len(self.words))
74-
75-
for _ in range(0, num_samples):
76-
77-
word_a, order_a = random.choice(guesses)
78-
word_b, order_b = random.choice(guesses)
106+
scores = self.get_score(guesses, word_to_distances, min_gap=min_gap, num_samples=num_samples)
79107

80-
if order_a < order_b * (1.0 - min_gap):
81-
scores += (word_to_distances[word_a] - word_to_distances[word_b] < 0)
108+
already_guessed_masked = self.already_guessed_mask(guesses)
109+
scores[already_guessed_masked] = 0
82110

83-
if order_a > order_b * (1.0 + min_gap):
84-
scores += (word_to_distances[word_a] - word_to_distances[word_b] > 0)
111+
best_scores_masked = self.best_scores(guesses, word_to_distances, top=100)
112+
scores[~best_scores_masked] = 0
85113

86114
top_score = max(scores)
87115
# top_score = np.percentile(scores, 90)
@@ -102,7 +130,7 @@ def sample_score(self, min_gap=0.1, num_samples=50, guesses=None, word_to_distan
102130
return closest_ids
103131

104132
def guess_next(self, guesses=None, word_to_distances=None):
105-
closest = self.sample_score(min_gap=0.5, num_samples=500, guesses=guesses, word_to_distances=word_to_distances)
133+
closest = self.sample_score(min_gap=0.3, num_samples=500, guesses=guesses, word_to_distances=word_to_distances)
106134
return self.words[closest[0]]
107135

108136

@@ -139,6 +167,8 @@ def signal_handler(sig, frame):
139167
print(next_word)
140168
try:
141169
order = int(input('Order: '))
170+
if order == 0:
171+
order = None
142172
except ValueError as e:
143173
print('Invalid order')
144174
continue

‎notebooks/run.ipynb‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 27,
6+
"id": "67e53669",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"!pip install seaborn"
11+
]
12+
},
13+
{
14+
"cell_type": "code",
15+
"execution_count": 1,
16+
"id": "fccce30e-725a-4c35-9382-b177232de919",
17+
"metadata": {},
18+
"outputs": [],
19+
"source": [
20+
"import sys\n",
21+
"sys.path.append('..')"
22+
]
23+
},
24+
{
25+
"cell_type": "code",
26+
"execution_count": 128,
27+
"id": "e4620afc",
28+
"metadata": {},
29+
"outputs": [],
30+
"source": [
31+
"from contexto.guess import Finder\n",
32+
"import numpy as np\n",
33+
"import seaborn as sns\n",
34+
"from sklearn.manifold import TSNE\n",
35+
"import matplotlib.pyplot as plt\n"
36+
]
37+
},
38+
{
39+
"cell_type": "code",
40+
"execution_count": 130,
41+
"id": "7a99fd91",
42+
"metadata": {},
43+
"outputs": [],
44+
"source": [
45+
"embedding = TSNE(n_components=2, learning_rate='auto', metric=\"cosine\").fit_transform(Finder().embeddings)"
46+
]
47+
},
48+
{
49+
"cell_type": "code",
50+
"execution_count": 281,
51+
"id": "20bb23f7",
52+
"metadata": {},
53+
"outputs": [],
54+
"source": [
55+
"def plot(scores, save_id=None):\n",
56+
" is_selected = scores > 0\n",
57+
"\n",
58+
" plt.figure(figsize=(15,8))\n",
59+
"\n",
60+
" not_selected_embedding = embedding[~is_selected]\n",
61+
" selected_embedding = embedding[is_selected]\n",
62+
"\n",
63+
" top_score = max(scores)\n",
64+
"\n",
65+
" top_embedding = embedding[scores == top_score][:1]\n",
66+
"\n",
67+
"\n",
68+
" ax = sns.scatterplot(\n",
69+
" x=not_selected_embedding[:, 0],\n",
70+
" y=not_selected_embedding[:, 1],\n",
71+
" legend=\"\",\n",
72+
" size=1,\n",
73+
" hue=0.1,\n",
74+
" )\n",
75+
"\n",
76+
" ax = sns.scatterplot(\n",
77+
" x=selected_embedding[:, 0],\n",
78+
" y=selected_embedding[:, 1],\n",
79+
" size=scores[is_selected],\n",
80+
" # hue=scores[is_selected],\n",
81+
" hue=0,\n",
82+
" legend=\"\",\n",
83+
" ax=ax\n",
84+
" )\n",
85+
"\n",
86+
" ax = sns.scatterplot(\n",
87+
" x=top_embedding[:, 0],\n",
88+
" y=top_embedding[:, 1],\n",
89+
" ax=ax,\n",
90+
" )\n",
91+
"\n",
92+
" ax = sns.scatterplot(\n",
93+
" x=top_embedding[:, 0],\n",
94+
" y=top_embedding[:, 1],\n",
95+
" ax=ax,\n",
96+
" )\n",
97+
"\n",
98+
" plt.tight_layout()\n",
99+
"\n",
100+
"\n",
101+
" # remove axis\n",
102+
" ax.set_xticks([])\n",
103+
" ax.set_yticks([])\n",
104+
"\n",
105+
" if save_id is not None:\n",
106+
" plt.savefig(f'../plots/{save_id:02d}.png')\n",
107+
" else:\n",
108+
" plt.show()"
109+
]
110+
},
111+
{
112+
"cell_type": "code",
113+
"execution_count": 275,
114+
"id": "8bb22475",
115+
"metadata": {},
116+
"outputs": [],
117+
"source": [
118+
"def score(guesses):\n",
119+
" finder = Finder()\n",
120+
" for guess, rank in guesses:\n",
121+
" finder.add_result(guess, rank)\n",
122+
" scores = finder.get_score(\n",
123+
" guesses=guesses,\n",
124+
" word_to_distances=finder.word_to_distances,\n",
125+
" min_gap=0.2,\n",
126+
" num_samples=1000\n",
127+
" )\n",
128+
" already_guessed_masked = finder.already_guessed_mask(guesses)\n",
129+
" scores[already_guessed_masked] = 0\n",
130+
" best_scores_masked = finder.best_scores(guesses=guesses, word_to_distances=finder.word_to_distances, top=100)\n",
131+
" scores[~best_scores_masked] = 0\n",
132+
"\n",
133+
" top_score = max(scores)\n",
134+
" closest_ids = list(np.arange(len(finder.words))[scores == top_score])\n",
135+
" return scores, [finder.words[idx] for idx in closest_ids]\n"
136+
]
137+
},
138+
{
139+
"cell_type": "code",
140+
"execution_count": 276,
141+
"id": "470dc398",
142+
"metadata": {},
143+
"outputs": [],
144+
"source": [
145+
"def process(guesses):\n",
146+
" scores, suggestion = score(guesses)\n",
147+
" print(\"Guesses: \", suggestion[:5])\n",
148+
" plot(scores)\n"
149+
]
150+
},
151+
{
152+
"cell_type": "code",
153+
"execution_count": null,
154+
"id": "4dff06d4",
155+
"metadata": {},
156+
"outputs": [],
157+
"source": [
158+
"process([\n",
159+
" ('human', ...),\n",
160+
"])"
161+
]
162+
},
163+
{
164+
"cell_type": "code",
165+
"execution_count": null,
166+
"id": "51a15483",
167+
"metadata": {},
168+
"outputs": [],
169+
"source": []
170+
}
171+
],
172+
"metadata": {
173+
"kernelspec": {
174+
"display_name": "Python 3 (ipykernel)",
175+
"language": "python",
176+
"name": "python3"
177+
},
178+
"language_info": {
179+
"codemirror_mode": {
180+
"name": "ipython",
181+
"version": 3
182+
},
183+
"file_extension": ".py",
184+
"mimetype": "text/x-python",
185+
"name": "python",
186+
"nbconvert_exporter": "python",
187+
"pygments_lexer": "ipython3",
188+
"version": "3.10.6"
189+
}
190+
},
191+
"nbformat": 4,
192+
"nbformat_minor": 5
193+
}

0 commit comments

Comments
 (0)