Skip to content

Commit 1f32472

Browse files
BorisPowerBorisPower
authored
Boris/embedding examples (#49)
* add Olympics Q&A fine-tuning tutorial notebooks * fix batch calculation for very small datasets * Add embeddings tutorial notebooks
1 parent 62f8d40 commit 1f32472

17 files changed

+3933
-4
lines changed

‎examples/embeddings/Classification.ipynb

+130
Large diffs are not rendered by default.

‎examples/embeddings/Clustering.ipynb

+262
Large diffs are not rendered by default.

‎examples/embeddings/Code_search.ipynb

+396
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,396 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"## Code search\n",
8+
"\n",
9+
"We index our own openai-python code repository, and show how it can be searched. We implement a simple version of file parsing and extracting of functions from python files. The dataset is created in the [Obtain_dataset Notebook](Obtain_dataset.ipynb)."
10+
]
11+
},
12+
{
13+
"cell_type": "code",
14+
"execution_count": 1,
15+
"metadata": {},
16+
"outputs": [
17+
{
18+
"name": "stdout",
19+
"output_type": "stream",
20+
"text": [
21+
"Total number of py files: 40\n",
22+
"Total number of functions extracted: 64\n"
23+
]
24+
}
25+
],
26+
"source": [
27+
"import os\n",
28+
"from glob import glob\n",
29+
"import pandas as pd\n",
30+
"\n",
31+
"def get_function_name(code):\n",
32+
" \"\"\"\n",
33+
" Extract function name from a line beginning with \"def \"\n",
34+
" \"\"\"\n",
35+
" assert code.startswith(\"def \")\n",
36+
" return code[len(\"def \"): code.index(\"(\")]\n",
37+
"\n",
38+
"def get_until_no_space(all_lines, i) -> str:\n",
39+
" \"\"\"\n",
40+
" Get all lines until a line outside the function definition is found.\n",
41+
" \"\"\"\n",
42+
" ret = [all_lines[i]]\n",
43+
" for j in range(i + 1, i + 10000):\n",
44+
" if j < len(all_lines):\n",
45+
" if len(all_lines[j]) == 0 or all_lines[j][0] in [\" \", \"\\t\", \")\"]:\n",
46+
" ret.append(all_lines[j])\n",
47+
" else:\n",
48+
" break\n",
49+
" return \"\\n\".join(ret)\n",
50+
"\n",
51+
"def get_functions(filepath):\n",
52+
" \"\"\"\n",
53+
" Get all functions in a Python file.\n",
54+
" \"\"\"\n",
55+
" whole_code = open(filepath).read().replace(\"\\r\", \"\\n\")\n",
56+
" all_lines = whole_code.split(\"\\n\")\n",
57+
" for i, l in enumerate(all_lines):\n",
58+
" if l.startswith(\"def \"):\n",
59+
" code = get_until_no_space(all_lines, i)\n",
60+
" function_name = get_function_name(code)\n",
61+
" yield {\"code\": code, \"function_name\": function_name, \"filepath\": filepath}\n",
62+
"\n",
63+
"\n",
64+
"# get user root directory\n",
65+
"root_dir = os.path.expanduser(\"~\")\n",
66+
"\n",
67+
"# path to code repository directory\n",
68+
"code_root = root_dir + \"/openai-python\"\n",
69+
"code_files = [y for x in os.walk(code_root) for y in glob(os.path.join(x[0], '*.py'))]\n",
70+
"print(\"Total number of py files:\", len(code_files))\n",
71+
"all_funcs = []\n",
72+
"for code_file in code_files:\n",
73+
" funcs = list(get_functions(code_file))\n",
74+
" for func in funcs:\n",
75+
" all_funcs.append(func)\n",
76+
"\n",
77+
"print(\"Total number of functions extracted:\", len(all_funcs))\n"
78+
]
79+
},
80+
{
81+
"cell_type": "markdown",
82+
"metadata": {},
83+
"source": [
84+
"For code search models we use babbage-code-search-code to obtain embeddings for code snippets, and code-search-text to embed natural language queries."
85+
]
86+
},
87+
{
88+
"cell_type": "code",
89+
"execution_count": 2,
90+
"metadata": {},
91+
"outputs": [
92+
{
93+
"data": {
94+
"text/html": [
95+
"<div>\n",
96+
"<style scoped>\n",
97+
" .dataframe tbody tr th:only-of-type {\n",
98+
" vertical-align: middle;\n",
99+
" }\n",
100+
"\n",
101+
" .dataframe tbody tr th {\n",
102+
" vertical-align: top;\n",
103+
" }\n",
104+
"\n",
105+
" .dataframe thead th {\n",
106+
" text-align: right;\n",
107+
" }\n",
108+
"</style>\n",
109+
"<table border=\"1\" class=\"dataframe\">\n",
110+
" <thead>\n",
111+
" <tr style=\"text-align: right;\">\n",
112+
" <th></th>\n",
113+
" <th>code</th>\n",
114+
" <th>function_name</th>\n",
115+
" <th>filepath</th>\n",
116+
" <th>code_embedding</th>\n",
117+
" </tr>\n",
118+
" </thead>\n",
119+
" <tbody>\n",
120+
" <tr>\n",
121+
" <th>0</th>\n",
122+
" <td>def semantic_search(engine, query, documents):...</td>\n",
123+
" <td>semantic_search</td>\n",
124+
" <td>/examples/semanticsearch/semanticsearch.py</td>\n",
125+
" <td>[-0.038976121693849564, -0.0031428150832653046...</td>\n",
126+
" </tr>\n",
127+
" <tr>\n",
128+
" <th>1</th>\n",
129+
" <td>def main():\\n parser = argparse.ArgumentPar...</td>\n",
130+
" <td>main</td>\n",
131+
" <td>/examples/semanticsearch/semanticsearch.py</td>\n",
132+
" <td>[-0.024289356544613838, -0.017748363316059113,...</td>\n",
133+
" </tr>\n",
134+
" <tr>\n",
135+
" <th>2</th>\n",
136+
" <td>def get_candidates(\\n prompt: str,\\n sto...</td>\n",
137+
" <td>get_candidates</td>\n",
138+
" <td>/examples/codex/backtranslation.py</td>\n",
139+
" <td>[-0.04161201789975166, -0.0169310811907053, 0....</td>\n",
140+
" </tr>\n",
141+
" <tr>\n",
142+
" <th>3</th>\n",
143+
" <td>def rindex(lst: List, value: str) -&gt; int:\\n ...</td>\n",
144+
" <td>rindex</td>\n",
145+
" <td>/examples/codex/backtranslation.py</td>\n",
146+
" <td>[-0.027255680412054062, -0.007931121625006199,...</td>\n",
147+
" </tr>\n",
148+
" <tr>\n",
149+
" <th>4</th>\n",
150+
" <td>def eval_candidate(\\n candidate_answer: str...</td>\n",
151+
" <td>eval_candidate</td>\n",
152+
" <td>/examples/codex/backtranslation.py</td>\n",
153+
" <td>[-0.00999179296195507, -0.01640152558684349, 0...</td>\n",
154+
" </tr>\n",
155+
" </tbody>\n",
156+
"</table>\n",
157+
"</div>"
158+
],
159+
"text/plain": [
160+
" code function_name \\\n",
161+
"0 def semantic_search(engine, query, documents):... semantic_search \n",
162+
"1 def main():\\n parser = argparse.ArgumentPar... main \n",
163+
"2 def get_candidates(\\n prompt: str,\\n sto... get_candidates \n",
164+
"3 def rindex(lst: List, value: str) -> int:\\n ... rindex \n",
165+
"4 def eval_candidate(\\n candidate_answer: str... eval_candidate \n",
166+
"\n",
167+
" filepath \\\n",
168+
"0 /examples/semanticsearch/semanticsearch.py \n",
169+
"1 /examples/semanticsearch/semanticsearch.py \n",
170+
"2 /examples/codex/backtranslation.py \n",
171+
"3 /examples/codex/backtranslation.py \n",
172+
"4 /examples/codex/backtranslation.py \n",
173+
"\n",
174+
" code_embedding \n",
175+
"0 [-0.038976121693849564, -0.0031428150832653046... \n",
176+
"1 [-0.024289356544613838, -0.017748363316059113,... \n",
177+
"2 [-0.04161201789975166, -0.0169310811907053, 0.... \n",
178+
"3 [-0.027255680412054062, -0.007931121625006199,... \n",
179+
"4 [-0.00999179296195507, -0.01640152558684349, 0... "
180+
]
181+
},
182+
"execution_count": 2,
183+
"metadata": {},
184+
"output_type": "execute_result"
185+
}
186+
],
187+
"source": [
188+
"from utils import get_embedding\n",
189+
"\n",
190+
"df = pd.DataFrame(all_funcs)\n",
191+
"df['code_embedding'] = df['code'].apply(lambda x: get_embedding(x, engine='babbage-code-search-code'))\n",
192+
"df['filepath'] = df['filepath'].apply(lambda x: x.replace(code_root, \"\"))\n",
193+
"df.to_csv(\"output/code_search_openai-python.csv\", index=False)\n",
194+
"df.head()"
195+
]
196+
},
197+
{
198+
"cell_type": "code",
199+
"execution_count": 5,
200+
"metadata": {},
201+
"outputs": [
202+
{
203+
"name": "stdout",
204+
"output_type": "stream",
205+
"text": [
206+
"/openai/tests/test_endpoints.py:test_completions_multiple_prompts score=0.681\n",
207+
"def test_completions_multiple_prompts():\n",
208+
" result = openai.Completion.create(\n",
209+
" prompt=[\"This was a test\", \"This was another test\"], n=5, engine=\"ada\"\n",
210+
" )\n",
211+
" assert len(result.choices) == 10\n",
212+
"\n",
213+
"----------------------------------------------------------------------\n",
214+
"/openai/tests/test_endpoints.py:test_completions score=0.675\n",
215+
"def test_completions():\n",
216+
" result = openai.Completion.create(prompt=\"This was a test\", n=5, engine=\"ada\")\n",
217+
" assert len(result.choices) == 5\n",
218+
"\n",
219+
"\n",
220+
"----------------------------------------------------------------------\n",
221+
"/openai/tests/test_api_requestor.py:test_requestor_sets_request_id score=0.635\n",
222+
"def test_requestor_sets_request_id(mocker: MockerFixture) -> None:\n",
223+
" # Fake out 'requests' and confirm that the X-Request-Id header is set.\n",
224+
"\n",
225+
" got_headers = {}\n",
226+
"\n",
227+
" def fake_request(self, *args, **kwargs):\n",
228+
" nonlocal got_headers\n",
229+
"----------------------------------------------------------------------\n"
230+
]
231+
}
232+
],
233+
"source": [
234+
"from utils import cosine_similarity\n",
235+
"\n",
236+
"def search_functions(df, code_query, n=3, pprint=True, n_lines=7):\n",
237+
" embedding = get_embedding(code_query, engine='babbage-code-search-text')\n",
238+
" df['similarities'] = df.code_embedding.apply(lambda x: cosine_similarity(x, embedding))\n",
239+
"\n",
240+
" res = df.sort_values('similarities', ascending=False).head(n)\n",
241+
" if pprint:\n",
242+
" for r in res.iterrows():\n",
243+
" print(r[1].filepath+\":\"+r[1].function_name + \" score=\" + str(round(r[1].similarities, 3)))\n",
244+
" print(\"\\n\".join(r[1].code.split(\"\\n\")[:n_lines]))\n",
245+
" print('-'*70)\n",
246+
" return res\n",
247+
"res = search_functions(df, 'Completions API tests', n=3)\n"
248+
]
249+
},
250+
{
251+
"cell_type": "code",
252+
"execution_count": 6,
253+
"metadata": {},
254+
"outputs": [
255+
{
256+
"name": "stdout",
257+
"output_type": "stream",
258+
"text": [
259+
"/openai/validators.py:format_inferrer_validator score=0.655\n",
260+
"def format_inferrer_validator(df):\n",
261+
" \"\"\"\n",
262+
" This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.\n",
263+
" It will also suggest to use ada, --no_packing and explain train/validation split benefits.\n",
264+
" \"\"\"\n",
265+
" ft_type = infer_task_type(df)\n",
266+
" immediate_msg = None\n",
267+
"----------------------------------------------------------------------\n",
268+
"/openai/validators.py:long_examples_validator score=0.649\n",
269+
"def long_examples_validator(df):\n",
270+
" \"\"\"\n",
271+
" This validator will suggest to the user to remove examples that are too long.\n",
272+
" \"\"\"\n",
273+
" immediate_msg = None\n",
274+
" optional_msg = None\n",
275+
" optional_fn = None\n",
276+
"----------------------------------------------------------------------\n",
277+
"/openai/validators.py:non_empty_completion_validator score=0.646\n",
278+
"def non_empty_completion_validator(df):\n",
279+
" \"\"\"\n",
280+
" This validator will ensure that no completion is empty.\n",
281+
" \"\"\"\n",
282+
" necessary_msg = None\n",
283+
" necessary_fn = None\n",
284+
" immediate_msg = None\n",
285+
"----------------------------------------------------------------------\n"
286+
]
287+
}
288+
],
289+
"source": [
290+
"res = search_functions(df, 'fine-tuning input data validation logic', n=3)"
291+
]
292+
},
293+
{
294+
"cell_type": "code",
295+
"execution_count": 7,
296+
"metadata": {},
297+
"outputs": [
298+
{
299+
"name": "stdout",
300+
"output_type": "stream",
301+
"text": [
302+
"/openai/validators.py:common_completion_suffix_validator score=0.665\n",
303+
"def common_completion_suffix_validator(df):\n",
304+
" \"\"\"\n",
305+
" This validator will suggest to add a common suffix to the completion if one doesn't already exist in case of classification or conditional generation.\n",
306+
" \"\"\"\n",
307+
" error_msg = None\n",
308+
" immediate_msg = None\n",
309+
" optional_msg = None\n",
310+
" optional_fn = None\n",
311+
"\n",
312+
" ft_type = infer_task_type(df)\n",
313+
"----------------------------------------------------------------------\n",
314+
"/openai/validators.py:get_outfnames score=0.66\n",
315+
"def get_outfnames(fname, split):\n",
316+
" suffixes = [\"_train\", \"_valid\"] if split else [\"\"]\n",
317+
" i = 0\n",
318+
" while True:\n",
319+
" index_suffix = f\" ({i})\" if i > 0 else \"\"\n",
320+
" candidate_fnames = [\n",
321+
" fname.split(\".\")[0] + \"_prepared\" + suffix + index_suffix + \".jsonl\"\n",
322+
" for suffix in suffixes\n",
323+
" ]\n",
324+
" if not any(os.path.isfile(f) for f in candidate_fnames):\n",
325+
"----------------------------------------------------------------------\n"
326+
]
327+
}
328+
],
329+
"source": [
330+
"res = search_functions(df, 'find common suffix', n=2, n_lines=10)"
331+
]
332+
},
333+
{
334+
"cell_type": "code",
335+
"execution_count": 8,
336+
"metadata": {},
337+
"outputs": [
338+
{
339+
"name": "stdout",
340+
"output_type": "stream",
341+
"text": [
342+
"/openai/cli.py:tools_register score=0.651\n",
343+
"def tools_register(parser):\n",
344+
" subparsers = parser.add_subparsers(\n",
345+
" title=\"Tools\", help=\"Convenience client side tools\"\n",
346+
" )\n",
347+
"\n",
348+
" def help(args):\n",
349+
" parser.print_help()\n",
350+
"\n",
351+
" parser.set_defaults(func=help)\n",
352+
"\n",
353+
" sub = subparsers.add_parser(\"fine_tunes.prepare_data\")\n",
354+
" sub.add_argument(\n",
355+
" \"-f\",\n",
356+
" \"--file\",\n",
357+
" required=True,\n",
358+
" help=\"JSONL, JSON, CSV, TSV, TXT or XLSX file containing prompt-completion examples to be analyzed.\"\n",
359+
" \"This should be the local file path.\",\n",
360+
" )\n",
361+
" sub.add_argument(\n",
362+
" \"-q\",\n",
363+
"----------------------------------------------------------------------\n"
364+
]
365+
}
366+
],
367+
"source": [
368+
"res = search_functions(df, 'Command line interface for fine-tuning', n=1, n_lines=20)"
369+
]
370+
}
371+
],
372+
"metadata": {
373+
"interpreter": {
374+
"hash": "be4b5d5b73a21c599de40d6deb1129796d12dc1cc33a738f7bac13269cfcafe8"
375+
},
376+
"kernelspec": {
377+
"display_name": "Python 3.7.3 64-bit ('base': conda)",
378+
"name": "python3"
379+
},
380+
"language_info": {
381+
"codemirror_mode": {
382+
"name": "ipython",
383+
"version": 3
384+
},
385+
"file_extension": ".py",
386+
"mimetype": "text/x-python",
387+
"name": "python",
388+
"nbconvert_exporter": "python",
389+
"pygments_lexer": "ipython3",
390+
"version": "3.7.3"
391+
},
392+
"orig_nbformat": 4
393+
},
394+
"nbformat": 4,
395+
"nbformat_minor": 2
396+
}

0 commit comments

Comments
 (0)