From 7af5079b9dee416b455d82c5bd28a82deac7b396 Mon Sep 17 00:00:00 2001 From: <> Date: Tue, 17 Sep 2024 15:50:02 +0000 Subject: [PATCH] Deployed 3e9ae81 with MkDocs version: 1.6.1 --- .nojekyll | 0 404.html | 3287 ++++++++ LICENSE/index.html | 3332 ++++++++ assets/_mkdocstrings.css | 119 + assets/images/favicon.png | Bin 0 -> 1870 bytes assets/javascripts/bundle.56dfad97.min.js | 16 + assets/javascripts/bundle.56dfad97.min.js.map | 7 + assets/javascripts/lunr/min/lunr.ar.min.js | 1 + assets/javascripts/lunr/min/lunr.da.min.js | 18 + assets/javascripts/lunr/min/lunr.de.min.js | 18 + assets/javascripts/lunr/min/lunr.du.min.js | 18 + assets/javascripts/lunr/min/lunr.el.min.js | 1 + assets/javascripts/lunr/min/lunr.es.min.js | 18 + assets/javascripts/lunr/min/lunr.fi.min.js | 18 + assets/javascripts/lunr/min/lunr.fr.min.js | 18 + assets/javascripts/lunr/min/lunr.he.min.js | 1 + assets/javascripts/lunr/min/lunr.hi.min.js | 1 + assets/javascripts/lunr/min/lunr.hu.min.js | 18 + assets/javascripts/lunr/min/lunr.hy.min.js | 1 + assets/javascripts/lunr/min/lunr.it.min.js | 18 + assets/javascripts/lunr/min/lunr.ja.min.js | 1 + assets/javascripts/lunr/min/lunr.jp.min.js | 1 + assets/javascripts/lunr/min/lunr.kn.min.js | 1 + assets/javascripts/lunr/min/lunr.ko.min.js | 1 + assets/javascripts/lunr/min/lunr.multi.min.js | 1 + assets/javascripts/lunr/min/lunr.nl.min.js | 18 + assets/javascripts/lunr/min/lunr.no.min.js | 18 + assets/javascripts/lunr/min/lunr.pt.min.js | 18 + assets/javascripts/lunr/min/lunr.ro.min.js | 18 + assets/javascripts/lunr/min/lunr.ru.min.js | 18 + assets/javascripts/lunr/min/lunr.sa.min.js | 1 + .../lunr/min/lunr.stemmer.support.min.js | 1 + assets/javascripts/lunr/min/lunr.sv.min.js | 18 + assets/javascripts/lunr/min/lunr.ta.min.js | 1 + assets/javascripts/lunr/min/lunr.te.min.js | 1 + assets/javascripts/lunr/min/lunr.th.min.js | 1 + assets/javascripts/lunr/min/lunr.tr.min.js | 18 + assets/javascripts/lunr/min/lunr.vi.min.js | 1 + assets/javascripts/lunr/min/lunr.zh.min.js | 1 + assets/javascripts/lunr/tinyseg.js | 206 + assets/javascripts/lunr/wordcut.js | 6708 ++++++++++++++++ .../workers/search.07f07601.min.js | 42 + .../workers/search.07f07601.min.js.map | 7 + assets/stylesheets/main.35f28582.min.css | 1 + assets/stylesheets/main.35f28582.min.css.map | 1 + assets/stylesheets/palette.06af60db.min.css | 1 + .../stylesheets/palette.06af60db.min.css.map | 1 + docs/about/index.html | 3394 ++++++++ docs/add_new_api/index.html | 3446 ++++++++ docs/anthropic/index.html | 3395 ++++++++ docs/azure_openai/index.html | 3399 ++++++++ docs/commands/index.html | 3610 +++++++++ docs/contribution/index.html | 3646 +++++++++ docs/environment_variables/index.html | 3405 ++++++++ docs/evaluation/index.html | 3697 +++++++++ docs/experiment_file/index.html | 3360 ++++++++ docs/gemini/index.html | 3394 ++++++++ docs/gen_ref_pages.py | 65 + docs/huggingface_tgi/index.html | 3400 ++++++++ docs/index.html | 3440 ++++++++ docs/models/index.html | 3443 ++++++++ docs/ollama/index.html | 3395 ++++++++ docs/openai/index.html | 3395 ++++++++ docs/pipeline/index.html | 3460 ++++++++ docs/prompto.png | Bin 0 -> 1598871 bytes docs/quart/index.html | 3419 ++++++++ docs/rate_limits/index.html | 3748 +++++++++ docs/vertexai/index.html | 3396 ++++++++ examples/anthropic/anthropic/index.html | 4482 +++++++++++ .../data/input/anthropic-example.jsonl | 5 + ...11-24-23-completed-anthropic-example.jsonl | 5 + ...024-11-24-23-input-anthropic-example.jsonl | 5 + ...07-2024-11-24-23-log-anthropic-example.txt | 1 + examples/anthropic/index.html | 3343 ++++++++ examples/azure-openai/azure-openai/index.html | 4542 +++++++++++ .../data/input/azure-openai-example.jsonl | 5 + ...28-03-completed-azure-openai-example.jsonl | 5 + ...-18-28-03-input-azure-openai-example.jsonl | 5 + ...2024-18-28-03-log-azure-openai-example.txt | 16 + examples/azure-openai/index.html | 3347 ++++++++ .../index.html | 4173 ++++++++++ examples/evaluation/completed_example.jsonl | 3 + .../data/input/evaluation-example.jsonl | 2 + ...8-01-11-completed-evaluation-example.jsonl | 2 + ...24-18-01-11-input-evaluation-example.jsonl | 2 + ...8-2024-18-01-11-log-evaluation-example.txt | 1 + ...024-18-05-34-completed-judge-example.jsonl | 6 + ...09-2024-18-05-34-input-judge-example.jsonl | 6 + .../11-09-2024-18-05-34-log-judge-example.txt | 1 + examples/evaluation/judge/settings.json | 5 + examples/evaluation/judge/template.txt | 4 + examples/evaluation/judge/template2.txt | 3 + .../index.html | 5142 ++++++++++++ .../gemini/data/input/gemini-example.jsonl | 6 + ...24-20-17-17-completed-gemini-example.jsonl | 6 + ...6-2024-20-17-17-input-gemini-example.jsonl | 6 + ...25-06-2024-20-17-17-log-gemini-example.txt | 2 + examples/gemini/gemini/index.html | 4578 +++++++++++ examples/gemini/index.html | 3344 ++++++++ examples/index.html | 3344 ++++++++ examples/notebooks/data2/input/test.jsonl | 3 + examples/notebooks/data2/input/test2.jsonl | 3 + .../25-06-2024-19-14-47-completed-test.jsonl | 3 + .../test/25-06-2024-19-14-47-input-test.jsonl | 3 + .../test/25-06-2024-19-14-47-log-test.txt | 4 + .../25-06-2024-19-15-29-completed-test2.jsonl | 3 + .../25-06-2024-19-15-29-input-test2.jsonl | 3 + .../test2/25-06-2024-19-15-29-log-test2.txt | 1 + .../index.html | 6350 +++++++++++++++ .../input/documentation_example.jsonl | 12 + .../documentation_example_groups_1.jsonl | 12 + .../documentation_example_groups_2.jsonl | 16 + .../notebooks/running_experiments/index.html | 5561 +++++++++++++ .../ollama/data/input/ollama-example.jsonl | 6 + ...24-18-43-36-completed-ollama-example.jsonl | 6 + ...6-2024-18-43-36-input-ollama-example.jsonl | 6 + ...25-06-2024-18-43-36-log-ollama-example.txt | 2 + examples/ollama/index.html | 3344 ++++++++ examples/ollama/ollama/index.html | 4507 +++++++++++ .../openai/data/input/openai-example.jsonl | 5 + ...24-19-56-26-completed-openai-example.jsonl | 5 + ...6-2024-19-56-26-input-openai-example.jsonl | 5 + ...25-06-2024-19-56-26-log-openai-example.txt | 1 + examples/openai/index.html | 3343 ++++++++ examples/openai/openai/index.html | 4434 +++++++++++ examples/quart/quart.jsonl | 3 + .../alpaca_sample_generation/index.html | 4365 +++++++++++ examples/system-demo/api_utils.py | 109 + ...4-18-05-54-completed-all_experiments.jsonl | 300 + ...-2024-18-05-54-input-all_experiments.jsonl | 300 + ...0-07-2024-18-05-54-log-all_experiments.txt | 5 + ...09-07-2024-19-59-34-completed-gemini.jsonl | 100 + .../09-07-2024-19-59-34-input-gemini.jsonl | 100 + .../gemini/09-07-2024-19-59-34-log-gemini.txt | 5 + ...09-07-2024-20-04-21-completed-ollama.jsonl | 100 + .../09-07-2024-20-04-21-input-ollama.jsonl | 100 + .../ollama/09-07-2024-20-04-21-log-ollama.txt | 1 + ...8-07-completed-openai-gpt-3pt5-turbo.jsonl | 100 + ...17-08-07-input-openai-gpt-3pt5-turbo.jsonl | 100 + ...024-17-08-07-log-openai-gpt-3pt5-turbo.txt | 1 + ...2024-17-14-54-completed-openai-gpt-4.jsonl | 100 + ...-07-2024-17-14-54-input-openai-gpt-4.jsonl | 100 + .../10-07-2024-17-14-54-log-openai-gpt-4.txt | 1 + ...024-17-19-15-completed-openai-gpt-4o.jsonl | 100 + ...07-2024-17-19-15-input-openai-gpt-4o.jsonl | 100 + .../10-07-2024-17-19-15-log-openai-gpt-4o.txt | 1 + ...-18-completed-openai-multiple-models.jsonl | 300 + ...7-31-18-input-openai-multiple-models.jsonl | 300 + ...24-17-31-18-log-openai-multiple-models.txt | 1 + ...09-07-2024-19-56-36-completed-openai.jsonl | 100 + .../09-07-2024-19-56-36-input-openai.jsonl | 100 + .../openai/09-07-2024-19-56-36-log-openai.txt | 1 + examples/system-demo/dataset_utils.py | 95 + examples/system-demo/experiment_1/index.html | 5248 +++++++++++++ examples/system-demo/experiment_2/index.html | 4568 +++++++++++ examples/system-demo/experiment_3/index.html | 5607 +++++++++++++ examples/system-demo/index.html | 3398 ++++++++ examples/system-demo/sample_prompts.json | 102 + .../data/input/vertexai-example.jsonl | 6 + ...-10-57-06-completed-vertexai-example.jsonl | 6 + ...2024-10-57-06-input-vertexai-example.jsonl | 6 + ...-06-2024-10-57-06-log-vertexai-example.txt | 2 + examples/vertexai/index.html | 3353 ++++++++ examples/vertexai/vertexai/index.html | 4608 +++++++++++ index.html | 3686 +++++++++ objects.inv | Bin 0 -> 1617 bytes reference/DOC_STRINGS/index.html | 3385 ++++++++ .../apis/anthropic/anthropic/index.html | 4925 ++++++++++++ .../apis/anthropic/anthropic_utils/index.html | 3550 +++++++++ .../src/prompto/apis/anthropic/index.html | 3368 ++++++++ .../apis/azure_openai/azure_openai/index.html | 5003 ++++++++++++ .../src/prompto/apis/azure_openai/index.html | 3368 ++++++++ reference/src/prompto/apis/base/index.html | 4318 ++++++++++ .../src/prompto/apis/gemini/gemini/index.html | 5310 +++++++++++++ .../apis/gemini/gemini_utils/index.html | 4001 ++++++++++ reference/src/prompto/apis/gemini/index.html | 3368 ++++++++ .../huggingface_tgi/index.html | 4723 +++++++++++ .../prompto/apis/huggingface_tgi/index.html | 3368 ++++++++ reference/src/prompto/apis/index.html | 3366 ++++++++ reference/src/prompto/apis/ollama/index.html | 3368 ++++++++ .../src/prompto/apis/ollama/ollama/index.html | 4889 ++++++++++++ .../apis/ollama/ollama_utils/index.html | 3557 +++++++++ reference/src/prompto/apis/openai/index.html | 3368 ++++++++ .../src/prompto/apis/openai/openai/index.html | 4894 ++++++++++++ .../apis/openai/openai_utils/index.html | 3568 +++++++++ reference/src/prompto/apis/quart/index.html | 3368 ++++++++ .../src/prompto/apis/quart/quart/index.html | 4572 +++++++++++ .../prompto/apis/quart/quart_api/index.html | 3413 ++++++++ .../prompto/apis/quart/quart_utils/index.html | 3599 +++++++++ reference/src/prompto/apis/testing/index.html | 3368 ++++++++ .../apis/testing/testing_api/index.html | 3413 ++++++++ .../src/prompto/apis/vertexai/index.html | 3368 ++++++++ .../prompto/apis/vertexai/vertexai/index.html | 5371 +++++++++++++ .../apis/vertexai/vertexai_utils/index.html | 3878 +++++++++ reference/src/prompto/experiment/index.html | 6933 +++++++++++++++++ .../prompto/experiment_pipeline/index.html | 4184 ++++++++++ reference/src/prompto/index.html | 3364 ++++++++ reference/src/prompto/judge/index.html | 4877 ++++++++++++ reference/src/prompto/scorer/index.html | 3869 +++++++++ .../scripts/check_experiment/index.html | 3810 +++++++++ .../prompto/scripts/convert_images/index.html | 3411 ++++++++ .../scripts/create_judge_file/index.html | 3623 +++++++++ reference/src/prompto/scripts/index.html | 3366 ++++++++ .../obtain_missing_id_jsonl/index.html | 3787 +++++++++ .../prompto/scripts/run_experiment/index.html | 4949 ++++++++++++ .../prompto/scripts/run_pipeline/index.html | 3654 +++++++++ reference/src/prompto/settings/index.html | 4341 +++++++++++ reference/src/prompto/utils/index.html | 6324 +++++++++++++++ reference/tests/apis/anthropic/index.html | 3409 ++++++++ reference/tests/apis/gemini/index.html | 3409 ++++++++ reference/tests/apis/index.html | 3364 ++++++++ reference/tests/apis/ollama/index.html | 3393 ++++++++ reference/tests/apis/vertexai/index.html | 3409 ++++++++ reference/tests/core/index.html | 3407 ++++++++ reference/tests/index.html | 3362 ++++++++ reference/tests/scripts/index.html | 3407 ++++++++ search/search_index.json | 1 + sitemap.xml | 3 + sitemap.xml.gz | Bin 0 -> 127 bytes src/prompto/__init__.py | 6 + src/prompto/apis/__init__.py | 134 + src/prompto/apis/anthropic/__init__.py | 3 + src/prompto/apis/anthropic/anthropic.py | 432 + src/prompto/apis/anthropic/anthropic_utils.py | 27 + src/prompto/apis/azure_openai/__init__.py | 3 + src/prompto/apis/azure_openai/azure_openai.py | 450 ++ src/prompto/apis/base.py | 109 + src/prompto/apis/gemini/__init__.py | 3 + src/prompto/apis/gemini/gemini.py | 612 ++ src/prompto/apis/gemini/gemini_utils.py | 120 + src/prompto/apis/huggingface_tgi/__init__.py | 3 + .../apis/huggingface_tgi/huggingface_tgi.py | 352 + src/prompto/apis/ollama/__init__.py | 3 + src/prompto/apis/ollama/ollama.py | 410 + src/prompto/apis/ollama/ollama_utils.py | 29 + src/prompto/apis/openai/__init__.py | 3 + src/prompto/apis/openai/openai.py | 413 + src/prompto/apis/openai/openai_utils.py | 37 + src/prompto/apis/quart/__init__.py | 3 + src/prompto/apis/quart/quart.py | 266 + src/prompto/apis/quart/quart_api.py | 90 + src/prompto/apis/quart/quart_utils.py | 36 + src/prompto/apis/testing/__init__.py | 3 + src/prompto/apis/testing/testing_api.py | 69 + src/prompto/apis/vertexai/__init__.py | 3 + src/prompto/apis/vertexai/vertexai.py | 635 ++ src/prompto/apis/vertexai/vertexai_utils.py | 99 + src/prompto/experiment.py | 722 ++ src/prompto/experiment_pipeline.py | 138 + src/prompto/judge.py | 263 + src/prompto/scorer.py | 92 + src/prompto/scripts/__init__.py | 0 src/prompto/scripts/check_experiment.py | 294 + src/prompto/scripts/convert_images.py | 44 + src/prompto/scripts/create_judge_file.py | 104 + .../scripts/obtain_missing_id_jsonl.py | 125 + src/prompto/scripts/run_experiment.py | 477 ++ src/prompto/scripts/run_pipeline.py | 116 + src/prompto/settings.py | 228 + src/prompto/utils.py | 569 ++ tests/__init__.py | 0 tests/apis/__init__.py | 0 tests/apis/anthropic/__init__.py | 0 tests/apis/anthropic/test_anthropic.py | 579 ++ .../anthropic/test_anthropic_chat_input.py | 220 + .../anthropic/test_anthropic_history_input.py | 302 + .../anthropic/test_anthropic_string_input.py | 123 + tests/apis/gemini/__init__.py | 0 tests/apis/gemini/test_gemini.py | 694 ++ tests/apis/gemini/test_gemini_chat_input.py | 402 + .../apis/gemini/test_gemini_history_input.py | 421 + tests/apis/gemini/test_gemini_string_input.py | 179 + tests/apis/ollama/__init__.py | 0 tests/apis/test_base.py | 81 + tests/apis/test_import_warnings.py | 169 + tests/apis/test_imports.py | 66 + tests/apis/vertexai/__init__.py | 0 tests/apis/vertexai/test_vertexai.py | 726 ++ .../apis/vertexai/test_vertexai_chat_input.py | 394 + .../vertexai/test_vertexai_history_input.py | 499 ++ .../vertexai/test_vertexai_string_input.py | 205 + tests/conftest.py | 648 ++ tests/core/__init__.py | 0 tests/core/test_experiment.py | 2742 +++++++ tests/core/test_experiment_generate_text.py | 322 + tests/core/test_experiment_pipeline.py | 160 + tests/core/test_experiment_process.py | 782 ++ tests/core/test_experiment_query_model.py | 564 ++ tests/core/test_experiment_send_requests.py | 585 ++ tests/core/test_judge.py | 579 ++ tests/core/test_scorer.py | 126 + tests/core/test_settings.py | 511 ++ tests/core/test_utils.py | 722 ++ tests/scripts/__init__.py | 0 tests/scripts/test_create_judge_file.py | 435 ++ tests/scripts/test_run_experiment.py | 768 ++ 296 files changed, 400767 insertions(+) create mode 100644 .nojekyll create mode 100644 404.html create mode 100644 LICENSE/index.html create mode 100644 assets/_mkdocstrings.css create mode 100644 assets/images/favicon.png create mode 100644 assets/javascripts/bundle.56dfad97.min.js create mode 100644 assets/javascripts/bundle.56dfad97.min.js.map create mode 100644 assets/javascripts/lunr/min/lunr.ar.min.js create mode 100644 assets/javascripts/lunr/min/lunr.da.min.js create mode 100644 assets/javascripts/lunr/min/lunr.de.min.js create mode 100644 assets/javascripts/lunr/min/lunr.du.min.js create mode 100644 assets/javascripts/lunr/min/lunr.el.min.js create mode 100644 assets/javascripts/lunr/min/lunr.es.min.js create mode 100644 assets/javascripts/lunr/min/lunr.fi.min.js create mode 100644 assets/javascripts/lunr/min/lunr.fr.min.js create mode 100644 assets/javascripts/lunr/min/lunr.he.min.js create mode 100644 assets/javascripts/lunr/min/lunr.hi.min.js create mode 100644 assets/javascripts/lunr/min/lunr.hu.min.js create mode 100644 assets/javascripts/lunr/min/lunr.hy.min.js create mode 100644 assets/javascripts/lunr/min/lunr.it.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ja.min.js create mode 100644 assets/javascripts/lunr/min/lunr.jp.min.js create mode 100644 assets/javascripts/lunr/min/lunr.kn.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ko.min.js create mode 100644 assets/javascripts/lunr/min/lunr.multi.min.js create mode 100644 assets/javascripts/lunr/min/lunr.nl.min.js create mode 100644 assets/javascripts/lunr/min/lunr.no.min.js create mode 100644 assets/javascripts/lunr/min/lunr.pt.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ro.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ru.min.js create mode 100644 assets/javascripts/lunr/min/lunr.sa.min.js create mode 100644 assets/javascripts/lunr/min/lunr.stemmer.support.min.js create mode 100644 assets/javascripts/lunr/min/lunr.sv.min.js create mode 100644 assets/javascripts/lunr/min/lunr.ta.min.js create mode 100644 assets/javascripts/lunr/min/lunr.te.min.js create mode 100644 assets/javascripts/lunr/min/lunr.th.min.js create mode 100644 assets/javascripts/lunr/min/lunr.tr.min.js create mode 100644 assets/javascripts/lunr/min/lunr.vi.min.js create mode 100644 assets/javascripts/lunr/min/lunr.zh.min.js create mode 100644 assets/javascripts/lunr/tinyseg.js create mode 100644 assets/javascripts/lunr/wordcut.js create mode 100644 assets/javascripts/workers/search.07f07601.min.js create mode 100644 assets/javascripts/workers/search.07f07601.min.js.map create mode 100644 assets/stylesheets/main.35f28582.min.css create mode 100644 assets/stylesheets/main.35f28582.min.css.map create mode 100644 assets/stylesheets/palette.06af60db.min.css create mode 100644 assets/stylesheets/palette.06af60db.min.css.map create mode 100644 docs/about/index.html create mode 100644 docs/add_new_api/index.html create mode 100644 docs/anthropic/index.html create mode 100644 docs/azure_openai/index.html create mode 100644 docs/commands/index.html create mode 100644 docs/contribution/index.html create mode 100644 docs/environment_variables/index.html create mode 100644 docs/evaluation/index.html create mode 100644 docs/experiment_file/index.html create mode 100644 docs/gemini/index.html create mode 100644 docs/gen_ref_pages.py create mode 100644 docs/huggingface_tgi/index.html create mode 100644 docs/index.html create mode 100644 docs/models/index.html create mode 100644 docs/ollama/index.html create mode 100644 docs/openai/index.html create mode 100644 docs/pipeline/index.html create mode 100644 docs/prompto.png create mode 100644 docs/quart/index.html create mode 100644 docs/rate_limits/index.html create mode 100644 docs/vertexai/index.html create mode 100644 examples/anthropic/anthropic/index.html create mode 100644 examples/anthropic/data/input/anthropic-example.jsonl create mode 100644 examples/anthropic/data/output/anthropic-example/16-07-2024-11-24-23-completed-anthropic-example.jsonl create mode 100644 examples/anthropic/data/output/anthropic-example/16-07-2024-11-24-23-input-anthropic-example.jsonl create mode 100644 examples/anthropic/data/output/anthropic-example/16-07-2024-11-24-23-log-anthropic-example.txt create mode 100644 examples/anthropic/index.html create mode 100644 examples/azure-openai/azure-openai/index.html create mode 100644 examples/azure-openai/data/input/azure-openai-example.jsonl create mode 100644 examples/azure-openai/data/output/azure-openai-example/25-06-2024-18-28-03-completed-azure-openai-example.jsonl create mode 100644 examples/azure-openai/data/output/azure-openai-example/25-06-2024-18-28-03-input-azure-openai-example.jsonl create mode 100644 examples/azure-openai/data/output/azure-openai-example/25-06-2024-18-28-03-log-azure-openai-example.txt create mode 100644 examples/azure-openai/index.html create mode 100644 examples/evaluation/Running_experiments_with_custom_evaluations/index.html create mode 100644 examples/evaluation/completed_example.jsonl create mode 100644 examples/evaluation/data/input/evaluation-example.jsonl create mode 100644 examples/evaluation/data/output/evaluation-example/28-08-2024-18-01-11-completed-evaluation-example.jsonl create mode 100644 examples/evaluation/data/output/evaluation-example/28-08-2024-18-01-11-input-evaluation-example.jsonl create mode 100644 examples/evaluation/data/output/evaluation-example/28-08-2024-18-01-11-log-evaluation-example.txt create mode 100644 examples/evaluation/data/output/judge-example/11-09-2024-18-05-34-completed-judge-example.jsonl create mode 100644 examples/evaluation/data/output/judge-example/11-09-2024-18-05-34-input-judge-example.jsonl create mode 100644 examples/evaluation/data/output/judge-example/11-09-2024-18-05-34-log-judge-example.txt create mode 100644 examples/evaluation/judge/settings.json create mode 100644 examples/evaluation/judge/template.txt create mode 100644 examples/evaluation/judge/template2.txt create mode 100644 examples/evaluation/running_llm_as_judge_experiment/index.html create mode 100644 examples/gemini/data/input/gemini-example.jsonl create mode 100644 examples/gemini/data/output/gemini-example/25-06-2024-20-17-17-completed-gemini-example.jsonl create mode 100644 examples/gemini/data/output/gemini-example/25-06-2024-20-17-17-input-gemini-example.jsonl create mode 100644 examples/gemini/data/output/gemini-example/25-06-2024-20-17-17-log-gemini-example.txt create mode 100644 examples/gemini/gemini/index.html create mode 100644 examples/gemini/index.html create mode 100644 examples/index.html create mode 100644 examples/notebooks/data2/input/test.jsonl create mode 100644 examples/notebooks/data2/input/test2.jsonl create mode 100644 examples/notebooks/data2/output/test/25-06-2024-19-14-47-completed-test.jsonl create mode 100644 examples/notebooks/data2/output/test/25-06-2024-19-14-47-input-test.jsonl create mode 100644 examples/notebooks/data2/output/test/25-06-2024-19-14-47-log-test.txt create mode 100644 examples/notebooks/data2/output/test2/25-06-2024-19-15-29-completed-test2.jsonl create mode 100644 examples/notebooks/data2/output/test2/25-06-2024-19-15-29-input-test2.jsonl create mode 100644 examples/notebooks/data2/output/test2/25-06-2024-19-15-29-log-test2.txt create mode 100644 examples/notebooks/grouping_prompts_and_specifying_rate_limits/index.html create mode 100644 examples/notebooks/parallel_data_example/input/documentation_example.jsonl create mode 100644 examples/notebooks/parallel_data_example/input/documentation_example_groups_1.jsonl create mode 100644 examples/notebooks/parallel_data_example/input/documentation_example_groups_2.jsonl create mode 100644 examples/notebooks/running_experiments/index.html create mode 100644 examples/ollama/data/input/ollama-example.jsonl create mode 100644 examples/ollama/data/output/ollama-example/25-06-2024-18-43-36-completed-ollama-example.jsonl create mode 100644 examples/ollama/data/output/ollama-example/25-06-2024-18-43-36-input-ollama-example.jsonl create mode 100644 examples/ollama/data/output/ollama-example/25-06-2024-18-43-36-log-ollama-example.txt create mode 100644 examples/ollama/index.html create mode 100644 examples/ollama/ollama/index.html create mode 100644 examples/openai/data/input/openai-example.jsonl create mode 100644 examples/openai/data/output/openai-example/25-06-2024-19-56-26-completed-openai-example.jsonl create mode 100644 examples/openai/data/output/openai-example/25-06-2024-19-56-26-input-openai-example.jsonl create mode 100644 examples/openai/data/output/openai-example/25-06-2024-19-56-26-log-openai-example.txt create mode 100644 examples/openai/index.html create mode 100644 examples/openai/openai/index.html create mode 100644 examples/quart/quart.jsonl create mode 100644 examples/system-demo/alpaca_sample_generation/index.html create mode 100644 examples/system-demo/api_utils.py create mode 100644 examples/system-demo/data/output/all_experiments/10-07-2024-18-05-54-completed-all_experiments.jsonl create mode 100644 examples/system-demo/data/output/all_experiments/10-07-2024-18-05-54-input-all_experiments.jsonl create mode 100644 examples/system-demo/data/output/all_experiments/10-07-2024-18-05-54-log-all_experiments.txt create mode 100644 examples/system-demo/data/output/gemini/09-07-2024-19-59-34-completed-gemini.jsonl create mode 100644 examples/system-demo/data/output/gemini/09-07-2024-19-59-34-input-gemini.jsonl create mode 100644 examples/system-demo/data/output/gemini/09-07-2024-19-59-34-log-gemini.txt create mode 100644 examples/system-demo/data/output/ollama/09-07-2024-20-04-21-completed-ollama.jsonl create mode 100644 examples/system-demo/data/output/ollama/09-07-2024-20-04-21-input-ollama.jsonl create mode 100644 examples/system-demo/data/output/ollama/09-07-2024-20-04-21-log-ollama.txt create mode 100644 examples/system-demo/data/output/openai-gpt-3pt5-turbo/10-07-2024-17-08-07-completed-openai-gpt-3pt5-turbo.jsonl create mode 100644 examples/system-demo/data/output/openai-gpt-3pt5-turbo/10-07-2024-17-08-07-input-openai-gpt-3pt5-turbo.jsonl create mode 100644 examples/system-demo/data/output/openai-gpt-3pt5-turbo/10-07-2024-17-08-07-log-openai-gpt-3pt5-turbo.txt create mode 100644 examples/system-demo/data/output/openai-gpt-4/10-07-2024-17-14-54-completed-openai-gpt-4.jsonl create mode 100644 examples/system-demo/data/output/openai-gpt-4/10-07-2024-17-14-54-input-openai-gpt-4.jsonl create mode 100644 examples/system-demo/data/output/openai-gpt-4/10-07-2024-17-14-54-log-openai-gpt-4.txt create mode 100644 examples/system-demo/data/output/openai-gpt-4o/10-07-2024-17-19-15-completed-openai-gpt-4o.jsonl create mode 100644 examples/system-demo/data/output/openai-gpt-4o/10-07-2024-17-19-15-input-openai-gpt-4o.jsonl create mode 100644 examples/system-demo/data/output/openai-gpt-4o/10-07-2024-17-19-15-log-openai-gpt-4o.txt create mode 100644 examples/system-demo/data/output/openai-multiple-models/10-07-2024-17-31-18-completed-openai-multiple-models.jsonl create mode 100644 examples/system-demo/data/output/openai-multiple-models/10-07-2024-17-31-18-input-openai-multiple-models.jsonl create mode 100644 examples/system-demo/data/output/openai-multiple-models/10-07-2024-17-31-18-log-openai-multiple-models.txt create mode 100644 examples/system-demo/data/output/openai/09-07-2024-19-56-36-completed-openai.jsonl create mode 100644 examples/system-demo/data/output/openai/09-07-2024-19-56-36-input-openai.jsonl create mode 100644 examples/system-demo/data/output/openai/09-07-2024-19-56-36-log-openai.txt create mode 100644 examples/system-demo/dataset_utils.py create mode 100644 examples/system-demo/experiment_1/index.html create mode 100644 examples/system-demo/experiment_2/index.html create mode 100644 examples/system-demo/experiment_3/index.html create mode 100644 examples/system-demo/index.html create mode 100644 examples/system-demo/sample_prompts.json create mode 100644 examples/vertexai/data/input/vertexai-example.jsonl create mode 100644 examples/vertexai/data/output/vertexai-example/26-06-2024-10-57-06-completed-vertexai-example.jsonl create mode 100644 examples/vertexai/data/output/vertexai-example/26-06-2024-10-57-06-input-vertexai-example.jsonl create mode 100644 examples/vertexai/data/output/vertexai-example/26-06-2024-10-57-06-log-vertexai-example.txt create mode 100644 examples/vertexai/index.html create mode 100644 examples/vertexai/vertexai/index.html create mode 100644 index.html create mode 100644 objects.inv create mode 100644 reference/DOC_STRINGS/index.html create mode 100644 reference/src/prompto/apis/anthropic/anthropic/index.html create mode 100644 reference/src/prompto/apis/anthropic/anthropic_utils/index.html create mode 100644 reference/src/prompto/apis/anthropic/index.html create mode 100644 reference/src/prompto/apis/azure_openai/azure_openai/index.html create mode 100644 reference/src/prompto/apis/azure_openai/index.html create mode 100644 reference/src/prompto/apis/base/index.html create mode 100644 reference/src/prompto/apis/gemini/gemini/index.html create mode 100644 reference/src/prompto/apis/gemini/gemini_utils/index.html create mode 100644 reference/src/prompto/apis/gemini/index.html create mode 100644 reference/src/prompto/apis/huggingface_tgi/huggingface_tgi/index.html create mode 100644 reference/src/prompto/apis/huggingface_tgi/index.html create mode 100644 reference/src/prompto/apis/index.html create mode 100644 reference/src/prompto/apis/ollama/index.html create mode 100644 reference/src/prompto/apis/ollama/ollama/index.html create mode 100644 reference/src/prompto/apis/ollama/ollama_utils/index.html create mode 100644 reference/src/prompto/apis/openai/index.html create mode 100644 reference/src/prompto/apis/openai/openai/index.html create mode 100644 reference/src/prompto/apis/openai/openai_utils/index.html create mode 100644 reference/src/prompto/apis/quart/index.html create mode 100644 reference/src/prompto/apis/quart/quart/index.html create mode 100644 reference/src/prompto/apis/quart/quart_api/index.html create mode 100644 reference/src/prompto/apis/quart/quart_utils/index.html create mode 100644 reference/src/prompto/apis/testing/index.html create mode 100644 reference/src/prompto/apis/testing/testing_api/index.html create mode 100644 reference/src/prompto/apis/vertexai/index.html create mode 100644 reference/src/prompto/apis/vertexai/vertexai/index.html create mode 100644 reference/src/prompto/apis/vertexai/vertexai_utils/index.html create mode 100644 reference/src/prompto/experiment/index.html create mode 100644 reference/src/prompto/experiment_pipeline/index.html create mode 100644 reference/src/prompto/index.html create mode 100644 reference/src/prompto/judge/index.html create mode 100644 reference/src/prompto/scorer/index.html create mode 100644 reference/src/prompto/scripts/check_experiment/index.html create mode 100644 reference/src/prompto/scripts/convert_images/index.html create mode 100644 reference/src/prompto/scripts/create_judge_file/index.html create mode 100644 reference/src/prompto/scripts/index.html create mode 100644 reference/src/prompto/scripts/obtain_missing_id_jsonl/index.html create mode 100644 reference/src/prompto/scripts/run_experiment/index.html create mode 100644 reference/src/prompto/scripts/run_pipeline/index.html create mode 100644 reference/src/prompto/settings/index.html create mode 100644 reference/src/prompto/utils/index.html create mode 100644 reference/tests/apis/anthropic/index.html create mode 100644 reference/tests/apis/gemini/index.html create mode 100644 reference/tests/apis/index.html create mode 100644 reference/tests/apis/ollama/index.html create mode 100644 reference/tests/apis/vertexai/index.html create mode 100644 reference/tests/core/index.html create mode 100644 reference/tests/index.html create mode 100644 reference/tests/scripts/index.html create mode 100644 search/search_index.json create mode 100644 sitemap.xml create mode 100644 sitemap.xml.gz create mode 100644 src/prompto/__init__.py create mode 100644 src/prompto/apis/__init__.py create mode 100644 src/prompto/apis/anthropic/__init__.py create mode 100644 src/prompto/apis/anthropic/anthropic.py create mode 100644 src/prompto/apis/anthropic/anthropic_utils.py create mode 100644 src/prompto/apis/azure_openai/__init__.py create mode 100644 src/prompto/apis/azure_openai/azure_openai.py create mode 100644 src/prompto/apis/base.py create mode 100644 src/prompto/apis/gemini/__init__.py create mode 100644 src/prompto/apis/gemini/gemini.py create mode 100644 src/prompto/apis/gemini/gemini_utils.py create mode 100644 src/prompto/apis/huggingface_tgi/__init__.py create mode 100644 src/prompto/apis/huggingface_tgi/huggingface_tgi.py create mode 100644 src/prompto/apis/ollama/__init__.py create mode 100644 src/prompto/apis/ollama/ollama.py create mode 100644 src/prompto/apis/ollama/ollama_utils.py create mode 100644 src/prompto/apis/openai/__init__.py create mode 100644 src/prompto/apis/openai/openai.py create mode 100644 src/prompto/apis/openai/openai_utils.py create mode 100644 src/prompto/apis/quart/__init__.py create mode 100644 src/prompto/apis/quart/quart.py create mode 100644 src/prompto/apis/quart/quart_api.py create mode 100644 src/prompto/apis/quart/quart_utils.py create mode 100644 src/prompto/apis/testing/__init__.py create mode 100644 src/prompto/apis/testing/testing_api.py create mode 100644 src/prompto/apis/vertexai/__init__.py create mode 100644 src/prompto/apis/vertexai/vertexai.py create mode 100644 src/prompto/apis/vertexai/vertexai_utils.py create mode 100644 src/prompto/experiment.py create mode 100644 src/prompto/experiment_pipeline.py create mode 100644 src/prompto/judge.py create mode 100644 src/prompto/scorer.py create mode 100644 src/prompto/scripts/__init__.py create mode 100644 src/prompto/scripts/check_experiment.py create mode 100644 src/prompto/scripts/convert_images.py create mode 100644 src/prompto/scripts/create_judge_file.py create mode 100644 src/prompto/scripts/obtain_missing_id_jsonl.py create mode 100644 src/prompto/scripts/run_experiment.py create mode 100644 src/prompto/scripts/run_pipeline.py create mode 100644 src/prompto/settings.py create mode 100644 src/prompto/utils.py create mode 100644 tests/__init__.py create mode 100644 tests/apis/__init__.py create mode 100644 tests/apis/anthropic/__init__.py create mode 100644 tests/apis/anthropic/test_anthropic.py create mode 100644 tests/apis/anthropic/test_anthropic_chat_input.py create mode 100644 tests/apis/anthropic/test_anthropic_history_input.py create mode 100644 tests/apis/anthropic/test_anthropic_string_input.py create mode 100644 tests/apis/gemini/__init__.py create mode 100644 tests/apis/gemini/test_gemini.py create mode 100644 tests/apis/gemini/test_gemini_chat_input.py create mode 100644 tests/apis/gemini/test_gemini_history_input.py create mode 100644 tests/apis/gemini/test_gemini_string_input.py create mode 100644 tests/apis/ollama/__init__.py create mode 100644 tests/apis/test_base.py create mode 100644 tests/apis/test_import_warnings.py create mode 100644 tests/apis/test_imports.py create mode 100644 tests/apis/vertexai/__init__.py create mode 100644 tests/apis/vertexai/test_vertexai.py create mode 100644 tests/apis/vertexai/test_vertexai_chat_input.py create mode 100644 tests/apis/vertexai/test_vertexai_history_input.py create mode 100644 tests/apis/vertexai/test_vertexai_string_input.py create mode 100644 tests/conftest.py create mode 100644 tests/core/__init__.py create mode 100644 tests/core/test_experiment.py create mode 100644 tests/core/test_experiment_generate_text.py create mode 100644 tests/core/test_experiment_pipeline.py create mode 100644 tests/core/test_experiment_process.py create mode 100644 tests/core/test_experiment_query_model.py create mode 100644 tests/core/test_experiment_send_requests.py create mode 100644 tests/core/test_judge.py create mode 100644 tests/core/test_scorer.py create mode 100644 tests/core/test_settings.py create mode 100644 tests/core/test_utils.py create mode 100644 tests/scripts/__init__.py create mode 100644 tests/scripts/test_create_judge_file.py create mode 100644 tests/scripts/test_run_experiment.py diff --git a/.nojekyll b/.nojekyll new file mode 100644 index 00000000..e69de29b diff --git a/404.html b/404.html new file mode 100644 index 00000000..736df76d --- /dev/null +++ b/404.html @@ -0,0 +1,3287 @@ + + + +
+ + + + + + + + + + + + + + +Copyright (c) 2024 The Alan Turing Institute
+Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.
+ + + + + + + + + + + + + +