From 6ac815b31c63e05c4e145c40ebd43a8c278bce69 Mon Sep 17 00:00:00 2001 From: Aliaksei Yaletski Date: Wed, 18 Sep 2024 14:26:19 +0200 Subject: [PATCH] +1 validation text (#262) --- ffun/ffun/cli/commands/processors_quality.py | 42 ++++++++++++ ffun/ffun/processors_quality/knowlege_base.py | 14 ++++ tags_quality_base/news/0012.toml | 24 +++++++ .../tags-actual/openai_llm_general/0012.toml | 2 + .../openai_llm_general/0012.toml | 68 +++++++++++++++++++ .../tags-last/openai_llm_general/0001.toml | 4 +- .../tags-last/openai_llm_general/0012.toml | 2 + 7 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 tags_quality_base/news/0012.toml create mode 100644 tags_quality_base/tags-actual/openai_llm_general/0012.toml create mode 100644 tags_quality_base/tags-expected/openai_llm_general/0012.toml create mode 100644 tags_quality_base/tags-last/openai_llm_general/0012.toml diff --git a/ffun/ffun/cli/commands/processors_quality.py b/ffun/ffun/cli/commands/processors_quality.py index b6a075d7..de6ea2ff 100644 --- a/ffun/ffun/cli/commands/processors_quality.py +++ b/ffun/ffun/cli/commands/processors_quality.py @@ -1,5 +1,6 @@ import asyncio import pathlib +from collections import Counter import typer @@ -135,3 +136,44 @@ async def run_deff_all(processor_name: str, knowlege_root: pathlib.Path = _root, @cli_app.command() def diff_all(processor: str, knowlege_root: pathlib.Path = _root, show_tag_diffs: bool = False) -> None: asyncio.run(run_deff_all(processor, knowlege_root=knowlege_root, show_tag_diffs=show_tag_diffs)) + + +async def run_prepare_news_item( + processor: str, entry_id: int, knowlege_root: pathlib.Path, requests_number: int, min_tags_count: int +) -> None: + results = [] + + async with with_app(): + kb = KnowlegeBase(knowlege_root) + + for i in range(requests_number): + logger.info("requesting_tags", step=i, steps_number=requests_number) + result = await single_run(processor, entry_id, kb, actual=True) + results.append(result) + + logger.info("requests_completed") + + tags: Counter[str] = Counter() + + for result in results: + tags.update(result.tags) + + tags_must_have = {tag for tag, count in tags.items() if count == requests_number} + tags_should_have = {tag for tag, count in tags.items() if min_tags_count <= count < requests_number} + + kb.save_expected_data(processor, entry_id, tags_must_have=tags_must_have, tags_should_have=tags_should_have) + + +@cli_app.command() +def prepere_news_item( + processor: str, entry: int, knowlege_root: pathlib.Path = _root, requests_number: int = 5, min_tags_count: int = 3 +) -> None: + asyncio.run( + run_prepare_news_item( + processor, + entry_id=entry, + knowlege_root=knowlege_root, + requests_number=requests_number, + min_tags_count=min_tags_count, + ) + ) diff --git a/ffun/ffun/processors_quality/knowlege_base.py b/ffun/ffun/processors_quality/knowlege_base.py index baf81c3e..cc0d5bca 100644 --- a/ffun/ffun/processors_quality/knowlege_base.py +++ b/ffun/ffun/processors_quality/knowlege_base.py @@ -109,6 +109,20 @@ def save_processor_result(self, processor: str, entry_id: int, result: Processor actual_path = self._dir_tags_actual / processor / f"{id_to_name(entry_id)}.toml" actual_path.write_text(content) + def save_expected_data( + self, processor: str, entry_id: int, tags_must_have: set[str], tags_should_have: set[str] + ) -> None: + if tags_must_have & tags_should_have: + raise NotImplementedError("tags_must_have and tags_should_have should not intersect") + + (self._dir_tags_expected / processor).mkdir(parents=True, exist_ok=True) + + tags_path = self._dir_tags_expected / processor / f"{id_to_name(entry_id)}.toml" + + data = {"tags_must_have": list(sorted(tags_must_have)), "tags_should_have": list(sorted(tags_should_have))} + + tags_path.write_text(toml.dumps(data)) + def copy_last_to_actual(self, processor: str, entry_id: int) -> None: last_path = self._dir_tags_last / processor / f"{id_to_name(entry_id)}.toml" actual_path = self._dir_tags_actual / processor / f"{id_to_name(entry_id)}.toml" diff --git a/tags_quality_base/news/0012.toml b/tags_quality_base/news/0012.toml new file mode 100644 index 00000000..aca8be1d --- /dev/null +++ b/tags_quality_base/news/0012.toml @@ -0,0 +1,24 @@ +--- +title = "SpaceX will return stranded astronauts in February 2025, NASA announces" +external_url = "https://en.wikinews.org/wiki/SpaceX_will_return_stranded_astronauts_in_February_2025,_NASA_announces" +external_tags = [] +published_at = "2024-08-30T12:00:00+00:00" + +[attribution] +title = ": SpaceX will return stranded astronauts in February 2025, NASA announces" +authors = ["Wikinews contributors"] +link = "https://en.wikinews.org/wiki/SpaceX_will_return_stranded_astronauts_in_February_2025,_NASA_announces" +license = "CC BY-NC-ND 4.0" +--- +

On Saturday, NASA (National Aeronautics and Space Administration) announced at a press conference that Sunita Williams and Barry Wilmore would return home from the International Space Station (ISS) on a SpaceX Crew Dragon spacecraft in February 2025. The Boeing Starliner, the spacecraft which had been expected to transport the crew from ISS earlier in June, would return to Earth "un-crewed" — on autopilot with planned landing in a desert in New Mexico, possibly next month, the NASA announcement said. +

+
Boeing's Starliner crew ship approaches the space station.
Image: Bob Hines/NASA.
+
Spacex Crew Dragon Endeavour approaching the ISS in May 2024 during Crew-8.
Image: NASA Johnson Space Center.
+

On its way to the ISS, Starliner has experienced helium leaks and several thruster malfunctions, which, according to National Public Radio (NPR, "inexplicably conked out as Starliner approached the station.". Steve Stich, the manager of NASA's Commercial Crew Program, said they didn't know if Starliner's thrusters would fail at critical moments. The Starliner test flight was Boeing's first crewed space mission; three previous attempts for this crewed mission had failed. +

The two NASA astronauts had been launched during Starliner's crewed test flight on June 5 for what was expected to be an eight-day long mission to the ISS. A successful mission is a requirement to certify Starliner for NASA’s Commercial Crew Program. Due to the change of plans, the astronauts would remain at ISS for several months as part of Expedition 71/72. +

Boeing wrote on social media about its commitment to safety and to cooperation with NASA, as they were continuing "to focus, first and foremost, on the safety of the crew and spacecraft. We are executing the mission as determined by NASA, and we are preparing the spacecraft for a safe and successful uncrewed return." +

The Crew Dragon would carry two astronauts to the ISS instead of the planned 4 to allow room for Williams and Wilmore, NASA announced. +

NASA said that SpaceX would reportedly leave Earth in September for the planned Crew-9 mission, with a change of plans to have only two crew on board, so that the remaining two free seats will be available for the stranded astronauts to return on Earth in February. +

SpaceX’s Crew Dragon—certified in 2020—will undergo some changes before launching on September 24. The preparation to bring Williams and Wilmore home to their families includes additional cargo, adjusting the seats for the Dragon, and Dragon-specific spacesuits for the two astronauts. The preparation also adds in a contingency plan, providing the two a flight home on the Crew-8 spacecraft. +

To date, SpaceX had completed nine crewed flights to space for NASA and several commercial flights, BBC reported. +

diff --git a/tags_quality_base/tags-actual/openai_llm_general/0012.toml b/tags_quality_base/tags-actual/openai_llm_general/0012.toml new file mode 100644 index 00000000..60070edc --- /dev/null +++ b/tags_quality_base/tags-actual/openai_llm_general/0012.toml @@ -0,0 +1,2 @@ +tags = [ "aerospace", "aerospace-engineering", "aerospace-industry", "astronaut-training", "astronauts", "astronauts-return", "barry-wilmore", "boeing", "boeing-innovations", "boeing-starliner", "boeing-starliner-issues", "commercial-crew-program", "commercial-spaceflight", "crew-8", "crew-8-preparations", "crew-9", "crew-9-planning", "crew-dragon", "crew-dragon-missions", "crew-mission-challenges", "crew-missions", "crew-transport", "crew-transport-adjustments", "crew-transportation", "expedition-71-72", "expedition-71-72-operations", "government-space-programs", "helium-leak-investigation", "helium-leaks", "human-spaceflight", "international-space-collaboration", "international-space-cooperation", "international-space-station", "launch-operations", "launch-vehicle-operations", "mission-operations", "nasa", "nasa-initiatives", "national-public-radio", "new-mexico", "private-space-ventures", "robotics-in-space", "space-agencies", "space-agency-collaboration", "space-agency-initiatives", "space-agency-press-releases", "space-announcements", "space-crew-updates", "space-exploration", "space-exploration-goals", "space-exploration-initiatives", "space-exploration-news", "space-industry-advancements", "space-industry-developments", "space-innovation", "space-logistics", "space-logistics-and-support", "space-mission-analytics", "space-mission-architecture", "space-mission-communication", "space-mission-coverage", "space-mission-impacts", "space-mission-management", "space-mission-operations", "space-mission-planning", "space-mission-reports", "space-mission-safety", "space-mission-scheduling", "space-mission-technology", "space-mission-timelines", "space-mission-updates", "space-missions", "space-news", "space-policy", "space-policy-and-governance", "space-research", "space-research-and-development", "space-research-collaboration", "space-safety", "space-safety-regulations", "space-safety-standards", "space-suit-designs", "space-systems", "space-systems-engineering", "space-technology", "space-technology-advancements", "space-technology-development", "space-technology-trends", "space-transportation", "space-transportation-systems", "spacecraft", "spacecraft-cargo-management", "spacecraft-certification", "spacecraft-design", "spacecraft-engineering", "spacecraft-integration", "spacecraft-landing-plans", "spacecraft-recovery", "spacecraft-safety", "spacecraft-safety-protocols", "spacecraft-seating-arrangements", "spacecraft-technology", "spaceflight-education", "spaceflight-history", "spaceflight-innovation", "spaceflight-safety", "spacex", "spacex-developments", "steve-stich", "sunita-williams", "thruster-malfunction-analysis", "thruster-malfunctions",] +created_at = 2024-09-18T12:15:07.608906Z diff --git a/tags_quality_base/tags-expected/openai_llm_general/0012.toml b/tags_quality_base/tags-expected/openai_llm_general/0012.toml new file mode 100644 index 00000000..6f21947e --- /dev/null +++ b/tags_quality_base/tags-expected/openai_llm_general/0012.toml @@ -0,0 +1,68 @@ +tags_must_have = [ + "aerospace", + "aerospace-engineering", + "aerospace-industry", + "astronauts", + "barry-wilmore", + "boeing", + "boeing-starliner", + "commercial-crew-program", + "commercial-spaceflight", + "crew-8", + "crew-9", + "crew-dragon", + "expedition-71-72", + "helium-leaks", + "human-spaceflight", + "launch-operations", + "nasa", + "space-agencies", + "space-announcements", + "space-exploration", + "space-innovation", + "space-logistics", + "space-mission-architecture", + "space-mission-scheduling", + "space-missions", + "space-news", + "space-safety", + "space-technology", + "spacecraft", + "spacecraft-safety", + "spacex", + "steve-stich", + "sunita-williams", + "thruster-malfunctions",] + +tags_should_have = [ + "astronauts-return", + "aviation-safety", + "crew-9-planning", + "crew-transport", + "government-contracting", + "mission-operations", + "mission-planning", + "national-public-radio", + "new-mexico", + "robotics", + "space-agency-collaboration", + "space-engineering", + "space-entrepreneurship", + "space-exploration-initiatives", + "space-infrastructure", + "space-mission-analytics", + "space-mission-failures", + "space-operations", + "space-reports", + "space-safety-standards", + "space-systems-engineering", + "space-transportation", + "space-transportation-systems", + "spacecraft-certification", + "spacecraft-landing-plans", + "spacecraft-operations", + "spacecraft-reliability", + "spacecraft-technology", + "spaceflight-history", + "spacesuit-designs", +] diff --git a/tags_quality_base/tags-last/openai_llm_general/0001.toml b/tags_quality_base/tags-last/openai_llm_general/0001.toml index b4d8e134..e5106425 100644 --- a/tags_quality_base/tags-last/openai_llm_general/0001.toml +++ b/tags_quality_base/tags-last/openai_llm_general/0001.toml @@ -1,2 +1,2 @@ -tags = [ "academic-writing", "artificial-intelligence", "artificial-intelligence-research", "artificial-life", "behavioral-economics", "big-data", "big-data-ethics", "book-analysis", "book-review", "business", "business-analysts", "business-intelligence", "case-study-analysis", "chaos-theory", "climate-change-prediction", "cognitive-science", "complexity-science", "computational-social-science", "critical-analysis", "cybernetics", "data-analysis", "data-analysis-techniques", "data-analysts", "data-cleaning", "data-communication", "data-driven-business", "data-driven-decision-making", "data-driven-economy", "data-driven-future", "data-driven-governance", "data-driven-innovation", "data-driven-insights", "data-driven-society", "data-driven-solutions", "data-driven-strategies", "data-driven-world", "data-ethics", "data-interpretation", "data-mining", "data-preprocessing", "data-privacy", "data-science", "data-scientists", "data-security", "data-visualization", "decision-makers", "decision-making", "decision-support-systems", "decision-theory", "disease-outbreak-prediction", "earthquake-prediction", "economic-forecasting", "economics", "economists", "election-forecasting", "environment", "feature-engineering", "finance", "financial-analysts", "financial-markets", "financial-modeling", "forecasting", "forecasting-biases", "forecasting-errors", "forecasting-methods", "fraud-detection", "game-theory", "health", "information-extraction", "information-extraction-from-data", "information-management", "information-retrieval", "information-synthesis", "information-theory", "knowledge-discovery", "knowledge-discovery-from-data", "knowledge-management", "knowledge-representation", "knowledge-synthesis", "literature-review", "machine-learning", "machine-learning-algorithms", "machine-learning-engineers", "market-research", "market-researchers", "market-trends", "meteorologists", "meteorology", "model-selection", "nate-silver", "natural-language-processing", "network-science", "operations-research", "operations-researchers", "overfitting", "pattern-recognition", "policy-makers", "political-science", "political-scientists", "politics", "prediction", "predictive-analytics", "predictive-maintenance", "predictive-modeling", "probability", "research", "research-methodology", "researchers", "risk-assessment", "risk-assessment-frameworks", "risk-management", "risk-management-strategies", "risk-managers", "scale", "security", "sentiment-analysis", "signal-to-noise-ratio", "social-media-analysis", "social-psychology", "social-science", "statistical-modeling", "statisticians", "statistics", "systematic-review", "systems-thinking", "technology", "terrorism-prediction", "the-signal-and-the-noise", "theoretical-framework", "time-series-analysis", "uncertainty", "uncertainty-quantification", "underfitting", "us-elections", "weather-forecasting",] -created_at = 2024-09-05T10:57:23.291918Z +tags = [ "academic-publishers", "accessibility", "accessibility-in-writing", "analytical-thinking", "analytics", "author-analysis", "basic-theses", "behavioral-economics", "bestsellers", "bestselling-authors", "bestselling-books", "book-reviews", "brief-summaries", "brigid", "business", "business-intelligence", "case-per-chapter", "case-studies", "case-study-methodology", "cognitive-bias", "cognitive-biases", "cognitive-science", "coherent-systems", "communication", "communication-studies", "communication-theory", "complexity", "complexity-theory", "contextual-analysis", "cool-facts", "critical-theory", "critical-thinking", "data-analysis", "data-analytics-firms", "data-collection", "data-driven-decision-making", "data-ethics", "data-interpretation", "data-science", "data-science-community", "data-visualization", "decision-making", "economics", "education", "environmental-science", "epistemology", "error-analysis", "errors", "errors-in-forecasting", "evidence-based-practice", "forecasting", "forecasting-accuracy", "forecasting-approaches", "forecasting-challenges", "forecasting-community", "forecasting-complexities", "forecasting-errors", "forecasting-implications", "forecasting-issues", "forecasting-methodologies", "forecasting-methods", "forecasting-misconceptions", "forecasting-practices", "forecasting-strategies", "forecasting-techniques", "forecasting-tools", "information-architecture", "information-distribution", "information-systems", "information-technology", "information-theory", "interpretation", "knowledge-management", "literary-critics", "literature", "meteorology", "methodology", "misconceptions", "nate-silver", "philosophy-of-science", "policy-analysis", "practical-application", "practical-approaches", "practical-cases", "practical-forecasting", "pragmatics", "predictive-analytics", "prisms-of-analysis", "psychology", "public-administration", "public-policy", "publications", "quantitative-analysis", "quantitative-research", "research", "research-institutions", "research-methodologies", "risk-analysis", "scale", "scientific-methods", "significant-tasks", "social-constructivism", "social-sciences", "sociology", "statistical-forecasting", "statistical-modeling", "statistics", "systematic-approaches", "systematic-review", "systematics", "the-signal-and-the-noise", "theoretical-approaches", "theoretical-forecasting", "theory", "theory-and-practice", "theory-building", "thesis-development", "tiendil", "us-elections", "weather-forecasting", "wikipedia",] +created_at = 2024-09-18T12:08:15.293682Z diff --git a/tags_quality_base/tags-last/openai_llm_general/0012.toml b/tags_quality_base/tags-last/openai_llm_general/0012.toml new file mode 100644 index 00000000..a18de257 --- /dev/null +++ b/tags_quality_base/tags-last/openai_llm_general/0012.toml @@ -0,0 +1,2 @@ +tags = [ "aerospace", "aerospace-developments", "aerospace-engineering", "aerospace-industry", "astronaut-return-plans", "astronaut-training", "astronauts", "astronauts-williams-wilmore", "barry-wilmore", "boeing", "boeing-starliner", "boeing-starliner-issues", "commercial-crew-program", "commercial-spaceflight", "crew-8", "crew-8-mission", "crew-9", "crew-9-plans", "crew-communication", "crew-dragon", "crew-dragon-missions", "crew-missions", "crew-operations", "crew-safety-measures", "crew-transportation", "crew-transportation-challenges", "crew-transportation-program", "crew-transportation-systems", "expedition-71-72", "government-space-programs", "helium-leak-investigations", "helium-leaks", "human-spaceflight", "international-space-cooperation", "international-space-station", "launch-operations", "launch-vehicle-integration", "mission-operations", "nasa", "nasa-johnson-space-center", "national-public-radio", "new-mexico", "private-space-ventures", "robotics-in-space", "space-agencies", "space-agency-activities", "space-agency-collaboration", "space-agency-collaborations", "space-agency-initiatives", "space-announcements", "space-crew-issues", "space-crew-management", "space-engineering", "space-exploration", "space-exploration-initiatives", "space-exploration-missions", "space-exploration-news", "space-exploration-policies", "space-industry-initiatives", "space-industry-trends", "space-innovation", "space-logistics", "space-mission-analysis", "space-mission-architecture", "space-mission-challenges", "space-mission-impacts", "space-mission-management", "space-mission-planning", "space-mission-reports", "space-mission-safety", "space-mission-scheduling", "space-mission-technology", "space-mission-updates", "space-missions", "space-news", "space-operations", "space-operations-management", "space-policy", "space-research", "space-safety", "space-safety-regulations", "space-safety-standards", "space-systems-engineering", "space-technology", "space-technology-advancements", "space-technology-development", "space-technology-innovation", "space-transportation", "space-transportation-systems", "spacecraft", "spacecraft-cargo-adjustments", "spacecraft-certification", "spacecraft-crew-transport", "spacecraft-design", "spacecraft-design-changes", "spacecraft-landing-plans", "spacecraft-maintenance", "spacecraft-operations", "spacecraft-preparations", "spacecraft-recovery", "spacecraft-safety", "spacecraft-suit-designs", "spacecraft-technology", "spaceflight-advancements", "spaceflight-education", "spaceflight-history", "spaceflight-issues", "spaceflight-operations", "spaceflight-safety", "spacex", "steve-stich", "sunita-williams", "thruster-failures", "thruster-malfunctions",] +created_at = 2024-09-18T12:17:27.689067Z