Frlg fixes (#1132)

* update frlg battle detectors for japanese

* log gift reset target

* frlg summary page detection

* battle menu tests

Author: kichithewolf

SerialPrograms/Source/PokemonFRLG/Inference/Dialogs/PokemonFRLG_DialogDetector.cpp

@@ -131,20 +131,29 @@ bool SelectionDialogDetector::detect(const ImageViewRGB32& screen){
 BattleDialogDetector::BattleDialogDetector(Color color)
     : m_dialog_top_box(0.0372308, 0.750154, 0.925538, 0.00934615)
     , m_dialog_right_box(0.956615, 0.7595, 0.00615385, 0.185885)
+    , m_dialog_top_jpn_box(0.043890, 0.749227, 0.910248, 0.008023) //jpn positions might work for other languages
+    , m_dialog_right_jpn_box(0.950583, 0.751901, 0.003556, 0.199230) //dialog might get in the way though
 {}
 void BattleDialogDetector::make_overlays(VideoOverlaySet& items) const{
     const BoxOption& GAME_BOX = GameSettings::instance().GAME_BOX;
     items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_top_box));
     items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_right_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_top_jpn_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_right_jpn_box));
 }
 bool BattleDialogDetector::detect(const ImageViewRGB32& screen){
     ImageViewRGB32 game_screen = extract_box_reference(screen, GameSettings::instance().GAME_BOX);
 
     ImageViewRGB32 dialog_top_image = extract_box_reference(game_screen, m_dialog_top_box);
     ImageViewRGB32 dialog_right_image = extract_box_reference(game_screen, m_dialog_right_box);
+    ImageViewRGB32 dialog_top_jpn_image = extract_box_reference(game_screen, m_dialog_top_jpn_box);
+    ImageViewRGB32 dialog_right_jpn_image = extract_box_reference(game_screen, m_dialog_right_jpn_box);
 
-    if (is_solid(dialog_top_image, { 0.176, 0.357, 0.467 }, 0.25, 20)
-        && is_solid(dialog_right_image, { 0.176, 0.357, 0.467 }, 0.25, 20)
+    if ((is_solid(dialog_top_image, { 0.176, 0.357, 0.467 }, 0.25, 20)
+        && is_solid(dialog_right_image, { 0.176, 0.357, 0.467 }, 0.25, 20))
+        ||
+        (is_solid(dialog_top_jpn_image, { 0.176, 0.357, 0.467 }, 0.25, 20)
+        && is_solid(dialog_right_jpn_image, { 0.176, 0.357, 0.467 }, 0.25, 20))
     ){
         return true;
     }
@@ -157,29 +166,47 @@ BattleMenuDetector::BattleMenuDetector(Color color)
     , m_menu_right_box(0.961538, 0.752231, 0.00615385, 0.200423)
     , m_dialog_top_box(0.036, 0.749115, 0.459077, 0.0135)
     , m_dialog_right_box(0.490154, 0.762615, 0.00615385, 0.178615) //right side, closest to the menu
+    , m_menu_top_jpn_box(0.593239, 0.743878, 0.373344, 0.009360) //very different positions!
+    , m_menu_right_jpn_box(0.961538, 0.752231, 0.00615385, 0.200423)
+    , m_dialog_top_jpn_box(0.043001, 0.750564, 0.520015, 0.002674)
+    , m_dialog_right_jpn_box(0.559460, 0.750564, 0.003556, 0.196556)
 {}
 void BattleMenuDetector::make_overlays(VideoOverlaySet& items) const{
     const BoxOption& GAME_BOX = GameSettings::instance().GAME_BOX;
     items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_menu_top_box));
     items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_menu_right_box));
     items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_top_box));
     items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_right_box));
+
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_menu_top_jpn_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_menu_right_jpn_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_top_jpn_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_right_jpn_box));
 }
 bool BattleMenuDetector::detect(const ImageViewRGB32& screen){
     ImageViewRGB32 game_screen = extract_box_reference(screen, GameSettings::instance().GAME_BOX);
 
     //Menu is white
     ImageViewRGB32 menu_top_image = extract_box_reference(game_screen, m_menu_top_box);
     ImageViewRGB32 menu_right_image = extract_box_reference(game_screen, m_menu_right_box);
+    ImageViewRGB32 menu_top_jpn_image = extract_box_reference(game_screen, m_menu_top_jpn_box);
+    ImageViewRGB32 menu_right_jpn_image = extract_box_reference(game_screen, m_menu_right_jpn_box);
 
     //Background dialog is teal
     ImageViewRGB32 dialog_top_image = extract_box_reference(game_screen, m_dialog_top_box);
     ImageViewRGB32 dialog_right_image = extract_box_reference(game_screen, m_dialog_right_box);
+    ImageViewRGB32 dialog_top_jpn_image = extract_box_reference(game_screen, m_dialog_top_jpn_box);
+    ImageViewRGB32 dialog_right_jpn_image = extract_box_reference(game_screen, m_dialog_right_jpn_box);
 
-    if (is_white(menu_top_image)
+    if ((is_white(menu_top_image) //All languages except japanese
         && is_white(menu_right_image)
         && is_solid(dialog_top_image, { 0.176, 0.357, 0.467 }, 0.25, 20) //40, 81, 106 teal
-        && is_solid(dialog_right_image, { 0.176, 0.357, 0.467 }, 0.25, 20)
+        && is_solid(dialog_right_image, { 0.176, 0.357, 0.467 }, 0.25, 20))
+        ||
+        (is_white(menu_top_jpn_image) //japanese
+        && is_white(menu_right_jpn_image)
+        && is_solid(dialog_top_jpn_image, { 0.176, 0.357, 0.467 }, 0.25, 20)
+        && is_solid(dialog_right_jpn_image, { 0.176, 0.357, 0.467 }, 0.25, 20))
     ){
         return true;
     }
@@ -191,12 +218,18 @@ AdvanceBattleDialogDetector::AdvanceBattleDialogDetector(Color color)
     : m_dialog_box(0.036, 0.748077, 0.926769, 0.204577)
     , m_dialog_top_box(0.0372308, 0.750154, 0.925538, 0.00934615)
     , m_dialog_right_box(0.956615, 0.7595, 0.00615385, 0.185885)
+    , m_dialog_jpn_box(0.043890, 0.749227, 0.911137, 0.203242) //this position is very different!
+    , m_dialog_top_jpn_box(0.043890, 0.749227, 0.910248, 0.008023)
+    , m_dialog_right_jpn_box(0.950583, 0.751901, 0.003556, 0.199230)
 {}
 void AdvanceBattleDialogDetector::make_overlays(VideoOverlaySet& items) const{
     const BoxOption& GAME_BOX = GameSettings::instance().GAME_BOX;
     items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_box));
     items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_top_box));
     items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_right_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_jpn_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_top_jpn_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_dialog_right_jpn_box));
 }
 bool AdvanceBattleDialogDetector::detect(const ImageViewRGB32& screen){
     ImageViewRGB32 game_screen = extract_box_reference(screen, GameSettings::instance().GAME_BOX);
@@ -217,13 +250,27 @@ bool AdvanceBattleDialogDetector::detect(const ImageViewRGB32& screen){
     cout << stats.average.b << endl;
     */
 
+    //japanese
+    ImageRGB32 filtered_region_jpn = filter_rgb32_range(
+        extract_box_reference(game_screen, m_dialog_jpn_box),
+        combine_rgb(164, 0, 0), combine_rgb(255, 114, 87), Color(0), replace_color_within_range
+    );
+    ImageStats stats2 = image_stats(filtered_region_jpn);
+
     ImageViewRGB32 dialog_top_image = extract_box_reference(game_screen, m_dialog_top_box);
     ImageViewRGB32 dialog_right_image = extract_box_reference(game_screen, m_dialog_right_box);
+    ImageViewRGB32 dialog_top_jpn_image = extract_box_reference(game_screen, m_dialog_top_jpn_box);
+    ImageViewRGB32 dialog_right_jpn_image = extract_box_reference(game_screen, m_dialog_right_jpn_box);
 
-    if (is_solid(dialog_top_image, { 0.176, 0.357, 0.467 }, 0.25, 20)
+    if ((is_solid(dialog_top_image, { 0.176, 0.357, 0.467 }, 0.25, 20)
         && is_solid(dialog_right_image, { 0.176, 0.357, 0.467 }, 0.25, 20)
         && (stats.average.r > stats.average.b + 180)
-        && (stats.average.r > stats.average.g + 180)
+        && (stats.average.r > stats.average.g + 180))
+        ||
+        (is_solid(dialog_top_jpn_image, { 0.176, 0.357, 0.467 }, 0.25, 20)
+        && is_solid(dialog_right_jpn_image, { 0.176, 0.357, 0.467 }, 0.25, 20)
+        && (stats2.average.r > stats2.average.b + 180)
+        && (stats2.average.r > stats2.average.g + 180))
     ){
         return true;
     }

SerialPrograms/Source/PokemonFRLG/Inference/Dialogs/PokemonFRLG_DialogDetector.h

@@ -138,6 +138,9 @@ class BattleDialogDetector : public StaticScreenDetector{
 private:
     ImageFloatBox m_dialog_top_box;
     ImageFloatBox m_dialog_right_box;
+
+    ImageFloatBox m_dialog_top_jpn_box;
+    ImageFloatBox m_dialog_right_jpn_box;
 };
 class BattleDialogWatcher : public DetectorToFinder<BattleDialogDetector>{
 public:
@@ -161,6 +164,11 @@ class BattleMenuDetector : public StaticScreenDetector{
     ImageFloatBox m_menu_right_box;
     ImageFloatBox m_dialog_top_box;
     ImageFloatBox m_dialog_right_box;
+
+    ImageFloatBox m_menu_top_jpn_box;
+    ImageFloatBox m_menu_right_jpn_box;
+    ImageFloatBox m_dialog_top_jpn_box;
+    ImageFloatBox m_dialog_right_jpn_box;
 };
 class BattleMenuWatcher : public DetectorToFinder<BattleMenuDetector>{
 public:
@@ -184,6 +192,10 @@ class AdvanceBattleDialogDetector : public StaticScreenDetector{
     ImageFloatBox m_dialog_box;
     ImageFloatBox m_dialog_top_box;
     ImageFloatBox m_dialog_right_box;
+
+    ImageFloatBox m_dialog_jpn_box;
+    ImageFloatBox m_dialog_top_jpn_box;
+    ImageFloatBox m_dialog_right_jpn_box;
 };
 class AdvanceBattleDialogWatcher : public DetectorToFinder<AdvanceBattleDialogDetector>{
 public:

SerialPrograms/Source/PokemonFRLG/Inference/Menus/PokemonFRLG_SummaryDetector.cpp

@@ -0,0 +1,103 @@
+/*  Summary Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include "CommonTools/Images/SolidColorTest.h"
+#include "CommonTools/Images/ImageFilter.h"
+#include "CommonFramework/ImageTools/ImageBoxes.h"
+#include "CommonFramework/ImageTools/ImageStats.h"
+#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
+#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
+#include "CommonTools/Images/SolidColorTest.h"
+#include "CommonTools/Images/WaterfillUtilities.h"
+#include "PokemonFRLG/PokemonFRLG_Settings.h"
+#include "PokemonFRLG_SummaryDetector.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonFRLG{
+
+SummaryDetector::SummaryDetector(Color color)
+    : m_page_1_box(0.441235, 0.033869, 0.016889, 0.030754) //white
+    , m_page_2_box(0.509681, 0.035206, 0.014223, 0.026742) //blue
+    , m_page_3_box(0.577239, 0.039218, 0.012445, 0.021394) //blue
+{}
+void SummaryDetector::make_overlays(VideoOverlaySet& items) const{
+    const BoxOption& GAME_BOX = GameSettings::instance().GAME_BOX;
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_page_1_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_page_2_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_page_3_box));
+}
+bool SummaryDetector::detect(const ImageViewRGB32& screen){
+    ImageViewRGB32 game_screen = extract_box_reference(screen, GameSettings::instance().GAME_BOX);
+
+    ImageViewRGB32 page_1_image = extract_box_reference(game_screen, m_page_1_box);
+    ImageViewRGB32 page_2_image = extract_box_reference(game_screen, m_page_2_box);
+    ImageViewRGB32 page_3_image = extract_box_reference(game_screen, m_page_3_box);
+    if (is_white(page_1_image)
+        && is_solid(page_2_image, { 0, 0.3333, 0.6666 }, 0.25, 20)
+        && is_solid(page_3_image, { 0, 0.3333, 0.6666 }, 0.25, 20)
+    ){
+        return true;
+    }
+    return false;
+}
+
+SummaryPage2Detector::SummaryPage2Detector(Color color)
+    : m_page_1_box(0.441235, 0.033869, 0.016889, 0.030754) //tan
+    , m_page_2_box(0.509681, 0.035206, 0.014223, 0.026742) //white
+    , m_page_3_box(0.577239, 0.039218, 0.012445, 0.021394) //blue
+{}
+void SummaryPage2Detector::make_overlays(VideoOverlaySet& items) const{
+    const BoxOption& GAME_BOX = GameSettings::instance().GAME_BOX;
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_page_1_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_page_2_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_page_3_box));
+}
+bool SummaryPage2Detector::detect(const ImageViewRGB32& screen){
+    ImageViewRGB32 game_screen = extract_box_reference(screen, GameSettings::instance().GAME_BOX);
+
+    ImageViewRGB32 page_1_image = extract_box_reference(game_screen, m_page_1_box);
+    ImageViewRGB32 page_2_image = extract_box_reference(game_screen, m_page_2_box);
+    ImageViewRGB32 page_3_image = extract_box_reference(game_screen, m_page_3_box);
+    if (is_solid(page_1_image, { 0.5117, 0.3763, 0.2294 }, 0.25, 20)
+        && is_white(page_2_image)
+        && is_solid(page_3_image, { 0, 0.3333, 0.6666 }, 0.25, 20)
+    ){
+        return true;
+    }
+    return false;
+}
+
+SummaryPage3Detector::SummaryPage3Detector(Color color)
+    : m_page_1_box(0.441235, 0.033869, 0.016889, 0.030754) //tan
+    , m_page_2_box(0.509681, 0.035206, 0.014223, 0.026742) //tan
+    , m_page_3_box(0.577239, 0.039218, 0.012445, 0.021394) //white
+{}
+void SummaryPage3Detector::make_overlays(VideoOverlaySet& items) const{
+    const BoxOption& GAME_BOX = GameSettings::instance().GAME_BOX;
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_page_1_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_page_2_box));
+    items.add(COLOR_RED, GAME_BOX.inner_to_outer(m_page_3_box));
+}
+bool SummaryPage3Detector::detect(const ImageViewRGB32& screen){
+    ImageViewRGB32 game_screen = extract_box_reference(screen, GameSettings::instance().GAME_BOX);
+
+    ImageViewRGB32 page_1_image = extract_box_reference(game_screen, m_page_1_box);
+    ImageViewRGB32 page_2_image = extract_box_reference(game_screen, m_page_2_box);
+    ImageViewRGB32 page_3_image = extract_box_reference(game_screen, m_page_3_box);
+    if (is_solid(page_1_image, { 0.5117, 0.3763, 0.2294 }, 0.25, 20)
+        && is_solid(page_2_image, { 0.5117, 0.3763, 0.2294 }, 0.25, 20)
+        && is_white(page_3_image)
+    ){
+        return true;
+    }
+    return false;
+}
+
+
+}
+}
+}

SerialPrograms/Source/PokemonFRLG/Inference/Menus/PokemonFRLG_SummaryDetector.h

@@ -0,0 +1,93 @@
+/*  Summary Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#ifndef PokemonAutomation_PokemonFRLG_SummaryDetector_H
+#define PokemonAutomation_PokemonFRLG_SummaryDetector_H
+
+#include <chrono>
+#include <atomic>
+#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
+#include "Common/Cpp/Color.h"
+#include "CommonFramework/ImageTools/ImageBoxes.h"
+#include "CommonTools/VisualDetector.h"
+#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h"
+
+namespace PokemonAutomation{
+    class CancellableScope;
+    class VideoFeed;
+namespace NintendoSwitch{
+namespace PokemonFRLG{
+
+// Detect the default (first) Pokemon Summary page
+// This has nature and OT, but no stats
+// This will work with all languages, as the page indicators are in the same positions
+class SummaryDetector : public StaticScreenDetector{
+public:
+    SummaryDetector(Color color);
+
+    virtual void make_overlays(VideoOverlaySet& items) const override;
+    virtual bool detect(const ImageViewRGB32& screen) override;
+
+private:
+    ImageFloatBox m_page_1_box;
+    ImageFloatBox m_page_2_box;
+    ImageFloatBox m_page_3_box;
+};
+class SummaryWatcher : public DetectorToFinder<SummaryDetector>{
+public:
+    SummaryWatcher(Color color)
+        : DetectorToFinder("SummaryWatcher", std::chrono::milliseconds(250), color)
+    {}
+};
+
+// Detect that we are on page 2 of a pokemon summary
+// This is the stats screen
+// Note that when switching pages, wait a bit before pressing left/right
+class SummaryPage2Detector : public StaticScreenDetector{
+public:
+    SummaryPage2Detector(Color color);
+
+    virtual void make_overlays(VideoOverlaySet& items) const override;
+    virtual bool detect(const ImageViewRGB32& screen) override;
+
+private:
+    ImageFloatBox m_page_1_box;
+    ImageFloatBox m_page_2_box;
+    ImageFloatBox m_page_3_box;
+};
+class SummaryPage2Watcher : public DetectorToFinder<SummaryPage2Detector>{
+public:
+    SummaryPage2Watcher(Color color)
+        : DetectorToFinder("SummaryPage2Watcher", std::chrono::milliseconds(250), color)
+    {}
+};
+
+// Detect that we are on page 3 of a pokemon summary
+// This contains the pokemon's moves
+class SummaryPage3Detector : public StaticScreenDetector{
+public:
+    SummaryPage3Detector(Color color);
+
+    virtual void make_overlays(VideoOverlaySet& items) const override;
+    virtual bool detect(const ImageViewRGB32& screen) override;
+
+private:
+    ImageFloatBox m_page_1_box;
+    ImageFloatBox m_page_2_box;
+    ImageFloatBox m_page_3_box;
+};
+class SummaryPage3Watcher : public DetectorToFinder<SummaryPage3Detector>{
+public:
+    SummaryPage3Watcher(Color color)
+        : DetectorToFinder("SummaryPage3Watcher", std::chrono::milliseconds(250), color)
+    {}
+};
+
+}
+}
+}
+
+#endif

SerialPrograms/Source/PokemonFRLG/PokemonFRLG_Navigation.cpp

@@ -18,6 +18,7 @@
 #include "PokemonFRLG/Inference/Sounds/PokemonFRLG_ShinySoundDetector.h"
 #include "PokemonFRLG/Inference/Menus/PokemonFRLG_StartMenuDetector.h"
 #include "PokemonFRLG/Inference/Menus/PokemonFRLG_LoadMenuDetector.h"
+#include "PokemonFRLG/Inference/Menus/PokemonFRLG_SummaryDetector.h"
 #include "PokemonFRLG/PokemonFRLG_Settings.h"
 #include "PokemonFRLG_Navigation.h"
 
@@ -181,6 +182,21 @@ bool try_open_slot_six(ConsoleHandle& console, ProControllerContext& context){
         console.log("open_slot_six(): Unable to enter summary.", COLOR_RED);
         return false;
     }
+
+    //Double check that we are on summary
+    SummaryWatcher sum1(COLOR_RED);
+    int sm1 = wait_until(
+        console, context,
+        std::chrono::seconds(5),
+        {{ sum1 }}
+    );
+    if (sm1 == 0){
+        console.log("Summary page dots detected.");
+    }else{
+        console.log("open_slot_six(): Unable to detect summary screen.", COLOR_RED);
+        return false;
+    }
+
     pbf_wait(context, 1000ms);
     context.wait_for_all_requests();
     return true;