Better reliability for FRLG auto-catching and species detection

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_PokemonSpriteReader.cpp

@@ -0,0 +1,98 @@
+/*  Pokemon FRLG Sprite Reader
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
+#include "CommonFramework/ImageTools/ImageStats.h"
+#include "CommonFramework/GlobalSettingsPanel.h"
+#include "CommonTools/ImageMatch/ImageCropper.h"
+#include "PokemonFRLG/Resources/PokemonFRLG_PokemonSprites.h"
+#include "PokemonFRLG/PokemonFRLG_Settings.h"
+#include "PokemonFRLG_PokemonSpriteReader.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonFRLG{
+
+
+PokemonSpriteMatcherExact::PokemonSpriteMatcherExact(const std::set<std::string>* subset, bool flipped)
+    : ExactImageDictionaryMatcher({1, 256})
+{
+    for (const auto& item : (flipped ? FLIPPED_POKEMON_SPRITES() : ALL_POKEMON_SPRITES())){
+        if (subset == nullptr || subset->find(item.first) != subset->end()){
+            add(item.first, item.second.sprite.copy());
+        }
+    }
+}
+
+
+PokemonSpriteMatcherCropped::PokemonSpriteMatcherCropped(const std::set<std::string>* subset, double min_euclidean_distance, bool flipped)
+    : CroppedImageDictionaryMatcher({1, 256})
+    , m_min_euclidean_distance_squared(min_euclidean_distance * min_euclidean_distance)
+{
+    for (const auto& item : (flipped ? FLIPPED_POKEMON_SPRITES() : ALL_POKEMON_SPRITES())){
+        if (subset == nullptr || subset->find(item.first) != subset->end()){
+            add(item.first, item.second.sprite);
+        }
+    }
+}
+
+auto PokemonSpriteMatcherCropped::get_crop_candidates(const ImageViewRGB32& image) const -> std::vector<ImageViewRGB32>{
+    ImageStats border = image_border_stats(image);
+    ImagePixelBox box = ImageMatch::enclosing_rectangle_with_pixel_filter(
+        image,
+        [&](Color pixel){
+            double r = (double)pixel.red() - border.average.r;
+            double g = (double)pixel.green() - border.average.g;
+            double b = (double)pixel.blue() - border.average.b;
+            bool stop = r*r + g*g + b*b >= m_min_euclidean_distance_squared;
+            return stop;
+        }
+    );
+    std::vector<ImageViewRGB32> ret;
+    ret.emplace_back(extract_box_reference(image, box));
+    return ret;
+}
+
+
+SummarySpriteReader::SummarySpriteReader(const std::set<std::string>& subset, Color color)
+    : m_color(color), 
+    m_box_sprite(0.113462, 0.209375, 0.274519, 0.381490),
+    sprite_matcher(&subset, 100, true),
+    exact_sprite_matcher(&subset, true)
+{}
+
+void SummarySpriteReader::make_overlays(VideoOverlaySet &items) const {
+    const BoxOption &GAME_BOX = GameSettings::instance().GAME_BOX;
+    items.add(m_color, GAME_BOX.inner_to_outer(m_box_sprite));
+}
+
+ImageMatch::ImageMatchResult SummarySpriteReader::read(const ImageViewRGB32& frame){
+    static const double CROPPED_ALPHA_SPREAD = 0.03;
+    static const double EXACT_ALPHA_SPREAD = 0.02;
+
+    static const double RETRY_ALPHA = 0.25;
+
+    ImageViewRGB32 game_screen =
+        extract_box_reference(frame, GameSettings::instance().GAME_BOX);
+
+    ImageViewRGB32 sprite_image = extract_box_reference(game_screen, m_box_sprite);
+
+    ImageMatch::ImageMatchResult result = sprite_matcher.match(sprite_image, CROPPED_ALPHA_SPREAD);
+    if (result.results.size() != 1 || result.results.begin()->first > RETRY_ALPHA){
+        result = exact_sprite_matcher.match(
+            sprite_image, m_box_sprite,
+            5, EXACT_ALPHA_SPREAD
+        );
+    }
+
+    return result;
+}
+
+
+
+}
+}
+}

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_PokemonSpriteReader.h

@@ -0,0 +1,57 @@
+/*  Pokemon FRLG Sprite Reader
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#ifndef PokemonAutomation_PokemonFRLG_PokemonSpriteReader_H
+#define PokemonAutomation_PokemonFRLG_PokemonSpriteReader_H
+
+#include <set>
+#include "CommonTools/ImageMatch/ExactImageDictionaryMatcher.h"
+#include "CommonTools/ImageMatch/CroppedImageDictionaryMatcher.h"
+#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonFRLG{
+
+
+class PokemonSpriteMatcherExact : public ImageMatch::ExactImageDictionaryMatcher{
+public:
+    PokemonSpriteMatcherExact(const std::set<std::string>* subset, bool flipped = true);
+};
+
+
+class PokemonSpriteMatcherCropped : public ImageMatch::CroppedImageDictionaryMatcher{
+public:
+    PokemonSpriteMatcherCropped(const std::set<std::string>* subset, double min_euclidean_distance = 100, bool flipped = true);
+
+private:
+    virtual std::vector<ImageViewRGB32> get_crop_candidates(const ImageViewRGB32& image) const override;
+
+private:
+    double m_min_euclidean_distance_squared;
+};
+
+
+class SummarySpriteReader{
+public:
+    SummarySpriteReader(const std::set<std::string>& subset, Color color = COLOR_RED);
+
+    void make_overlays(VideoOverlaySet& items) const;
+
+    ImageMatch::ImageMatchResult read(const ImageViewRGB32& frame);
+
+private:
+    Color m_color;
+    ImageFloatBox m_box_sprite;
+    PokemonSpriteMatcherCropped sprite_matcher;
+    PokemonSpriteMatcherExact exact_sprite_matcher;
+};
+
+
+}
+}
+}
+#endif

SerialPrograms/Source/PokemonFRLG/Inference/Sounds/PokemonFRLG_CatchFanfareDetector.cpp

@@ -0,0 +1,45 @@
+/*  Catch Fanfare Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include "CommonTools/Audio/AudioTemplateCache.h"
+#include "CommonTools/Audio/SpectrogramMatcher.h"
+#include "NintendoSwitch/NintendoSwitch_ConsoleHandle.h"
+#include "PokemonFRLG/PokemonFRLG_Settings.h"
+#include "PokemonFRLG_CatchFanfareDetector.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonFRLG{
+
+
+CatchFanfareDetector::CatchFanfareDetector(Logger& logger, DetectedCallback detected_callback)
+    : AudioPerSpectrumDetectorBase(
+        logger,
+        "CatchFanfareDetector",
+        "Catch fanfare",
+        COLOR_BLUE,
+        detected_callback
+    )
+{}
+
+
+float CatchFanfareDetector::get_score_threshold() const{
+    return (float)GameSettings::instance().CATCH_FANFARE_THRESHOLD;
+}
+
+std::unique_ptr<SpectrogramMatcher> CatchFanfareDetector::build_spectrogram_matcher(size_t sample_rate){
+    return std::make_unique<SpectrogramMatcher>(
+        "Catch fanfare",
+        AudioTemplateCache::instance().get_throw("PokemonFRLG/CatchFanfare", sample_rate),
+        SpectrogramMatcher::Mode::SPIKE_CONV, sample_rate, 50
+    );
+}
+
+
+
+}
+}
+}

SerialPrograms/Source/PokemonFRLG/Inference/Sounds/PokemonFRLG_CatchFanfareDetector.h

@@ -0,0 +1,36 @@
+/*  Catch Fanfare Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#ifndef PokemonAutomation_PokemonFRLG_CatchFanfareDetector_H
+#define PokemonAutomation_PokemonFRLG_CatchFanfareDetector_H
+
+#include "CommonTools/Audio/AudioPerSpectrumDetectorBase.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonFRLG{
+
+
+class CatchFanfareDetector : public AudioPerSpectrumDetectorBase{
+public:
+    //  Warning: The callback will be called from the audio inference thread.
+    CatchFanfareDetector(Logger& logger, DetectedCallback detected_callback);
+
+    // Implement AudioPerSpectrumDetectorBase::get_score_threshold()
+    virtual float get_score_threshold() const override;
+
+protected:
+    // Implement AudioPerSpectrumDetectorBase::build_spectrogram_matcher()
+    virtual std::unique_ptr<SpectrogramMatcher> build_spectrogram_matcher(size_t sample_rate) override;
+};
+
+
+
+
+}
+}
+}
+#endif

SerialPrograms/Source/PokemonFRLG/PokemonFRLG_Settings.cpp

@@ -76,6 +76,12 @@ GameSettings::GameSettings()
         LockMode::LOCK_WHILE_RUNNING,
         1000, 0, 48000 //2000
     )
+    , m_catch_audio_settings("<font size=4><b>Catch Audio Settings:</b></font>")
+    , CATCH_FANFARE_THRESHOLD(
+        "<b>Shiny Sound Threshold:</b><br>Maximum error coefficient to trigger catch detection.",
+        LockMode::LOCK_WHILE_RUNNING,
+        0.80, 0, 1.0
+    )
 {
     PA_ADD_STATIC(m_game_device_settings);
     PA_ADD_OPTION(DEVICE);
@@ -87,6 +93,7 @@ GameSettings::GameSettings()
     PA_ADD_STATIC(m_shiny_audio_settings);
     PA_ADD_OPTION(SHINY_SOUND_THRESHOLD);
     PA_ADD_OPTION(SHINY_SOUND_LOW_FREQUENCY);
+    PA_ADD_OPTION(m_catch_audio_settings);
 
     GameSettings::on_config_value_changed(this);
     DEVICE.add_listener(*this);

SerialPrograms/Source/PokemonFRLG/PokemonFRLG_Settings.h

@@ -45,6 +45,9 @@ class GameSettings : public BatchOption, private ConfigOption::Listener{
     FloatingPointOption SHINY_SOUND_THRESHOLD;
     FloatingPointOption SHINY_SOUND_LOW_FREQUENCY;
 
+    SectionDividerOption m_catch_audio_settings;
+    FloatingPointOption CATCH_FANFARE_THRESHOLD;
+
 private:
     virtual void on_config_value_changed(void* object) override;
 };

SerialPrograms/Source/PokemonFRLG/Programs/RngManipulation/PokemonFRLG_RngCalibration.cpp

@@ -123,7 +123,7 @@ RngTimings prepare_timings(
     console.log("CSF calibration (frames): " + std::to_string(calibrations.csf_offset));
     console.log("In-game calibration (frames x2): " + std::to_string(calibrations.ingame_offset));
 
-    double seed_delay = SEED_DELAY + calibrations.seed_offset + FIXED_SEED_OFFSET;
+    double seed_delay = SEED_DELAY + (calibrations.seed_offset * FRLG_FRAME_DURATION) + FIXED_SEED_OFFSET;
     double csf_delay = (CONTINUE_SCREEN_FRAMES + calibrations.csf_offset) * FRLG_FRAME_DURATION;
     double teachy_delay = TEACHY_ADVANCES * FRLG_FRAME_DURATION / 313;
     double ingame_delay = (modified_ingame_advances - TEACHY_ADVANCES) * FRLG_FRAME_DURATION / 2 - (should_use_teachy_tv ? 14067 : 0);
@@ -442,6 +442,7 @@ bool update_history(
     }
 
     if (search_hits.size() == 1){
+        console.log("Hit " + to_hex_string(search_hits[0].seed) + " / " + std::to_string(search_hits[0].advance));
         console.log("Updating calibrations...");
         calibration_history.calibrations.emplace_back(calibrations);
         calibration_history.results.emplace_back(search_hits[0]);
@@ -515,7 +516,7 @@ bool update_history(
         }
         calibration_history.calibrations.emplace_back(calibrations);
         calibration_history.results.emplace_back(most_likely_hit);
-        console.log("   " + std::to_string(most_likely_hit.seed) + " / " + std::to_string(most_likely_hit.advance));
+        console.log("   " + to_hex_string(most_likely_hit.seed) + " / " + std::to_string(most_likely_hit.advance));
     }