From 54849f1cac8ee5bd3f8eafc92f424f29526291a0 Mon Sep 17 00:00:00 2001 From: Nat Date: Fri, 4 Apr 2025 12:41:57 +0800 Subject: [PATCH 1/6] FEATURE: Translate categories and display them when inline translations enabled --- .../automatic_translation_backfill.rb | 33 +++++++++--- .../discourse_translator/category_locale.rb | 12 +++++ .../category_translation.rb | 14 +++++ .../discourse_ai/category_translator.rb | 37 +++++++++++++ app/services/discourse_ai/topic_translator.rb | 4 +- app/services/discourse_translator/base.rb | 2 + .../discourse_translator/discourse_ai.rb | 5 ++ ...15139_create_category_translation_table.rb | 23 ++++++++ .../extensions/category_extension.rb | 11 ++++ .../inline_translation.rb | 28 +++++++--- .../translated_content_normalizer.rb | 2 + plugin.rb | 1 + .../automatic_translation_backfill_spec.rb | 52 +++++++------------ 13 files changed, 175 insertions(+), 49 deletions(-) create mode 100644 app/models/discourse_translator/category_locale.rb create mode 100644 app/models/discourse_translator/category_translation.rb create mode 100644 app/services/discourse_ai/category_translator.rb create mode 100644 db/migrate/20250401015139_create_category_translation_table.rb create mode 100644 lib/discourse_translator/extensions/category_extension.rb diff --git a/app/jobs/scheduled/automatic_translation_backfill.rb b/app/jobs/scheduled/automatic_translation_backfill.rb index 0366a50..d8c14f2 100644 --- a/app/jobs/scheduled/automatic_translation_backfill.rb +++ b/app/jobs/scheduled/automatic_translation_backfill.rb @@ -24,10 +24,10 @@ def fetch_untranslated_model_ids(model, content_column, limit, target_locale) SELECT m.id FROM #{model.table_name} m #{limit_to_public_clause(model)} - WHERE m.deleted_at IS NULL - AND m.#{content_column} != '' - AND m.user_id > 0 - #{max_age_clause} + WHERE m.#{content_column} != '' + #{not_deleted_clause(model)} + #{non_bot_clause(model)} + #{max_age_clause(model)} ORDER BY m.updated_at DESC ) EXCEPT @@ -91,22 +91,29 @@ def process_batch topic_ids = fetch_untranslated_model_ids(Topic, "title", records_to_translate, target_locale) post_ids = fetch_untranslated_model_ids(Post, "raw", records_to_translate, target_locale) + category_ids = + fetch_untranslated_model_ids(Category, "name", records_to_translate, target_locale) - next if topic_ids.empty? && post_ids.empty? + next if topic_ids.empty? && post_ids.empty? && category_ids.empty? DiscourseTranslator::VerboseLogger.log( - "Translating #{topic_ids.size} topics and #{post_ids.size} posts to #{target_locale}", + "Translating #{topic_ids.size} topics, #{post_ids.size} posts, #{category_ids.size} categories, to #{target_locale}", ) translate_records(Topic, topic_ids, target_locale) translate_records(Post, post_ids, target_locale) + translate_records(Category, category_ids, target_locale) end end - def max_age_clause + def max_age_clause(model) return "" if SiteSetting.automatic_translation_backfill_max_age_days <= 0 - "AND m.created_at > NOW() - INTERVAL '#{SiteSetting.automatic_translation_backfill_max_age_days} days'" + if model == Post || model == Topic + "AND m.created_at > NOW() - INTERVAL '#{SiteSetting.automatic_translation_backfill_max_age_days} days'" + else + "" + end end def limit_to_public_clause(model) @@ -130,5 +137,15 @@ def limit_to_public_clause(model) limit_to_public_clause end + + def non_bot_clause(model) + return "AND m.user_id > 0" if model == Post || model == Topic + "" + end + + def not_deleted_clause(model) + return "AND m.deleted_at IS NULL" if model == Post || model == Topic + "" + end end end diff --git a/app/models/discourse_translator/category_locale.rb b/app/models/discourse_translator/category_locale.rb new file mode 100644 index 0000000..e1dbde3 --- /dev/null +++ b/app/models/discourse_translator/category_locale.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class CategoryLocale < ActiveRecord::Base + self.table_name = "discourse_translator_category_locales" + + belongs_to :category + + validates :category_id, presence: true + validates :detected_locale, presence: true + end +end diff --git a/app/models/discourse_translator/category_translation.rb b/app/models/discourse_translator/category_translation.rb new file mode 100644 index 0000000..a769ff3 --- /dev/null +++ b/app/models/discourse_translator/category_translation.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class CategoryTranslation < ActiveRecord::Base + self.table_name = "discourse_translator_category_translations" + + belongs_to :category + + validates :category_id, presence: true + validates :locale, presence: true + validates :translation, presence: true + validates :locale, uniqueness: { scope: :category_id } + end +end diff --git a/app/services/discourse_ai/category_translator.rb b/app/services/discourse_ai/category_translator.rb new file mode 100644 index 0000000..789779a --- /dev/null +++ b/app/services/discourse_ai/category_translator.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module DiscourseAi + class CategoryTranslator < BaseTranslator + PROMPT_TEMPLATE = <<~TEXT.freeze + You are a translation service specializing in translating forum category names to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: + + 1. Translate the category name to target_language asked + 2. Keep proper nouns and technical terms in their original language + 3. Keep the translated category name length short, and close to the original length + 4. Ensure the translation maintains the original meaning + + Provide your translation in the following JSON format: + + + {"translation": "Your target_language translation here"} + + + Here are three examples of correct translation + + Original: {"name":"Cats and Dogs", "target_language":"Chinese"} + Correct translation: {"translation": "猫和狗"} + + Original: {"name":"General", "target_language":"French"} + Correct translation: {"translation": "Général"} + + Original: {"name": "Q&A", "target_language": "Portuguese"} + Correct translation: {"translation": "Perguntas e Respostas"} + + Remember to keep proper nouns like "Minecraft" and "Toyota" in their original form. Translate the category name now and provide your answer in the specified JSON format. + TEXT + + private def prompt_template + PROMPT_TEMPLATE + end + end +end diff --git a/app/services/discourse_ai/topic_translator.rb b/app/services/discourse_ai/topic_translator.rb index 99ca595..7343135 100644 --- a/app/services/discourse_ai/topic_translator.rb +++ b/app/services/discourse_ai/topic_translator.rb @@ -3,9 +3,9 @@ module DiscourseAi class TopicTranslator < BaseTranslator PROMPT_TEMPLATE = <<~TEXT.freeze - You are a translation service specializing in translating forum post titles from English to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: + You are a translation service specializing in translating forum post titles to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: - 1. Translate the given title from English to target_language asked. + 1. Translate the given title to target_language asked. 2. Keep proper nouns and technical terms in their original language. 3. Attempt to keep the translated title length close to the original when possible. 4. Ensure the translation maintains the original meaning and tone. diff --git a/app/services/discourse_translator/base.rb b/app/services/discourse_translator/base.rb index f43c7f1..2b5e7c4 100644 --- a/app/services/discourse_translator/base.rb +++ b/app/services/discourse_translator/base.rb @@ -137,6 +137,8 @@ def self.get_untranslated(translatable, raw: false) raw ? translatable.raw : translatable.cooked when "Topic" translatable.title + when "Category" + translatable.name end end end diff --git a/app/services/discourse_translator/discourse_ai.rb b/app/services/discourse_translator/discourse_ai.rb index 947a3cb..594fc10 100644 --- a/app/services/discourse_translator/discourse_ai.rb +++ b/app/services/discourse_translator/discourse_ai.rb @@ -42,6 +42,11 @@ def self.translate!(translatable, target_locale_sym = I18n.locale) .join("") when "Topic" ::DiscourseAi::TopicTranslator.new(text_for_translation(translatable), language).translate + when "Category" + ::DiscourseAi::CategoryTranslator.new( + text_for_translation(translatable), + language, + ).translate end DiscourseTranslator::TranslatedContentNormalizer.normalize(translatable, translated) diff --git a/db/migrate/20250401015139_create_category_translation_table.rb b/db/migrate/20250401015139_create_category_translation_table.rb new file mode 100644 index 0000000..bdc5ad5 --- /dev/null +++ b/db/migrate/20250401015139_create_category_translation_table.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class CreateCategoryTranslationTable < ActiveRecord::Migration[7.2] + def change + create_table :discourse_translator_category_locales do |t| + t.integer :category_id, null: false + t.string :detected_locale, limit: 20, null: false + t.timestamps + end + + create_table :discourse_translator_category_translations do |t| + t.integer :category_id, null: false + t.string :locale, null: false + t.text :translation, null: false + t.timestamps + end + + add_index :discourse_translator_category_translations, + %i[category_id locale], + unique: true, + name: "idx_category_translations_on_category_id_and_locale" + end +end diff --git a/lib/discourse_translator/extensions/category_extension.rb b/lib/discourse_translator/extensions/category_extension.rb new file mode 100644 index 0000000..e8c4f49 --- /dev/null +++ b/lib/discourse_translator/extensions/category_extension.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module DiscourseTranslator + module Extensions + module CategoryExtension + extend ActiveSupport::Concern + prepended { before_update :clear_translations, if: :name_changed? } + include Translatable + end + end +end diff --git a/lib/discourse_translator/inline_translation.rb b/lib/discourse_translator/inline_translation.rb index f7544d0..3b22b88 100644 --- a/lib/discourse_translator/inline_translation.rb +++ b/lib/discourse_translator/inline_translation.rb @@ -15,6 +15,8 @@ def inject(plugin) # always return early if topic and posts are in the user's effective_locale. # this prevents the need to load translations. + # posts + plugin.register_modifier(:basic_post_serializer_cooked) do |cooked, serializer| if !SiteSetting.experimental_inline_translation || serializer.object.locale_matches?(InlineTranslation.effective_locale) || @@ -25,6 +27,14 @@ def inject(plugin) end end + plugin.add_to_serializer(:basic_post, :is_translated) do + SiteSetting.experimental_inline_translation && + !object.locale_matches?(InlineTranslation.effective_locale) && + object.translation_for(InlineTranslation.effective_locale).present? + end + + # topics + plugin.register_modifier(:topic_serializer_fancy_title) do |fancy_title, serializer| if !SiteSetting.experimental_inline_translation || serializer.object.locale_matches?(InlineTranslation.effective_locale) || @@ -54,12 +64,6 @@ def inject(plugin) end end - plugin.add_to_serializer(:basic_post, :is_translated) do - SiteSetting.experimental_inline_translation && - !object.locale_matches?(InlineTranslation.effective_locale) && - object.translation_for(InlineTranslation.effective_locale).present? - end - plugin.add_to_serializer(:topic_view, :is_translated) do SiteSetting.experimental_inline_translation && !object.topic.locale_matches?(InlineTranslation.effective_locale) && @@ -72,6 +76,18 @@ def inject(plugin) plugin.register_topic_preloader_associations(:translations) do SiteSetting.translator_enabled && SiteSetting.experimental_inline_translation end + + # categories + + plugin.register_modifier(:site_category_serializer_name) do |name, serializer| + if !SiteSetting.experimental_inline_translation || + serializer.object.locale_matches?(InlineTranslation.effective_locale) || + serializer.scope&.request&.params&.[]("show") == "original" + name + else + serializer.object.translation_for(InlineTranslation.effective_locale).presence + end + end end end end diff --git a/lib/discourse_translator/translated_content_normalizer.rb b/lib/discourse_translator/translated_content_normalizer.rb index 24884f5..0dbf519 100644 --- a/lib/discourse_translator/translated_content_normalizer.rb +++ b/lib/discourse_translator/translated_content_normalizer.rb @@ -8,6 +8,8 @@ def self.normalize(translatable, content) PrettyText.cook(content) when "Topic" PrettyText.cleanup(content, {}) + when "Category" + content end end end diff --git a/plugin.rb b/plugin.rb index 12157bf..1f1475f 100644 --- a/plugin.rb +++ b/plugin.rb @@ -29,6 +29,7 @@ module ::DiscourseTranslator Guardian.prepend(DiscourseTranslator::Extensions::GuardianExtension) Post.prepend(DiscourseTranslator::Extensions::PostExtension) Topic.prepend(DiscourseTranslator::Extensions::TopicExtension) + Category.prepend(DiscourseTranslator::Extensions::CategoryExtension) TopicViewSerializer.prepend(DiscourseTranslator::Extensions::TopicViewSerializerExtension) end diff --git a/spec/jobs/automatic_translation_backfill_spec.rb b/spec/jobs/automatic_translation_backfill_spec.rb index 56dc036..06cee85 100644 --- a/spec/jobs/automatic_translation_backfill_spec.rb +++ b/spec/jobs/automatic_translation_backfill_spec.rb @@ -8,41 +8,24 @@ end def expect_google_check_language - Excon - .expects(:post) - .with(DiscourseTranslator::Google::SUPPORT_URI, anything, anything) - .returns( - Struct.new(:status, :body).new( - 200, - %{ { "data": { "languages": [ { "language": "es" }, { "language": "de" }] } } }, - ), - ) - .at_least_once + stub_request(:post, DiscourseTranslator::Google::SUPPORT_URI).to_return( + status: 200, + body: %{ { "data": { "languages": [ { "language": "es" }, { "language": "de" }] } } }, + ) end def expect_google_detect(locale) - Excon - .expects(:post) - .with(DiscourseTranslator::Google::DETECT_URI, anything, anything) - .returns( - Struct.new(:status, :body).new( - 200, - %{ { "data": { "detections": [ [ { "language": "#{locale}" } ] ] } } }, - ), - ) - .once + stub_request(:post, DiscourseTranslator::Google::DETECT_URI).to_return( + status: 200, + body: %{ { "data": { "detections": [ [ { "language": "#{locale}" } ] ] } } }, + ) end def expect_google_translate(text) - Excon - .expects(:post) - .with(DiscourseTranslator::Google::TRANSLATE_URI, body: anything, headers: anything) - .returns( - Struct.new(:status, :body).new( - 200, - %{ { "data": { "translations": [ { "translatedText": "#{text}" } ] } } }, - ), - ) + stub_request(:post, DiscourseTranslator::Google::TRANSLATE_URI).to_return( + status: 200, + body: %{ { "data": { "translations": [ { "translatedText": "#{text}" } ] } } }, + ) end describe "backfilling" do @@ -59,7 +42,6 @@ def expect_google_translate(text) end it "does not backfill if backfill limit is set to 0" do - SiteSetting.automatic_translation_backfill_rate = 1 SiteSetting.automatic_translation_target_languages = "de" SiteSetting.automatic_translation_backfill_rate = 0 expect_any_instance_of(Jobs::AutomaticTranslationBackfill).not_to receive(:process_batch) @@ -164,14 +146,17 @@ def expect_google_translate(text) expect_google_check_language end - it "backfills all (1) topics and (4) posts as it is within the maximum per job run" do - topic = Fabricate(:topic) + it "backfills all (1) category (1) topic (4) posts as it is within the maximum per job run" do + category = Fabricate(:category) + topic = Fabricate(:topic, category: category) posts = Fabricate.times(4, :post, topic: topic) topic.set_detected_locale("es") posts.each { |p| p.set_detected_locale("es") } + Category.all.each { |c| c.set_detected_locale("es") } - expect_google_translate("hallo").times(5) + expect_google_translate("hallo") + # .times(6) described_class.new.execute @@ -179,6 +164,7 @@ def expect_google_translate(text) expect(posts.map { |p| p.translations.pluck(:locale, :translation).flatten }).to eq( [%w[de hallo]] * 4, ) + expect(category.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) end end end From 0836516d451911282a41d1b9a1aba3c6476edd85 Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 8 Apr 2025 10:26:47 +0800 Subject: [PATCH 2/6] Tags --- .../automatic_translation_backfill.rb | 34 +++++++++-------- app/models/discourse_translator/tag_locale.rb | 12 ++++++ .../discourse_translator/tag_translation.rb | 14 +++++++ app/services/discourse_ai/tag_translator.rb | 38 +++++++++++++++++++ app/services/discourse_translator/base.rb | 2 + .../discourse_translator/discourse_ai.rb | 2 + ...0401022618_create_tag_translation_table.rb | 23 +++++++++++ .../extensions/tag_extension.rb | 11 ++++++ .../inline_translation.rb | 18 +++++++++ .../translated_content_normalizer.rb | 2 + plugin.rb | 1 + 11 files changed, 141 insertions(+), 16 deletions(-) create mode 100644 app/models/discourse_translator/tag_locale.rb create mode 100644 app/models/discourse_translator/tag_translation.rb create mode 100644 app/services/discourse_ai/tag_translator.rb create mode 100644 db/migrate/20250401022618_create_tag_translation_table.rb create mode 100644 lib/discourse_translator/extensions/tag_extension.rb diff --git a/app/jobs/scheduled/automatic_translation_backfill.rb b/app/jobs/scheduled/automatic_translation_backfill.rb index d8c14f2..bb5f35c 100644 --- a/app/jobs/scheduled/automatic_translation_backfill.rb +++ b/app/jobs/scheduled/automatic_translation_backfill.rb @@ -87,22 +87,24 @@ def translate_records(type, record_ids, target_locale) def process_batch records_to_translate = SiteSetting.automatic_translation_backfill_rate - backfill_locales.each_with_index do |target_locale, i| - topic_ids = - fetch_untranslated_model_ids(Topic, "title", records_to_translate, target_locale) - post_ids = fetch_untranslated_model_ids(Post, "raw", records_to_translate, target_locale) - category_ids = - fetch_untranslated_model_ids(Category, "name", records_to_translate, target_locale) - - next if topic_ids.empty? && post_ids.empty? && category_ids.empty? - - DiscourseTranslator::VerboseLogger.log( - "Translating #{topic_ids.size} topics, #{post_ids.size} posts, #{category_ids.size} categories, to #{target_locale}", - ) - - translate_records(Topic, topic_ids, target_locale) - translate_records(Post, post_ids, target_locale) - translate_records(Category, category_ids, target_locale) + backfill_locales.each do |target_locale| + [ + [Topic, "title"], + [Post, "raw"], + [Category, "name"], + [Tag, "name"], + ].each do |model, content_column| + ids = + fetch_untranslated_model_ids(model, content_column, records_to_translate, target_locale) + + next if ids.empty? + + DiscourseTranslator::VerboseLogger.log( + "Translating #{ids.size} #{model.name} to #{target_locale}", + ) + + translate_records(model, ids, target_locale) + end end end diff --git a/app/models/discourse_translator/tag_locale.rb b/app/models/discourse_translator/tag_locale.rb new file mode 100644 index 0000000..716336b --- /dev/null +++ b/app/models/discourse_translator/tag_locale.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class TagLocale < ActiveRecord::Base + self.table_name = "discourse_translator_tag_locales" + + belongs_to :tag + + validates :tag_id, presence: true + validates :detected_locale, presence: true + end +end diff --git a/app/models/discourse_translator/tag_translation.rb b/app/models/discourse_translator/tag_translation.rb new file mode 100644 index 0000000..79d3fef --- /dev/null +++ b/app/models/discourse_translator/tag_translation.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class TagTranslation < ActiveRecord::Base + self.table_name = "discourse_translator_tag_translations" + + belongs_to :tag + + validates :tag_id, presence: true + validates :locale, presence: true + validates :translation, presence: true + validates :locale, uniqueness: { scope: :tag_id } + end +end diff --git a/app/services/discourse_ai/tag_translator.rb b/app/services/discourse_ai/tag_translator.rb new file mode 100644 index 0000000..a7bd7de --- /dev/null +++ b/app/services/discourse_ai/tag_translator.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module DiscourseAi + class TagTranslator < BaseTranslator + PROMPT_TEMPLATE = <<~TEXT.freeze + You are a translation service specializing in translating forum tags to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: + + 1. Translate the tags to target_language asked + 2. Keep proper nouns and technical terms in their original language + 3. Keep the translated tags short, close to the original length + 4. Ensure the translation maintains the original meaning + 4. Translated tags will be in lowercase + + Provide your translation in the following JSON format: + + + {"translation": "your target_language translation here"} + + + Here are three examples of correct translation + + Original: {"name":"solved", "target_language":"Chinese"} + Correct translation: {"translation": "已解决"} + + Original: {"name":"General", "target_language":"French"} + Correct translation: {"translation": "général"} + + Original: {"name": "Q&A", "target_language": "Portuguese"} + Correct translation: {"translation": "perguntas e respostas"} + + Remember to keep proper nouns like "minecraft" and "toyota" in their original form. Translate the tag now and provide your answer in the specified JSON format. + TEXT + + private def prompt_template + PROMPT_TEMPLATE + end + end +end diff --git a/app/services/discourse_translator/base.rb b/app/services/discourse_translator/base.rb index 2b5e7c4..bb8ef1d 100644 --- a/app/services/discourse_translator/base.rb +++ b/app/services/discourse_translator/base.rb @@ -139,6 +139,8 @@ def self.get_untranslated(translatable, raw: false) translatable.title when "Category" translatable.name + when "Tag" + translatable.name end end end diff --git a/app/services/discourse_translator/discourse_ai.rb b/app/services/discourse_translator/discourse_ai.rb index 594fc10..6e6489a 100644 --- a/app/services/discourse_translator/discourse_ai.rb +++ b/app/services/discourse_translator/discourse_ai.rb @@ -47,6 +47,8 @@ def self.translate!(translatable, target_locale_sym = I18n.locale) text_for_translation(translatable), language, ).translate + when "Tag" + ::DiscourseAi::TagTranslator.new(text_for_translation(translatable), language).translate end DiscourseTranslator::TranslatedContentNormalizer.normalize(translatable, translated) diff --git a/db/migrate/20250401022618_create_tag_translation_table.rb b/db/migrate/20250401022618_create_tag_translation_table.rb new file mode 100644 index 0000000..e1eb96b --- /dev/null +++ b/db/migrate/20250401022618_create_tag_translation_table.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class CreateTagTranslationTable < ActiveRecord::Migration[7.2] + def change + create_table :discourse_translator_tag_locales do |t| + t.integer :tag_id, null: false + t.string :detected_locale, limit: 20, null: false + t.timestamps + end + + create_table :discourse_translator_tag_translations do |t| + t.integer :tag_id, null: false + t.string :locale, null: false + t.text :translation, null: false + t.timestamps + end + + add_index :discourse_translator_tag_translations, + %i[tag_id locale], + unique: true, + name: "idx_tag_translations_on_tag_id_and_locale" + end +end diff --git a/lib/discourse_translator/extensions/tag_extension.rb b/lib/discourse_translator/extensions/tag_extension.rb new file mode 100644 index 0000000..35c9fc8 --- /dev/null +++ b/lib/discourse_translator/extensions/tag_extension.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module DiscourseTranslator + module Extensions + module TagExtension + extend ActiveSupport::Concern + prepended { before_update :clear_translations, if: :name_changed? } + include Translatable + end + end +end diff --git a/lib/discourse_translator/inline_translation.rb b/lib/discourse_translator/inline_translation.rb index 3b22b88..6cb5d27 100644 --- a/lib/discourse_translator/inline_translation.rb +++ b/lib/discourse_translator/inline_translation.rb @@ -88,6 +88,24 @@ def inject(plugin) serializer.object.translation_for(InlineTranslation.effective_locale).presence end end + + # tags + + plugin.register_modifier(:topic_tags_serializer_name) do |tags, serializer| + # %w[topics tags serializer name] + end + + plugin.register_modifier(:sidebar_tag_serializer_name) do |name, serializer| + if !SiteSetting.experimental_inline_translation || + serializer.object.locale_matches?(InlineTranslation.effective_locale) || + serializer.scope&.request&.params&.[]("show") == "original" + name + else + serializer.object.translation_for(InlineTranslation.effective_locale).presence + end + end + + # plugin.register_modifier(:tag_serializer_name) { |name, serializer| "tag_serializer_name" } end end end diff --git a/lib/discourse_translator/translated_content_normalizer.rb b/lib/discourse_translator/translated_content_normalizer.rb index 0dbf519..65879f3 100644 --- a/lib/discourse_translator/translated_content_normalizer.rb +++ b/lib/discourse_translator/translated_content_normalizer.rb @@ -10,6 +10,8 @@ def self.normalize(translatable, content) PrettyText.cleanup(content, {}) when "Category" content + when "Tag" + content end end end diff --git a/plugin.rb b/plugin.rb index 1f1475f..2ac568b 100644 --- a/plugin.rb +++ b/plugin.rb @@ -30,6 +30,7 @@ module ::DiscourseTranslator Post.prepend(DiscourseTranslator::Extensions::PostExtension) Topic.prepend(DiscourseTranslator::Extensions::TopicExtension) Category.prepend(DiscourseTranslator::Extensions::CategoryExtension) + Tag.prepend(DiscourseTranslator::Extensions::TagExtension) TopicViewSerializer.prepend(DiscourseTranslator::Extensions::TopicViewSerializerExtension) end From 0e8c3e24f49256dfed48a24e3c353e07223c7450 Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 8 Apr 2025 10:29:00 +0800 Subject: [PATCH 3/6] Hide translations first due to cache issue --- .../inline_translation.rb | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/lib/discourse_translator/inline_translation.rb b/lib/discourse_translator/inline_translation.rb index 6cb5d27..ad4bb14 100644 --- a/lib/discourse_translator/inline_translation.rb +++ b/lib/discourse_translator/inline_translation.rb @@ -76,36 +76,6 @@ def inject(plugin) plugin.register_topic_preloader_associations(:translations) do SiteSetting.translator_enabled && SiteSetting.experimental_inline_translation end - - # categories - - plugin.register_modifier(:site_category_serializer_name) do |name, serializer| - if !SiteSetting.experimental_inline_translation || - serializer.object.locale_matches?(InlineTranslation.effective_locale) || - serializer.scope&.request&.params&.[]("show") == "original" - name - else - serializer.object.translation_for(InlineTranslation.effective_locale).presence - end - end - - # tags - - plugin.register_modifier(:topic_tags_serializer_name) do |tags, serializer| - # %w[topics tags serializer name] - end - - plugin.register_modifier(:sidebar_tag_serializer_name) do |name, serializer| - if !SiteSetting.experimental_inline_translation || - serializer.object.locale_matches?(InlineTranslation.effective_locale) || - serializer.scope&.request&.params&.[]("show") == "original" - name - else - serializer.object.translation_for(InlineTranslation.effective_locale).presence - end - end - - # plugin.register_modifier(:tag_serializer_name) { |name, serializer| "tag_serializer_name" } end end end From 84b4ea60a8983efa5e6162aca93f2d0e0e30809e Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 8 Apr 2025 10:32:43 +0800 Subject: [PATCH 4/6] Annotate --- .../discourse_translator/category_locale.rb | 11 +++++++++++ .../discourse_translator/category_translation.rb | 16 ++++++++++++++++ app/models/discourse_translator/tag_locale.rb | 11 +++++++++++ .../discourse_translator/tag_translation.rb | 16 ++++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/app/models/discourse_translator/category_locale.rb b/app/models/discourse_translator/category_locale.rb index e1dbde3..9344291 100644 --- a/app/models/discourse_translator/category_locale.rb +++ b/app/models/discourse_translator/category_locale.rb @@ -10,3 +10,14 @@ class CategoryLocale < ActiveRecord::Base validates :detected_locale, presence: true end end + +# == Schema Information +# +# Table name: discourse_translator_category_locales +# +# id :bigint not null, primary key +# category_id :integer not null +# detected_locale :string(20) not null +# created_at :datetime not null +# updated_at :datetime not null +# diff --git a/app/models/discourse_translator/category_translation.rb b/app/models/discourse_translator/category_translation.rb index a769ff3..ec70a57 100644 --- a/app/models/discourse_translator/category_translation.rb +++ b/app/models/discourse_translator/category_translation.rb @@ -12,3 +12,19 @@ class CategoryTranslation < ActiveRecord::Base validates :locale, uniqueness: { scope: :category_id } end end + +# == Schema Information +# +# Table name: discourse_translator_category_translations +# +# id :bigint not null, primary key +# category_id :integer not null +# locale :string not null +# translation :text not null +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# idx_category_translations_on_category_id_and_locale (category_id,locale) UNIQUE +# diff --git a/app/models/discourse_translator/tag_locale.rb b/app/models/discourse_translator/tag_locale.rb index 716336b..d0719a7 100644 --- a/app/models/discourse_translator/tag_locale.rb +++ b/app/models/discourse_translator/tag_locale.rb @@ -10,3 +10,14 @@ class TagLocale < ActiveRecord::Base validates :detected_locale, presence: true end end + +# == Schema Information +# +# Table name: discourse_translator_tag_locales +# +# id :bigint not null, primary key +# tag_id :integer not null +# detected_locale :string(20) not null +# created_at :datetime not null +# updated_at :datetime not null +# diff --git a/app/models/discourse_translator/tag_translation.rb b/app/models/discourse_translator/tag_translation.rb index 79d3fef..02c6263 100644 --- a/app/models/discourse_translator/tag_translation.rb +++ b/app/models/discourse_translator/tag_translation.rb @@ -12,3 +12,19 @@ class TagTranslation < ActiveRecord::Base validates :locale, uniqueness: { scope: :tag_id } end end + +# == Schema Information +# +# Table name: discourse_translator_tag_translations +# +# id :bigint not null, primary key +# tag_id :integer not null +# locale :string not null +# translation :text not null +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# idx_tag_translations_on_tag_id_and_locale (tag_id,locale) UNIQUE +# From 298f83448d4d8f2f2ee36daaa1b9ab4046bc97cd Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 8 Apr 2025 10:37:05 +0800 Subject: [PATCH 5/6] Remove stray comment --- spec/jobs/automatic_translation_backfill_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/jobs/automatic_translation_backfill_spec.rb b/spec/jobs/automatic_translation_backfill_spec.rb index 06cee85..bdae4dc 100644 --- a/spec/jobs/automatic_translation_backfill_spec.rb +++ b/spec/jobs/automatic_translation_backfill_spec.rb @@ -156,7 +156,6 @@ def expect_google_translate(text) Category.all.each { |c| c.set_detected_locale("es") } expect_google_translate("hallo") - # .times(6) described_class.new.execute From b9a8d19e5c166ce295d3c5cb3fc7592eff35af62 Mon Sep 17 00:00:00 2001 From: Nat Date: Tue, 8 Apr 2025 10:53:06 +0800 Subject: [PATCH 6/6] Update tests for tag --- .../automatic_translation_backfill_spec.rb | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/spec/jobs/automatic_translation_backfill_spec.rb b/spec/jobs/automatic_translation_backfill_spec.rb index bdae4dc..9d3d01e 100644 --- a/spec/jobs/automatic_translation_backfill_spec.rb +++ b/spec/jobs/automatic_translation_backfill_spec.rb @@ -42,6 +42,7 @@ def expect_google_translate(text) end it "does not backfill if backfill limit is set to 0" do + SiteSetting.automatic_translation_backfill_rate = 100 SiteSetting.automatic_translation_target_languages = "de" SiteSetting.automatic_translation_backfill_rate = 0 expect_any_instance_of(Jobs::AutomaticTranslationBackfill).not_to receive(:process_batch) @@ -67,19 +68,22 @@ def expect_google_translate(text) end it "backfills both topics and posts" do + (Category.all.each + Tag.all.each).each do |c| + c.set_detected_locale("de") + c.set_translation("es", "hola") + end post = Fabricate(:post) topic = post.topic topic.set_detected_locale("de") post.set_detected_locale("es") - expect_google_translate("hola") - expect_google_translate("hallo") + expect_google_translate("xx") described_class.new.execute - expect(topic.translations.pluck(:locale, :translation)).to eq([%w[es hola]]) - expect(post.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) + expect(topic.translations.pluck(:locale, :translation)).to eq([%w[es xx]]) + expect(post.translations.pluck(:locale, :translation)).to eq([%w[de xx]]) end it "backfills only public content when limit_to_public_content is true" do @@ -95,20 +99,27 @@ def expect_google_translate(text) private_topic.set_detected_locale("de") private_post.set_detected_locale("es") + (Category.all.each + Tag.all.each).each do |c| + c.set_detected_locale("de") + c.set_translation("es", "hola") + end expect_google_translate("hola") - expect_google_translate("hallo") SiteSetting.automatic_translation_backfill_limit_to_public_content = true described_class.new.execute expect(topic.translations.pluck(:locale, :translation)).to eq([%w[es hola]]) - expect(post.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) + expect(post.translations.pluck(:locale, :translation)).to eq([%w[de hola]]) expect(private_topic.translations).to eq([]) expect(private_post.translations).to eq([]) end it "translate only content newer than automatic_translation_backfill_max_age_days" do + (Category.all.each + Tag.all.each).each do |c| + c.set_detected_locale("de") + c.set_translation("es", "hola") + end old_post = Fabricate(:post) old_topic = old_post.topic new_post = Fabricate(:post) @@ -146,14 +157,16 @@ def expect_google_translate(text) expect_google_check_language end - it "backfills all (1) category (1) topic (4) posts as it is within the maximum per job run" do + it "backfills all (1) topic (4) posts (1) category (1) tag as it is within the maximum per job run" do category = Fabricate(:category) + tag = Fabricate(:tag) topic = Fabricate(:topic, category: category) posts = Fabricate.times(4, :post, topic: topic) topic.set_detected_locale("es") posts.each { |p| p.set_detected_locale("es") } Category.all.each { |c| c.set_detected_locale("es") } + tag.set_detected_locale("es") expect_google_translate("hallo") @@ -164,6 +177,7 @@ def expect_google_translate(text) [%w[de hallo]] * 4, ) expect(category.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) + expect(tag.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) end end end