diff --git a/app/jobs/scheduled/automatic_translation_backfill.rb b/app/jobs/scheduled/automatic_translation_backfill.rb index 0366a50..bb5f35c 100644 --- a/app/jobs/scheduled/automatic_translation_backfill.rb +++ b/app/jobs/scheduled/automatic_translation_backfill.rb @@ -24,10 +24,10 @@ def fetch_untranslated_model_ids(model, content_column, limit, target_locale) SELECT m.id FROM #{model.table_name} m #{limit_to_public_clause(model)} - WHERE m.deleted_at IS NULL - AND m.#{content_column} != '' - AND m.user_id > 0 - #{max_age_clause} + WHERE m.#{content_column} != '' + #{not_deleted_clause(model)} + #{non_bot_clause(model)} + #{max_age_clause(model)} ORDER BY m.updated_at DESC ) EXCEPT @@ -87,26 +87,35 @@ def translate_records(type, record_ids, target_locale) def process_batch records_to_translate = SiteSetting.automatic_translation_backfill_rate - backfill_locales.each_with_index do |target_locale, i| - topic_ids = - fetch_untranslated_model_ids(Topic, "title", records_to_translate, target_locale) - post_ids = fetch_untranslated_model_ids(Post, "raw", records_to_translate, target_locale) - - next if topic_ids.empty? && post_ids.empty? - - DiscourseTranslator::VerboseLogger.log( - "Translating #{topic_ids.size} topics and #{post_ids.size} posts to #{target_locale}", - ) + backfill_locales.each do |target_locale| + [ + [Topic, "title"], + [Post, "raw"], + [Category, "name"], + [Tag, "name"], + ].each do |model, content_column| + ids = + fetch_untranslated_model_ids(model, content_column, records_to_translate, target_locale) + + next if ids.empty? + + DiscourseTranslator::VerboseLogger.log( + "Translating #{ids.size} #{model.name} to #{target_locale}", + ) - translate_records(Topic, topic_ids, target_locale) - translate_records(Post, post_ids, target_locale) + translate_records(model, ids, target_locale) + end end end - def max_age_clause + def max_age_clause(model) return "" if SiteSetting.automatic_translation_backfill_max_age_days <= 0 - "AND m.created_at > NOW() - INTERVAL '#{SiteSetting.automatic_translation_backfill_max_age_days} days'" + if model == Post || model == Topic + "AND m.created_at > NOW() - INTERVAL '#{SiteSetting.automatic_translation_backfill_max_age_days} days'" + else + "" + end end def limit_to_public_clause(model) @@ -130,5 +139,15 @@ def limit_to_public_clause(model) limit_to_public_clause end + + def non_bot_clause(model) + return "AND m.user_id > 0" if model == Post || model == Topic + "" + end + + def not_deleted_clause(model) + return "AND m.deleted_at IS NULL" if model == Post || model == Topic + "" + end end end diff --git a/app/models/discourse_translator/category_locale.rb b/app/models/discourse_translator/category_locale.rb new file mode 100644 index 0000000..9344291 --- /dev/null +++ b/app/models/discourse_translator/category_locale.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class CategoryLocale < ActiveRecord::Base + self.table_name = "discourse_translator_category_locales" + + belongs_to :category + + validates :category_id, presence: true + validates :detected_locale, presence: true + end +end + +# == Schema Information +# +# Table name: discourse_translator_category_locales +# +# id :bigint not null, primary key +# category_id :integer not null +# detected_locale :string(20) not null +# created_at :datetime not null +# updated_at :datetime not null +# diff --git a/app/models/discourse_translator/category_translation.rb b/app/models/discourse_translator/category_translation.rb new file mode 100644 index 0000000..ec70a57 --- /dev/null +++ b/app/models/discourse_translator/category_translation.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class CategoryTranslation < ActiveRecord::Base + self.table_name = "discourse_translator_category_translations" + + belongs_to :category + + validates :category_id, presence: true + validates :locale, presence: true + validates :translation, presence: true + validates :locale, uniqueness: { scope: :category_id } + end +end + +# == Schema Information +# +# Table name: discourse_translator_category_translations +# +# id :bigint not null, primary key +# category_id :integer not null +# locale :string not null +# translation :text not null +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# idx_category_translations_on_category_id_and_locale (category_id,locale) UNIQUE +# diff --git a/app/models/discourse_translator/tag_locale.rb b/app/models/discourse_translator/tag_locale.rb new file mode 100644 index 0000000..d0719a7 --- /dev/null +++ b/app/models/discourse_translator/tag_locale.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class TagLocale < ActiveRecord::Base + self.table_name = "discourse_translator_tag_locales" + + belongs_to :tag + + validates :tag_id, presence: true + validates :detected_locale, presence: true + end +end + +# == Schema Information +# +# Table name: discourse_translator_tag_locales +# +# id :bigint not null, primary key +# tag_id :integer not null +# detected_locale :string(20) not null +# created_at :datetime not null +# updated_at :datetime not null +# diff --git a/app/models/discourse_translator/tag_translation.rb b/app/models/discourse_translator/tag_translation.rb new file mode 100644 index 0000000..02c6263 --- /dev/null +++ b/app/models/discourse_translator/tag_translation.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class TagTranslation < ActiveRecord::Base + self.table_name = "discourse_translator_tag_translations" + + belongs_to :tag + + validates :tag_id, presence: true + validates :locale, presence: true + validates :translation, presence: true + validates :locale, uniqueness: { scope: :tag_id } + end +end + +# == Schema Information +# +# Table name: discourse_translator_tag_translations +# +# id :bigint not null, primary key +# tag_id :integer not null +# locale :string not null +# translation :text not null +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# idx_tag_translations_on_tag_id_and_locale (tag_id,locale) UNIQUE +# diff --git a/app/services/discourse_ai/category_translator.rb b/app/services/discourse_ai/category_translator.rb new file mode 100644 index 0000000..789779a --- /dev/null +++ b/app/services/discourse_ai/category_translator.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module DiscourseAi + class CategoryTranslator < BaseTranslator + PROMPT_TEMPLATE = <<~TEXT.freeze + You are a translation service specializing in translating forum category names to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: + + 1. Translate the category name to target_language asked + 2. Keep proper nouns and technical terms in their original language + 3. Keep the translated category name length short, and close to the original length + 4. Ensure the translation maintains the original meaning + + Provide your translation in the following JSON format: + + + {"translation": "Your target_language translation here"} + + + Here are three examples of correct translation + + Original: {"name":"Cats and Dogs", "target_language":"Chinese"} + Correct translation: {"translation": "猫和狗"} + + Original: {"name":"General", "target_language":"French"} + Correct translation: {"translation": "Général"} + + Original: {"name": "Q&A", "target_language": "Portuguese"} + Correct translation: {"translation": "Perguntas e Respostas"} + + Remember to keep proper nouns like "Minecraft" and "Toyota" in their original form. Translate the category name now and provide your answer in the specified JSON format. + TEXT + + private def prompt_template + PROMPT_TEMPLATE + end + end +end diff --git a/app/services/discourse_ai/tag_translator.rb b/app/services/discourse_ai/tag_translator.rb new file mode 100644 index 0000000..a7bd7de --- /dev/null +++ b/app/services/discourse_ai/tag_translator.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module DiscourseAi + class TagTranslator < BaseTranslator + PROMPT_TEMPLATE = <<~TEXT.freeze + You are a translation service specializing in translating forum tags to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: + + 1. Translate the tags to target_language asked + 2. Keep proper nouns and technical terms in their original language + 3. Keep the translated tags short, close to the original length + 4. Ensure the translation maintains the original meaning + 4. Translated tags will be in lowercase + + Provide your translation in the following JSON format: + + + {"translation": "your target_language translation here"} + + + Here are three examples of correct translation + + Original: {"name":"solved", "target_language":"Chinese"} + Correct translation: {"translation": "已解决"} + + Original: {"name":"General", "target_language":"French"} + Correct translation: {"translation": "général"} + + Original: {"name": "Q&A", "target_language": "Portuguese"} + Correct translation: {"translation": "perguntas e respostas"} + + Remember to keep proper nouns like "minecraft" and "toyota" in their original form. Translate the tag now and provide your answer in the specified JSON format. + TEXT + + private def prompt_template + PROMPT_TEMPLATE + end + end +end diff --git a/app/services/discourse_ai/topic_translator.rb b/app/services/discourse_ai/topic_translator.rb index 99ca595..7343135 100644 --- a/app/services/discourse_ai/topic_translator.rb +++ b/app/services/discourse_ai/topic_translator.rb @@ -3,9 +3,9 @@ module DiscourseAi class TopicTranslator < BaseTranslator PROMPT_TEMPLATE = <<~TEXT.freeze - You are a translation service specializing in translating forum post titles from English to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: + You are a translation service specializing in translating forum post titles to the asked target_language. Your task is to provide accurate and contextually appropriate translations while adhering to the following guidelines: - 1. Translate the given title from English to target_language asked. + 1. Translate the given title to target_language asked. 2. Keep proper nouns and technical terms in their original language. 3. Attempt to keep the translated title length close to the original when possible. 4. Ensure the translation maintains the original meaning and tone. diff --git a/app/services/discourse_translator/base.rb b/app/services/discourse_translator/base.rb index f43c7f1..bb8ef1d 100644 --- a/app/services/discourse_translator/base.rb +++ b/app/services/discourse_translator/base.rb @@ -137,6 +137,10 @@ def self.get_untranslated(translatable, raw: false) raw ? translatable.raw : translatable.cooked when "Topic" translatable.title + when "Category" + translatable.name + when "Tag" + translatable.name end end end diff --git a/app/services/discourse_translator/discourse_ai.rb b/app/services/discourse_translator/discourse_ai.rb index 947a3cb..6e6489a 100644 --- a/app/services/discourse_translator/discourse_ai.rb +++ b/app/services/discourse_translator/discourse_ai.rb @@ -42,6 +42,13 @@ def self.translate!(translatable, target_locale_sym = I18n.locale) .join("") when "Topic" ::DiscourseAi::TopicTranslator.new(text_for_translation(translatable), language).translate + when "Category" + ::DiscourseAi::CategoryTranslator.new( + text_for_translation(translatable), + language, + ).translate + when "Tag" + ::DiscourseAi::TagTranslator.new(text_for_translation(translatable), language).translate end DiscourseTranslator::TranslatedContentNormalizer.normalize(translatable, translated) diff --git a/db/migrate/20250401015139_create_category_translation_table.rb b/db/migrate/20250401015139_create_category_translation_table.rb new file mode 100644 index 0000000..bdc5ad5 --- /dev/null +++ b/db/migrate/20250401015139_create_category_translation_table.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class CreateCategoryTranslationTable < ActiveRecord::Migration[7.2] + def change + create_table :discourse_translator_category_locales do |t| + t.integer :category_id, null: false + t.string :detected_locale, limit: 20, null: false + t.timestamps + end + + create_table :discourse_translator_category_translations do |t| + t.integer :category_id, null: false + t.string :locale, null: false + t.text :translation, null: false + t.timestamps + end + + add_index :discourse_translator_category_translations, + %i[category_id locale], + unique: true, + name: "idx_category_translations_on_category_id_and_locale" + end +end diff --git a/db/migrate/20250401022618_create_tag_translation_table.rb b/db/migrate/20250401022618_create_tag_translation_table.rb new file mode 100644 index 0000000..e1eb96b --- /dev/null +++ b/db/migrate/20250401022618_create_tag_translation_table.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class CreateTagTranslationTable < ActiveRecord::Migration[7.2] + def change + create_table :discourse_translator_tag_locales do |t| + t.integer :tag_id, null: false + t.string :detected_locale, limit: 20, null: false + t.timestamps + end + + create_table :discourse_translator_tag_translations do |t| + t.integer :tag_id, null: false + t.string :locale, null: false + t.text :translation, null: false + t.timestamps + end + + add_index :discourse_translator_tag_translations, + %i[tag_id locale], + unique: true, + name: "idx_tag_translations_on_tag_id_and_locale" + end +end diff --git a/lib/discourse_translator/extensions/category_extension.rb b/lib/discourse_translator/extensions/category_extension.rb new file mode 100644 index 0000000..e8c4f49 --- /dev/null +++ b/lib/discourse_translator/extensions/category_extension.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module DiscourseTranslator + module Extensions + module CategoryExtension + extend ActiveSupport::Concern + prepended { before_update :clear_translations, if: :name_changed? } + include Translatable + end + end +end diff --git a/lib/discourse_translator/extensions/tag_extension.rb b/lib/discourse_translator/extensions/tag_extension.rb new file mode 100644 index 0000000..35c9fc8 --- /dev/null +++ b/lib/discourse_translator/extensions/tag_extension.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module DiscourseTranslator + module Extensions + module TagExtension + extend ActiveSupport::Concern + prepended { before_update :clear_translations, if: :name_changed? } + include Translatable + end + end +end diff --git a/lib/discourse_translator/inline_translation.rb b/lib/discourse_translator/inline_translation.rb index f7544d0..ad4bb14 100644 --- a/lib/discourse_translator/inline_translation.rb +++ b/lib/discourse_translator/inline_translation.rb @@ -15,6 +15,8 @@ def inject(plugin) # always return early if topic and posts are in the user's effective_locale. # this prevents the need to load translations. + # posts + plugin.register_modifier(:basic_post_serializer_cooked) do |cooked, serializer| if !SiteSetting.experimental_inline_translation || serializer.object.locale_matches?(InlineTranslation.effective_locale) || @@ -25,6 +27,14 @@ def inject(plugin) end end + plugin.add_to_serializer(:basic_post, :is_translated) do + SiteSetting.experimental_inline_translation && + !object.locale_matches?(InlineTranslation.effective_locale) && + object.translation_for(InlineTranslation.effective_locale).present? + end + + # topics + plugin.register_modifier(:topic_serializer_fancy_title) do |fancy_title, serializer| if !SiteSetting.experimental_inline_translation || serializer.object.locale_matches?(InlineTranslation.effective_locale) || @@ -54,12 +64,6 @@ def inject(plugin) end end - plugin.add_to_serializer(:basic_post, :is_translated) do - SiteSetting.experimental_inline_translation && - !object.locale_matches?(InlineTranslation.effective_locale) && - object.translation_for(InlineTranslation.effective_locale).present? - end - plugin.add_to_serializer(:topic_view, :is_translated) do SiteSetting.experimental_inline_translation && !object.topic.locale_matches?(InlineTranslation.effective_locale) && diff --git a/lib/discourse_translator/translated_content_normalizer.rb b/lib/discourse_translator/translated_content_normalizer.rb index 24884f5..65879f3 100644 --- a/lib/discourse_translator/translated_content_normalizer.rb +++ b/lib/discourse_translator/translated_content_normalizer.rb @@ -8,6 +8,10 @@ def self.normalize(translatable, content) PrettyText.cook(content) when "Topic" PrettyText.cleanup(content, {}) + when "Category" + content + when "Tag" + content end end end diff --git a/plugin.rb b/plugin.rb index 12157bf..2ac568b 100644 --- a/plugin.rb +++ b/plugin.rb @@ -29,6 +29,8 @@ module ::DiscourseTranslator Guardian.prepend(DiscourseTranslator::Extensions::GuardianExtension) Post.prepend(DiscourseTranslator::Extensions::PostExtension) Topic.prepend(DiscourseTranslator::Extensions::TopicExtension) + Category.prepend(DiscourseTranslator::Extensions::CategoryExtension) + Tag.prepend(DiscourseTranslator::Extensions::TagExtension) TopicViewSerializer.prepend(DiscourseTranslator::Extensions::TopicViewSerializerExtension) end diff --git a/spec/jobs/automatic_translation_backfill_spec.rb b/spec/jobs/automatic_translation_backfill_spec.rb index 56dc036..9d3d01e 100644 --- a/spec/jobs/automatic_translation_backfill_spec.rb +++ b/spec/jobs/automatic_translation_backfill_spec.rb @@ -8,41 +8,24 @@ end def expect_google_check_language - Excon - .expects(:post) - .with(DiscourseTranslator::Google::SUPPORT_URI, anything, anything) - .returns( - Struct.new(:status, :body).new( - 200, - %{ { "data": { "languages": [ { "language": "es" }, { "language": "de" }] } } }, - ), - ) - .at_least_once + stub_request(:post, DiscourseTranslator::Google::SUPPORT_URI).to_return( + status: 200, + body: %{ { "data": { "languages": [ { "language": "es" }, { "language": "de" }] } } }, + ) end def expect_google_detect(locale) - Excon - .expects(:post) - .with(DiscourseTranslator::Google::DETECT_URI, anything, anything) - .returns( - Struct.new(:status, :body).new( - 200, - %{ { "data": { "detections": [ [ { "language": "#{locale}" } ] ] } } }, - ), - ) - .once + stub_request(:post, DiscourseTranslator::Google::DETECT_URI).to_return( + status: 200, + body: %{ { "data": { "detections": [ [ { "language": "#{locale}" } ] ] } } }, + ) end def expect_google_translate(text) - Excon - .expects(:post) - .with(DiscourseTranslator::Google::TRANSLATE_URI, body: anything, headers: anything) - .returns( - Struct.new(:status, :body).new( - 200, - %{ { "data": { "translations": [ { "translatedText": "#{text}" } ] } } }, - ), - ) + stub_request(:post, DiscourseTranslator::Google::TRANSLATE_URI).to_return( + status: 200, + body: %{ { "data": { "translations": [ { "translatedText": "#{text}" } ] } } }, + ) end describe "backfilling" do @@ -59,7 +42,7 @@ def expect_google_translate(text) end it "does not backfill if backfill limit is set to 0" do - SiteSetting.automatic_translation_backfill_rate = 1 + SiteSetting.automatic_translation_backfill_rate = 100 SiteSetting.automatic_translation_target_languages = "de" SiteSetting.automatic_translation_backfill_rate = 0 expect_any_instance_of(Jobs::AutomaticTranslationBackfill).not_to receive(:process_batch) @@ -85,19 +68,22 @@ def expect_google_translate(text) end it "backfills both topics and posts" do + (Category.all.each + Tag.all.each).each do |c| + c.set_detected_locale("de") + c.set_translation("es", "hola") + end post = Fabricate(:post) topic = post.topic topic.set_detected_locale("de") post.set_detected_locale("es") - expect_google_translate("hola") - expect_google_translate("hallo") + expect_google_translate("xx") described_class.new.execute - expect(topic.translations.pluck(:locale, :translation)).to eq([%w[es hola]]) - expect(post.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) + expect(topic.translations.pluck(:locale, :translation)).to eq([%w[es xx]]) + expect(post.translations.pluck(:locale, :translation)).to eq([%w[de xx]]) end it "backfills only public content when limit_to_public_content is true" do @@ -113,20 +99,27 @@ def expect_google_translate(text) private_topic.set_detected_locale("de") private_post.set_detected_locale("es") + (Category.all.each + Tag.all.each).each do |c| + c.set_detected_locale("de") + c.set_translation("es", "hola") + end expect_google_translate("hola") - expect_google_translate("hallo") SiteSetting.automatic_translation_backfill_limit_to_public_content = true described_class.new.execute expect(topic.translations.pluck(:locale, :translation)).to eq([%w[es hola]]) - expect(post.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) + expect(post.translations.pluck(:locale, :translation)).to eq([%w[de hola]]) expect(private_topic.translations).to eq([]) expect(private_post.translations).to eq([]) end it "translate only content newer than automatic_translation_backfill_max_age_days" do + (Category.all.each + Tag.all.each).each do |c| + c.set_detected_locale("de") + c.set_translation("es", "hola") + end old_post = Fabricate(:post) old_topic = old_post.topic new_post = Fabricate(:post) @@ -164,14 +157,18 @@ def expect_google_translate(text) expect_google_check_language end - it "backfills all (1) topics and (4) posts as it is within the maximum per job run" do - topic = Fabricate(:topic) + it "backfills all (1) topic (4) posts (1) category (1) tag as it is within the maximum per job run" do + category = Fabricate(:category) + tag = Fabricate(:tag) + topic = Fabricate(:topic, category: category) posts = Fabricate.times(4, :post, topic: topic) topic.set_detected_locale("es") posts.each { |p| p.set_detected_locale("es") } + Category.all.each { |c| c.set_detected_locale("es") } + tag.set_detected_locale("es") - expect_google_translate("hallo").times(5) + expect_google_translate("hallo") described_class.new.execute @@ -179,6 +176,8 @@ def expect_google_translate(text) expect(posts.map { |p| p.translations.pluck(:locale, :translation).flatten }).to eq( [%w[de hallo]] * 4, ) + expect(category.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) + expect(tag.translations.pluck(:locale, :translation)).to eq([%w[de hallo]]) end end end