automatic converting from non-utf to utf encoding for Assets

Author: GarPit

app/models/kms/asset.rb

@@ -7,7 +7,6 @@ class Asset < ActiveRecord::Base
     # inspired by Locomotive
     before_validation :store_text
 
-    validate :encoded_as_utf8, on: :create
     validate :unique_filename, on: :create
 
     attr_accessor :text, :performing_plain_text
@@ -57,7 +56,10 @@ def store_text
 
       return if !text_or_javascript? || data.blank?
 
-      sanitized_source = replace_urls(data)
+      cd = CharDet.detect(data)
+      encoded_source = data.force_encoding(cd['encoding']).encode('UTF-8') rescue data
+
+      sanitized_source = replace_urls(encoded_source)
 
       self.file = ::CarrierWave::SanitizedFile.new({
         tempfile: StringIO.new(sanitized_source),
@@ -69,12 +71,6 @@ def store_text
 
     protected
 
-    def encoded_as_utf8
-      text.encode('UTF-8')
-    rescue Encoding::UndefinedConversionError
-      self.errors.add(:file, I18n.t(:wrong_encoding, scope: [:activerecord, :errors, :messages]))
-    end
-
     def unique_filename
       if Asset.where(file: file.filename).exists?
         self.errors.add(:file, I18n.t(:duplicate, scope: [:activerecord, :errors, :messages]))

config/locales/en.yml

@@ -174,7 +174,6 @@ en:
   activerecord:
     errors:
       messages:
-        wrong_encoding: "is not in UTF-8 encoding"
         duplicate: "is duplicate"
       models:
         kms/page:

config/locales/ru.yml

@@ -174,7 +174,6 @@ ru:
   activerecord:
     errors:
       messages:
-        wrong_encoding: "с некорректной кодировкой (необходима UTF-8)"
         duplicate: "с таким именем уже существует"
       models:
         kms/page:

kms.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'dragonfly', '~> 1.1.3'
   s.add_dependency 'pg', '~> 0.21'
   s.add_dependency 'textacular', '~> 5.0.0'
+  s.add_dependency 'rchardet', '~> 1.7.0'
 
   s.add_development_dependency 'combustion', '~> 0.5.4'
   s.add_development_dependency 'factory_girl_rails'

lib/kms/dependencies.rb

@@ -13,3 +13,4 @@
 require 'cancancan'
 require 'dragonfly'
 require 'textacular'
+require 'rchardet'

spec/internal/public/invalid_encoding.css

@@ -1,5 +1,5 @@
 /*
-  ��� ������� �����������
+  Êîäèðîâêà cp1251
 */
 body {
   font-size: 20px;

spec/models/kms/asset_spec.rb

@@ -3,9 +3,10 @@
 module Kms
   describe Kms::Asset, type: :model do
     context 'when file uploaded in not UTF-8 encoding' do
-      let(:asset) { FactoryGirl.build(:invalid_encoding_css_asset) }
-      it 'should be invalid' do
-        expect(asset).to_not be_valid
+      let(:asset) { FactoryGirl.create(:invalid_encoding_css_asset) }
+      it 'will be converted into utf-8' do
+        cd = CharDet.detect(asset.text)
+        expect(cd['encoding']).to eq 'utf-8'
       end
     end