diff options
| author |   Alex Legler <alex@a3li.li> | 2015-02-23 21:56:21 +0100 |
|---|---|---|
| committer | Alex Legler <alex@a3li.li> | 2015-02-23 21:56:21 +0100 |
| commit | bbc8b2a61544af2859276cf8badb53ca3b1244c5 (patch) | |
| tree | 170928e4febff76c8d19a64e93800d9d07168a09 | |
| parent | Don't store the flags in raw_filename. (diff) | |
| download | backend-bbc8b2a61544af2859276cf8badb53ca3b1244c5.tar.gz backend-bbc8b2a61544af2859276cf8badb53ca3b1244c5.tar.bz2 backend-bbc8b2a61544af2859276cf8badb53ca3b1244c5.zip | |
Test (hopefully) faster encoding fix method
| -rw-r--r-- | lib/utils.rb | 9 |
1 files changed, 8 insertions, 1 deletions
diff --git a/lib/utils.rb b/lib/utils.rb index aaa04c7..fc4427e 100644 --- a/lib/utils.rb +++ b/lib/utils.rb @@ -2,8 +2,15 @@ require 'charlock_holmes' module Ag module Utils + module_function - def fix_encoding(str, fail_hard = false) + def fix_encoding(str) + s = str.encode('UTF-8', 'UTF-8', invalid: :replace, replace: '') + s = s.unpack('C*').pack('U*') unless s.valid_encoding? + s + end + + def fix_encoding_old(str, fail_hard = false) detection = CharlockHolmes::EncodingDetector.detect(str) CharlockHolmes::Converter.convert(str, detection[:encoding], 'UTF-8') rescue => e |