/* * Copyright (C) 2010 ZXing authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ use crate::{DecodeHintType, DecodeHintValue, DecodingHintDictionary}; use super::CharacterSet; /** * Common string-related functions. * * @author Sean Owen * @author Alex Dupre */ pub struct StringUtils { // private static final Charset PLATFORM_DEFAULT_ENCODING = Charset.defaultCharset(); // public static final Charset SHIFT_JIS_CHARSET = Charset.forName("SJIS"); // public static final Charset GB2312_CHARSET = Charset.forName("GB2312"); // private static final Charset EUC_JP = Charset.forName("EUC_JP"); // private static final boolean ASSUME_SHIFT_JIS = // SHIFT_JIS_CHARSET.equals(PLATFORM_DEFAULT_ENCODING) || // EUC_JP.equals(PLATFORM_DEFAULT_ENCODING); // // Retained for ABI compatibility with earlier versions // public static final String SHIFT_JIS = "SJIS"; // public static final String GB2312 = "GB2312"; } // const PLATFORM_DEFAULT_ENCODING: &dyn Encoding = encoding::all::UTF_8; // const SHIFT_JIS_CHARSET: &dyn Encoding = // encoding::label::encoding_from_whatwg_label("SJIS").unwrap(); // const GB2312_CHARSET: &dyn Encoding = // encoding::label::encoding_from_whatwg_label("GB2312").unwrap(); // const EUC_JP: &dyn Encoding = encoding::label::encoding_from_whatwg_label("EUC_JP").unwrap(); const ASSUME_SHIFT_JIS: bool = false; // static SHIFT_JIS: &'static str = "SJIS"; // static GB2312: &'static str = "GB2312"; pub static SHIFT_JIS_CHARSET: CharacterSet = CharacterSet::SJIS; // private static final boolean ASSUME_SHIFT_JIS = // SHIFT_JIS_CHARSET.equals(PLATFORM_DEFAULT_ENCODING) || // EUC_JP.equals(PLATFORM_DEFAULT_ENCODING); impl StringUtils { /** * @param bytes bytes encoding a string, whose encoding should be guessed * @param hints decode hints if applicable * @return name of guessed encoding; at the moment will only guess one of: * "SJIS", "UTF8", "ISO8859_1", or the platform default encoding if none * of these can possibly be correct */ pub fn guessEncoding(bytes: &[u8], hints: &DecodingHintDictionary) -> Option<&'static str> { let c = StringUtils::guessCharset(bytes, hints)?; if c == CharacterSet::SJIS { Some("SJIS") } else if c == CharacterSet::UTF8 { Some("UTF8") } else if c == CharacterSet::ISO8859_1 { Some("ISO8859_1") } else { Some(c.get_charset_name()) } } /** * @param bytes bytes encoding a string, whose encoding should be guessed * @param hints decode hints if applicable * @return Charset of guessed encoding; at the moment will only guess one of: * {@link #SHIFT_JIS_CHARSET}, {@link StandardCharsets#UTF_8}, * {@link StandardCharsets#ISO_8859_1}, {@link StandardCharsets#UTF_16}, * or the platform default encoding if * none of these can possibly be correct */ pub fn guessCharset(bytes: &[u8], hints: &DecodingHintDictionary) -> Option { if let Some(DecodeHintValue::CharacterSet(cs_name)) = hints.get(&DecodeHintType::CHARACTER_SET) { // if let DecodeHintValue::CharacterSet(cs_name) = hint { return CharacterSet::get_character_set_by_name(cs_name); // } } // if hints.contains_key(&DecodeHintType::CHARACTER_SET) { // return Charset.forName(hints.get(DecodeHintType.CHARACTER_SET).toString()); // } // First try UTF-16, assuming anything with its BOM is UTF-16 if bytes.len() > 2 && ((bytes[0] == 0xFE && bytes[1] == 0xFF) || (bytes[0] == 0xFF && bytes[1] == 0xFE)) { if bytes[0] == 0xFE && bytes[1] == 0xFF { return Some(CharacterSet::UnicodeBigUnmarked); } else { return Some(CharacterSet::UTF16LE); } } // For now, merely tries to distinguish ISO-8859-1, UTF-8 and Shift_JIS, // which should be by far the most common encodings. let length = bytes.len(); let mut can_be_iso88591 = true; let mut can_be_shift_jis = true; let mut can_be_utf8 = true; let mut utf8_bytes_left = 0; let mut utf2_bytes_chars = 0; let mut utf3_bytes_chars = 0; let mut utf4_bytes_chars = 0; let mut sjis_bytes_left = 0; let mut sjis_katakana_chars = 0; let mut sjis_cur_katakana_word_length = 0; let mut sjis_cur_double_bytes_word_length = 0; let mut sjis_max_katakana_word_length = 0; let mut sjis_max_double_bytes_word_length = 0; let mut iso_high_other = 0; let utf8bom = bytes.len() > 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF; // for i in 0..length { for value in bytes.iter().take(length).copied() { // for (int i = 0; // i < length && (canBeISO88591 || canBeShiftJIS || canBeUTF8); // i++) { if !(can_be_iso88591 || can_be_shift_jis || can_be_utf8) { break; } // let value = bytes[i]; // UTF-8 stuff if can_be_utf8 { if utf8_bytes_left > 0 { if (value & 0x80) == 0 { can_be_utf8 = false; } else { utf8_bytes_left -= 1; } } else if (value & 0x80) != 0 { if (value & 0x40) == 0 { can_be_utf8 = false; } else { utf8_bytes_left += 1; if (value & 0x20) == 0 { utf2_bytes_chars += 1; } else { utf8_bytes_left += 1; if (value & 0x10) == 0 { utf3_bytes_chars += 1; } else { utf8_bytes_left += 1; if (value & 0x08) == 0 { utf4_bytes_chars += 1; } else { can_be_utf8 = false; } } } } } } // ISO-8859-1 stuff if can_be_iso88591 { if value > 0x7F && value < 0xA0 { can_be_iso88591 = false; } else if value > 0x9F && (value < 0xC0 || value == 0xD7 || value == 0xF7) { iso_high_other += 1; } } // Shift_JIS stuff if can_be_shift_jis { if sjis_bytes_left > 0 { if value < 0x40 || value == 0x7F || value > 0xFC { can_be_shift_jis = false; } else { sjis_bytes_left -= 1; } } else if value == 0x80 || value == 0xA0 || value > 0xEF { can_be_shift_jis = false; } else if value > 0xA0 && value < 0xE0 { sjis_katakana_chars += 1; sjis_cur_double_bytes_word_length = 0; sjis_cur_katakana_word_length += 1; if sjis_cur_katakana_word_length > sjis_max_katakana_word_length { sjis_max_katakana_word_length = sjis_cur_katakana_word_length; } } else if value > 0x7F { sjis_bytes_left += 1; //sjisDoubleBytesChars++; sjis_cur_katakana_word_length = 0; sjis_cur_double_bytes_word_length += 1; if sjis_cur_double_bytes_word_length > sjis_max_double_bytes_word_length { sjis_max_double_bytes_word_length = sjis_cur_double_bytes_word_length; } } else { //sjisLowChars++; sjis_cur_katakana_word_length = 0; sjis_cur_double_bytes_word_length = 0; } } } if can_be_utf8 && utf8_bytes_left > 0 { can_be_utf8 = false; } if can_be_shift_jis && sjis_bytes_left > 0 { can_be_shift_jis = false; } // Easy -- if there is BOM or at least 1 valid not-single byte character (and no evidence it can't be UTF-8), done if can_be_utf8 && (utf8bom || utf2_bytes_chars + utf3_bytes_chars + utf4_bytes_chars > 0) { return Some(CharacterSet::UTF8); } // Easy -- if assuming Shift_JIS or >= 3 valid consecutive not-ascii characters (and no evidence it can't be), done if can_be_shift_jis && (ASSUME_SHIFT_JIS || sjis_max_katakana_word_length >= 3 || sjis_max_double_bytes_word_length >= 3) { return Some(CharacterSet::SJIS); //encoding::label::encoding_from_whatwg_label("SJIS"); } // Distinguishing Shift_JIS and ISO-8859-1 can be a little tough for short words. The crude heuristic is: // - If we saw // - only two consecutive katakana chars in the whole text, or // - at least 10% of bytes that could be "upper" not-alphanumeric Latin1, // - then we conclude Shift_JIS, else ISO-8859-1 if can_be_iso88591 && can_be_shift_jis { return if (sjis_max_katakana_word_length == 2 && sjis_katakana_chars == 2) || iso_high_other * 10 >= length { Some(CharacterSet::SJIS) } else { Some(CharacterSet::ISO8859_1) }; } // Otherwise, try in order ISO-8859-1, Shift JIS, UTF-8 and fall back to default platform encoding if can_be_iso88591 { return Some(CharacterSet::ISO8859_1); } if can_be_shift_jis { return Some(CharacterSet::SJIS); } if can_be_utf8 { return Some(CharacterSet::UTF8); } // Otherwise, we take a wild guess with platform encoding Some(CharacterSet::UTF8) } }