repurpose CharacterSetECI as encoding abstraction

This commit is contained in:
Henry Schimke
2023-03-02 15:50:56 -06:00
parent c4fec7d2ee
commit a0b8b68869
29 changed files with 304 additions and 337 deletions

View File

@@ -23,12 +23,13 @@
// import java.nio.charset.StandardCharsets;
// import java.util.Random;
use encoding::{Encoding, EncodingRef};
use rand::Rng;
use std::collections::HashMap;
use crate::common::StringUtils;
use super::CharacterSetECI;
#[test]
fn test_random() {
let mut r = rand::thread_rng();
@@ -38,10 +39,9 @@ fn test_random() {
// *byte = r.gen();
// }
assert_eq!(
encoding::all::UTF_8.name(),
CharacterSetECI::UTF8,
StringUtils::guessCharset(&bytes, &HashMap::new())
.unwrap()
.name()
);
}
@@ -50,7 +50,7 @@ fn test_short_shift_jis1() {
// 金魚
do_test(
&[0x8b, 0xe0, 0x8b, 0x9b],
encoding::label::encoding_from_whatwg_label("SJIS").unwrap(),
CharacterSetECI::SJIS,
"SJIS",
);
}
@@ -58,7 +58,7 @@ fn test_short_shift_jis1() {
#[test]
fn test_short_iso885911() {
// båd
do_test(&[0x62, 0xe5, 0x64], encoding::all::ISO_8859_1, "ISO8859_1");
do_test(&[0x62, 0xe5, 0x64], CharacterSetECI::ISO8859_1, "ISO8859_1");
}
#[test]
@@ -66,7 +66,7 @@ fn test_short_utf8() {
// Español
do_test(
&[0x45, 0x73, 0x70, 0x61, 0xc3, 0xb1, 0x6f, 0x6c],
encoding::all::UTF_8,
CharacterSetECI::UTF8,
"UTF8",
);
}
@@ -76,7 +76,7 @@ fn test_mixed_shift_jis1() {
// Hello 金!
do_test(
&[0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x8b, 0xe0, 0x21],
encoding::label::encoding_from_whatwg_label("SJIS").unwrap(),
CharacterSetECI::SJIS,
"SJIS",
);
}
@@ -86,8 +86,8 @@ fn test_utf16_be() {
// 调压柜
do_test(
&[0xFE, 0xFF, 0x8c, 0x03, 0x53, 0x8b, 0x67, 0xdc],
encoding::all::UTF_16BE,
encoding::all::UTF_16BE.name(),
CharacterSetECI::UnicodeBigUnmarked,
&CharacterSetECI::UnicodeBigUnmarked.getCharsetName(),
);
}
@@ -96,15 +96,15 @@ fn test_utf16_le() {
// 调压柜
do_test(
&[0xFF, 0xFE, 0x03, 0x8c, 0x8b, 0x53, 0xdc, 0x67],
encoding::all::UTF_16LE,
encoding::all::UTF_16LE.name(),
CharacterSetECI::UTF16LE,
&CharacterSetECI::UTF16LE.getCharsetName(),
);
}
fn do_test(bytes: &[u8], charset: EncodingRef, encoding: &str) {
fn do_test(bytes: &[u8], charset: CharacterSetECI, encoding: &str) {
let guessedCharset = StringUtils::guessCharset(bytes, &HashMap::new()).unwrap();
let guessedEncoding = StringUtils::guessEncoding(bytes, &HashMap::new()).unwrap();
assert_eq!(charset.name(), guessedCharset.name());
assert_eq!(charset, guessedCharset);
assert_eq!(encoding, guessedEncoding);
}

View File

@@ -14,15 +14,6 @@
* limitations under the License.
*/
// package com.google.zxing.common;
// import com.google.zxing.FormatException;
// import java.nio.charset.Charset;
// import java.util.HashMap;
// import java.util.Map;
use encoding::EncodingRef;
use crate::common::Result;
@@ -59,6 +50,7 @@ pub enum CharacterSetECI {
Cp1252, //(23, "windows-1252"),
Cp1256, //(24, "windows-1256"),
UnicodeBigUnmarked, //(25, "UTF-16BE", "UnicodeBig"),
UTF16LE,
UTF8, //(26, "UTF-8"),
ASCII, //(new int[] {27, 170}, "US-ASCII"),
Big5, //(28),
@@ -101,8 +93,8 @@ impl CharacterSetECI {
Self::getValue(self)
}
pub fn getValue(cs_eci: &CharacterSetECI) -> u32 {
match cs_eci {
pub fn getValue(&self) -> u32 {
match self {
CharacterSetECI::Cp437 => 0,
CharacterSetECI::ISO8859_1 => 1,
CharacterSetECI::ISO8859_2 => 4,
@@ -125,6 +117,7 @@ impl CharacterSetECI {
CharacterSetECI::Cp1252 => 23,
CharacterSetECI::Cp1256 => 24,
CharacterSetECI::UnicodeBigUnmarked => 25,
CharacterSetECI::UTF16LE => 100025,
CharacterSetECI::UTF8 => 26,
CharacterSetECI::ASCII => 27,
CharacterSetECI::Big5 => 28,
@@ -136,8 +129,8 @@ impl CharacterSetECI {
pub fn getCharset(&self, ) -> EncodingRef {
let name = match self {
// CharacterSetECI::Cp437 => "CP437",
CharacterSetECI::Cp437 => "UTF-8",
CharacterSetECI::ISO8859_1 => "ISO-8859-1",
CharacterSetECI::Cp437 => "cp437",
CharacterSetECI::ISO8859_1 => return encoding::all::ISO_8859_1,
CharacterSetECI::ISO8859_2 => "ISO-8859-2",
CharacterSetECI::ISO8859_3 => "ISO-8859-3",
CharacterSetECI::ISO8859_4 => "ISO-8859-4",
@@ -152,12 +145,13 @@ impl CharacterSetECI {
// CharacterSetECI::ISO8859_14 => "ISO-8859-14",
CharacterSetECI::ISO8859_15 => "ISO-8859-15",
CharacterSetECI::ISO8859_16 => "ISO-8859-16",
CharacterSetECI::SJIS => "Shift_JIS",
CharacterSetECI::SJIS => "shift_jis",
CharacterSetECI::Cp1250 => "windows-1250",
CharacterSetECI::Cp1251 => "windows-1251",
CharacterSetECI::Cp1252 => "windows-1252",
CharacterSetECI::Cp1256 => "windows-1256",
CharacterSetECI::UnicodeBigUnmarked => "UTF-16BE",
CharacterSetECI::UTF16LE => "UTF-16LE",
CharacterSetECI::UTF8 => "UTF-8",
CharacterSetECI::ASCII => "US-ASCII",
CharacterSetECI::Big5 => "Big5",
@@ -167,6 +161,41 @@ impl CharacterSetECI {
encoding::label::encoding_from_whatwg_label(name).unwrap()
}
pub fn getCharsetName(&self, ) -> &'static str {
match self {
// CharacterSetECI::Cp437 => "CP437",
CharacterSetECI::Cp437 => "cp437",
CharacterSetECI::ISO8859_1 => "iso-8859-1",
CharacterSetECI::ISO8859_2 => "iso-8859-2",
CharacterSetECI::ISO8859_3 => "iso-8859-3",
CharacterSetECI::ISO8859_4 => "iso-8859-4",
CharacterSetECI::ISO8859_5 => "iso-8859-5",
// CharacterSetECI::ISO8859_6 => "ISO-8859-6",
CharacterSetECI::ISO8859_7 => "iso-8859-7",
// CharacterSetECI::ISO8859_8 => "ISO-8859-8",
CharacterSetECI::ISO8859_9 => "iso-8859-9",
// CharacterSetECI::ISO8859_10 => "ISO-8859-10",
// CharacterSetECI::ISO8859_11 => "ISO-8859-11",
CharacterSetECI::ISO8859_13 => "iso-8859-13",
// CharacterSetECI::ISO8859_14 => "ISO-8859-14",
CharacterSetECI::ISO8859_15 => "iso-8859-15",
CharacterSetECI::ISO8859_16 => "iso-8859-16",
CharacterSetECI::SJIS => "shift_jis",
CharacterSetECI::Cp1250 => "windows-1250",
CharacterSetECI::Cp1251 => "windows-1251",
CharacterSetECI::Cp1252 => "windows-1252",
CharacterSetECI::Cp1256 => "windows-1256",
CharacterSetECI::UnicodeBigUnmarked => "utf-16be",
CharacterSetECI::UTF16LE => "utf-16le",
CharacterSetECI::UTF8 => "utf-8",
CharacterSetECI::ASCII => "us-ascii",
CharacterSetECI::Big5 => "big5",
CharacterSetECI::GB18030 => "gb2312",
CharacterSetECI::EUC_KR => "euc-kr",
}
}
/**
* @param charset Java character set object
* @return CharacterSetECI representing ECI for character encoding, or null if it is legal
@@ -179,7 +208,7 @@ impl CharacterSetECI {
charset.name()
};
match name {
"CP437" => Some(CharacterSetECI::Cp437),
"cp437" => Some(CharacterSetECI::Cp437),
"iso-8859-1" => Some(CharacterSetECI::ISO8859_1),
"iso-8859-2" => Some(CharacterSetECI::ISO8859_2),
"iso-8859-3" => Some(CharacterSetECI::ISO8859_3),
@@ -201,7 +230,7 @@ impl CharacterSetECI {
"windows-1252" => Some(CharacterSetECI::Cp1252),
"windows-1256" => Some(CharacterSetECI::Cp1256),
"utf-16be" => Some(CharacterSetECI::UnicodeBigUnmarked),
"utf-8" => Some(CharacterSetECI::UTF8),
"utf-8" | "utf8" => Some(CharacterSetECI::UTF8),
"us-ascii" => Some(CharacterSetECI::ASCII),
"big5" => Some(CharacterSetECI::Big5),
"gb2312" => Some(CharacterSetECI::GB18030),
@@ -255,34 +284,34 @@ impl CharacterSetECI {
* but unsupported
*/
pub fn getCharacterSetECIByName(name: &str) -> Option<CharacterSetECI> {
match name {
"CP437" => Some(CharacterSetECI::Cp437),
"ISO-8859-1" => Some(CharacterSetECI::ISO8859_1),
"ISO-8859-2" => Some(CharacterSetECI::ISO8859_2),
"ISO-8859-3" => Some(CharacterSetECI::ISO8859_3),
"ISO-8859-4" => Some(CharacterSetECI::ISO8859_4),
"ISO-8859-5" => Some(CharacterSetECI::ISO8859_5),
match name.to_lowercase().as_str() {
"cp437" => Some(CharacterSetECI::Cp437),
"iso-8859-1" => Some(CharacterSetECI::ISO8859_1),
"iso-8859-2" => Some(CharacterSetECI::ISO8859_2),
"iso-8859-3" => Some(CharacterSetECI::ISO8859_3),
"iso-8859-4" => Some(CharacterSetECI::ISO8859_4),
"iso-8859-5" => Some(CharacterSetECI::ISO8859_5),
// "ISO-8859-6" => Some(CharacterSetECI::ISO8859_6),
"ISO-8859-7" => Some(CharacterSetECI::ISO8859_7),
"iso-8859-7" => Some(CharacterSetECI::ISO8859_7),
// "ISO-8859-8" => Some(CharacterSetECI::ISO8859_8),
"ISO-8859-9" => Some(CharacterSetECI::ISO8859_9),
"iso-8859-9" => Some(CharacterSetECI::ISO8859_9),
// "ISO-8859-10" => Some(CharacterSetECI::ISO8859_10),
// "ISO-8859-11" => Some(CharacterSetECI::ISO8859_11),
"ISO-8859-13" => Some(CharacterSetECI::ISO8859_13),
"iso-8859-13" => Some(CharacterSetECI::ISO8859_13),
// "ISO-8859-14" => Some(CharacterSetECI::ISO8859_14),
"ISO-8859-15" => Some(CharacterSetECI::ISO8859_15),
"ISO-8859-16" => Some(CharacterSetECI::ISO8859_16),
"Shift_JIS" => Some(CharacterSetECI::SJIS),
"iso-8859-15" => Some(CharacterSetECI::ISO8859_15),
"iso-8859-16" => Some(CharacterSetECI::ISO8859_16),
"shift_jis" => Some(CharacterSetECI::SJIS),
"windows-1250" => Some(CharacterSetECI::Cp1250),
"windows-1251" => Some(CharacterSetECI::Cp1251),
"windows-1252" => Some(CharacterSetECI::Cp1252),
"windows-1256" => Some(CharacterSetECI::Cp1256),
"UTF-16BE" => Some(CharacterSetECI::UnicodeBigUnmarked),
"UTF-8" => Some(CharacterSetECI::UTF8),
"US-ASCII" => Some(CharacterSetECI::ASCII),
"Big5" => Some(CharacterSetECI::Big5),
"GB2312" => Some(CharacterSetECI::GB18030),
"EUC-KR" => Some(CharacterSetECI::EUC_KR),
"utf-16be" => Some(CharacterSetECI::UnicodeBigUnmarked),
"utf-8" | "utf8" => Some(CharacterSetECI::UTF8),
"us-ascii" => Some(CharacterSetECI::ASCII),
"big5" => Some(CharacterSetECI::Big5),
"gb2312" => Some(CharacterSetECI::GB18030),
"euc-kr" => Some(CharacterSetECI::EUC_KR),
_ => None,
}
}
@@ -296,7 +325,14 @@ impl CharacterSetECI {
}
pub fn decode(&self, input:&[u8]) -> Result<String> {
if self == &CharacterSetECI::Cp437 {
use codepage_437::BorrowFromCp437;
use codepage_437::CP437_CONTROL;
Ok(String::borrow_from_cp437(&input, &CP437_CONTROL))
}else {
self.getCharset().decode(input, encoding::DecoderTrap::Strict).map_err(|e| Exceptions::format_with(e.to_string()))
}
}
pub fn decode_replace(&self, input:&[u8]) -> Result<String> {

View File

@@ -14,31 +14,17 @@
* limitations under the License.
*/
// package com.google.zxing.common;
// import java.nio.charset.Charset;
// import java.nio.charset.CharsetEncoder;
// import java.nio.charset.StandardCharsets;
// import java.nio.charset.UnsupportedCharsetException;
// import java.util.ArrayList;
// import java.util.List;
use encoding::{Encoding, EncodingRef};
use unicode_segmentation::UnicodeSegmentation;
use super::CharacterSetECI;
use once_cell::sync::Lazy;
static ENCODERS: Lazy<Vec<EncodingRef>> = Lazy::new(|| {
static ENCODERS: Lazy<Vec<CharacterSetECI>> = Lazy::new(|| {
let mut enc_vec = Vec::new();
for name in NAMES {
if let Some(enc) = CharacterSetECI::getCharacterSetECIByName(name) {
// try {
enc_vec.push(CharacterSetECI::getCharset(&enc));
// } catch (UnsupportedCharsetException e) {
// continue
// }
enc_vec.push(enc);
}
}
enc_vec
@@ -83,7 +69,7 @@ const NAMES: [&str; 20] = [
*/
#[derive(Clone)]
pub struct ECIEncoderSet {
encoders: Vec<EncodingRef>,
encoders: Vec<CharacterSetECI>,
priorityEncoderIndex: Option<usize>,
}
@@ -98,22 +84,23 @@ impl ECIEncoderSet {
*/
pub fn new(
stringToEncodeMain: &str,
priorityCharset: Option<EncodingRef>,
priorityCharset: Option<CharacterSetECI>,
fnc1: Option<&str>,
) -> Self {
// List of encoders that potentially encode characters not in ISO-8859-1 in one byte.
let mut encoders: Vec<EncodingRef>;
let mut encoders: Vec<CharacterSetECI>;
let mut priorityEncoderIndexValue = None;
let mut neededEncoders: Vec<EncodingRef> = Vec::new();
let mut neededEncoders: Vec<CharacterSetECI> = Vec::new();
let stringToEncode = stringToEncodeMain.graphemes(true).collect::<Vec<&str>>();
//we always need the ISO-8859-1 encoder. It is the default encoding
neededEncoders.push(encoding::all::ISO_8859_1);
neededEncoders.push(CharacterSetECI::ISO8859_1);
let mut needUnicodeEncoder = if let Some(pc) = priorityCharset {
pc.name().starts_with("UTF") || pc.name().starts_with("utf")
//pc.name().starts_with("UTF") || pc.name().starts_with("utf")
pc == CharacterSetECI::UTF8 || pc == CharacterSetECI::UnicodeBigUnmarked
} else {
false
};
@@ -126,7 +113,7 @@ impl ECIEncoderSet {
// for (CharsetEncoder encoder : neededEncoders) {
let c = stringToEncode.get(i).unwrap();
if (fnc1.is_some() && c == fnc1.as_ref().unwrap())
|| encoder.encode(c, encoding::EncoderTrap::Strict).is_ok()
|| encoder.encode(c).is_ok()
{
canEncode = true;
break;
@@ -140,8 +127,7 @@ impl ECIEncoderSet {
// for (CharsetEncoder encoder : ENCODERS) {
if encoder
.encode(
stringToEncode.get(i).unwrap(),
encoding::EncoderTrap::Strict,
stringToEncode.get(i).unwrap()
)
.is_ok()
{
@@ -163,7 +149,7 @@ impl ECIEncoderSet {
if neededEncoders.len() == 1 && !needUnicodeEncoder {
//the entire input can be encoded by the ISO-8859-1 encoder
encoders = vec![encoding::all::ISO_8859_1];
encoders = vec![CharacterSetECI::ISO8859_1];
} else {
// we need more than one single byte encoder or we need a Unicode encoder.
// In this case we append a UTF-8 and UTF-16 encoder to the list
@@ -177,8 +163,8 @@ impl ECIEncoderSet {
encoders.push(encoder);
}
encoders.push(encoding::all::UTF_8);
encoders.push(encoding::all::UTF_16BE);
encoders.push(CharacterSetECI::UTF8);
encoders.push(CharacterSetECI::UnicodeBigUnmarked);
}
//Compute priorityEncoderIndex by looking up priorityCharset in encoders
@@ -187,7 +173,8 @@ impl ECIEncoderSet {
// for i in 0..encoders.len() {
for (i, encoder) in encoders.iter().enumerate() {
// for (int i = 0; i < encoders.length; i++) {
if priorityCharset.as_ref().unwrap().name() == encoder.name() {
// if priorityCharset.as_ref().unwrap().name() == encoder.name() {
if priorityCharset.as_ref().unwrap() == encoder {
priorityEncoderIndexValue = Some(i);
break;
}
@@ -195,7 +182,7 @@ impl ECIEncoderSet {
}
// }
//invariants
assert_eq!(encoders[0].name(), encoding::all::ISO_8859_1.name());
assert_eq!(encoders[0], CharacterSetECI::ISO8859_1);
Self {
encoders,
priorityEncoderIndex: priorityEncoderIndexValue,
@@ -212,13 +199,13 @@ impl ECIEncoderSet {
pub fn getCharsetName(&self, index: usize) -> Option<&'static str> {
if index < self.len() {
Some(self.encoders[index].name())
Some(self.encoders[index].getCharsetName())
} else {
None
}
}
pub fn getCharset(&self, index: usize) -> Option<EncodingRef> {
pub fn getCharset(&self, index: usize) -> Option<CharacterSetECI> {
if index < self.len() {
Some(self.encoders[index])
} else {
@@ -227,9 +214,10 @@ impl ECIEncoderSet {
}
pub fn getECIValue(&self, encoderIndex: usize) -> u32 {
CharacterSetECI::getValue(
&CharacterSetECI::getCharacterSetECI(self.encoders[encoderIndex]).unwrap(),
)
self.encoders[encoderIndex].getValue()
// CharacterSetECI::getValue(
// &CharacterSetECI::getCharacterSetECI(self.encoders[encoderIndex]).unwrap(),
// )
}
/*
@@ -242,7 +230,7 @@ impl ECIEncoderSet {
pub fn canEncode(&self, c: &str, encoderIndex: usize) -> Option<bool> {
if encoderIndex < self.len() {
let encoder = self.encoders[encoderIndex];
let enc_data = encoder.encode(c, encoding::EncoderTrap::Strict);
let enc_data = encoder.encode(c);
Some(enc_data.is_ok())
} else {
@@ -253,7 +241,7 @@ impl ECIEncoderSet {
pub fn encode_char(&self, c: &str, encoderIndex: usize) -> Option<Vec<u8>> {
if encoderIndex < self.len() {
let encoder = self.encoders[encoderIndex];
let enc_data = encoder.encode(c, encoding::EncoderTrap::Strict);
let enc_data = encoder.encode(c);
enc_data.ok()
// assert!(enc_data.is_ok());
// enc_data.unwrap()
@@ -265,7 +253,7 @@ impl ECIEncoderSet {
pub fn encode_string(&self, s: &str, encoderIndex: usize) -> Option<Vec<u8>> {
if encoderIndex < self.len() {
let encoder = self.encoders[encoderIndex];
encoder.encode(s, encoding::EncoderTrap::Strict).ok()
encoder.encode(s).ok()
} else {
None
}

View File

@@ -23,8 +23,6 @@
use std::fmt;
use encoding::{Encoding, EncodingRef};
use crate::common::Result;
use super::CharacterSetECI;
@@ -37,7 +35,7 @@ use super::CharacterSetECI;
pub struct ECIStringBuilder {
current_bytes: Vec<u8>,
result: String,
current_charset: Option<EncodingRef>, //= StandardCharsets.ISO_8859_1;
current_charset: Option<CharacterSetECI>, //= StandardCharsets.ISO_8859_1;
}
impl ECIStringBuilder {
@@ -45,14 +43,14 @@ impl ECIStringBuilder {
Self {
current_bytes: Vec::new(),
result: String::new(),
current_charset: Some(encoding::all::ISO_8859_1),
current_charset: Some(CharacterSetECI::ISO8859_1),
}
}
pub fn with_capacity(initial_capacity: usize) -> Self {
Self {
current_bytes: Vec::with_capacity(initial_capacity),
result: String::with_capacity(initial_capacity),
current_charset: Some(encoding::all::ISO_8859_1),
current_charset: Some(CharacterSetECI::ISO8859_1),
}
}
@@ -106,16 +104,18 @@ impl ECIStringBuilder {
pub fn appendECI(&mut self, value: u32) -> Result<()> {
self.encodeCurrentBytesIfAny();
if let Ok(character_set_eci) = CharacterSetECI::getCharacterSetECIByValue(value) {
// dbg!(
// character_set_eci,
// CharacterSetECI::getCharset(&character_set_eci).name(),
// CharacterSetECI::getCharset(&character_set_eci).whatwg_name()
// );
self.current_charset = Some(CharacterSetECI::getCharset(&character_set_eci));
} else {
self.current_charset = None
}
self.current_charset = CharacterSetECI::getCharacterSetECIByValue(value).ok();
// if let Ok(character_set_eci) = CharacterSetECI::getCharacterSetECIByValue(value) {
// // dbg!(
// // character_set_eci,
// // CharacterSetECI::getCharset(&character_set_eci).name(),
// // CharacterSetECI::getCharset(&character_set_eci).whatwg_name()
// // );
// self.current_charset = Some(character_set_eci);
// } else {
// self.current_charset = None
// }
// self.current_charset = CharacterSetECI::getCharset(&character_set_eci);
Ok(())
@@ -126,7 +126,7 @@ impl ECIStringBuilder {
/// This function can panic
pub fn encodeCurrentBytesIfAny(&mut self) {
if let Some(encoder) = self.current_charset {
if encoder.name() == encoding::all::UTF_8.name() {
if encoder == CharacterSetECI::UTF8 {
if !self.current_bytes.is_empty() {
self.result.push_str(
&String::from_utf8(std::mem::take(&mut self.current_bytes)).unwrap(),
@@ -137,7 +137,7 @@ impl ECIStringBuilder {
let bytes = std::mem::take(&mut self.current_bytes);
self.current_bytes.clear();
let encoded_value = encoder
.decode(&bytes, encoding::DecoderTrap::Strict)
.decode(&bytes)
.unwrap();
self.result.push_str(&encoded_value);
}

View File

@@ -16,13 +16,12 @@
use std::{fmt, rc::Rc};
use encoding::EncodingRef;
use unicode_segmentation::UnicodeSegmentation;
use crate::common::Result;
use crate::Exceptions;
use super::{ECIEncoderSet, ECIInput};
use super::{ECIEncoderSet, ECIInput, CharacterSetECI};
//* approximated (latch + 2 codewords)
pub const COST_PER_ECI: usize = 3;
@@ -194,7 +193,7 @@ impl MinimalECIInput {
*/
pub fn new(
stringToEncodeInput: &str,
priorityCharset: Option<EncodingRef>,
priorityCharset: Option<CharacterSetECI>,
fnc1: Option<&str>,
) -> Self {
let stringToEncode = stringToEncodeInput.graphemes(true).collect::<Vec<&str>>();

View File

@@ -14,12 +14,12 @@
* limitations under the License.
*/
use encoding::{Encoding, EncodingRef};
use crate::{DecodeHintType, DecodeHintValue, DecodingHintDictionary};
use once_cell::sync::Lazy;
use super::CharacterSetECI;
/**
* Common string-related functions.
*
@@ -50,8 +50,8 @@ const ASSUME_SHIFT_JIS: bool = false;
// static SHIFT_JIS: &'static str = "SJIS";
// static GB2312: &'static str = "GB2312";
pub static SHIFT_JIS_CHARSET: Lazy<EncodingRef> =
Lazy::new(|| encoding::label::encoding_from_whatwg_label("SJIS").unwrap());
pub static SHIFT_JIS_CHARSET: CharacterSetECI =
CharacterSetECI::SJIS;
// private static final boolean ASSUME_SHIFT_JIS =
// SHIFT_JIS_CHARSET.equals(PLATFORM_DEFAULT_ENCODING) ||
@@ -67,14 +67,14 @@ impl StringUtils {
*/
pub fn guessEncoding(bytes: &[u8], hints: &DecodingHintDictionary) -> Option<&'static str> {
let c = StringUtils::guessCharset(bytes, hints)?;
if c.name() == encoding::label::encoding_from_whatwg_label("SJIS")?.name() {
if c == CharacterSetECI::SJIS {
Some("SJIS")
} else if c.name() == encoding::all::UTF_8.name() {
} else if c == CharacterSetECI::UTF8 {
Some("UTF8")
} else if c.name() == encoding::all::ISO_8859_1.name() {
} else if c == CharacterSetECI::ISO8859_1 {
Some("ISO8859_1")
} else {
Some(c.name())
Some(&c.getCharsetName())
}
}
@@ -87,12 +87,12 @@ impl StringUtils {
* or the platform default encoding if
* none of these can possibly be correct
*/
pub fn guessCharset(bytes: &[u8], hints: &DecodingHintDictionary) -> Option<EncodingRef> {
pub fn guessCharset(bytes: &[u8], hints: &DecodingHintDictionary) -> Option<CharacterSetECI> {
if let Some(DecodeHintValue::CharacterSet(cs_name)) =
hints.get(&DecodeHintType::CHARACTER_SET)
{
// if let DecodeHintValue::CharacterSet(cs_name) = hint {
return encoding::label::encoding_from_whatwg_label(cs_name);
return CharacterSetECI::getCharacterSetECIByName(cs_name)
// }
}
// if hints.contains_key(&DecodeHintType::CHARACTER_SET) {
@@ -104,9 +104,9 @@ impl StringUtils {
&& ((bytes[0] == 0xFE && bytes[1] == 0xFF) || (bytes[0] == 0xFF && bytes[1] == 0xFE))
{
if bytes[0] == 0xFE && bytes[1] == 0xFF {
return Some(encoding::all::UTF_16BE);
return Some(CharacterSetECI::UnicodeBigUnmarked);
} else {
return Some(encoding::all::UTF_16LE);
return Some(CharacterSetECI::UTF16LE);
}
}
@@ -224,7 +224,7 @@ impl StringUtils {
// Easy -- if there is BOM or at least 1 valid not-single byte character (and no evidence it can't be UTF-8), done
if can_be_utf8 && (utf8bom || utf2_bytes_chars + utf3_bytes_chars + utf4_bytes_chars > 0) {
return Some(encoding::all::UTF_8);
return Some(CharacterSetECI::UTF8);
}
// Easy -- if assuming Shift_JIS or >= 3 valid consecutive not-ascii characters (and no evidence it can't be), done
if can_be_shift_jis
@@ -232,7 +232,7 @@ impl StringUtils {
|| sjis_max_katakana_word_length >= 3
|| sjis_max_double_bytes_word_length >= 3)
{
return encoding::label::encoding_from_whatwg_label("SJIS");
return Some(CharacterSetECI::SJIS); //encoding::label::encoding_from_whatwg_label("SJIS");
}
// Distinguishing Shift_JIS and ISO-8859-1 can be a little tough for short words. The crude heuristic is:
// - If we saw
@@ -243,23 +243,23 @@ impl StringUtils {
return if (sjis_max_katakana_word_length == 2 && sjis_katakana_chars == 2)
|| iso_high_other * 10 >= length
{
encoding::label::encoding_from_whatwg_label("SJIS")
Some(CharacterSetECI::SJIS)
} else {
Some(encoding::all::ISO_8859_1)
Some(CharacterSetECI::ISO8859_1)
};
}
// Otherwise, try in order ISO-8859-1, Shift JIS, UTF-8 and fall back to default platform encoding
if can_be_iso88591 {
return Some(encoding::all::ISO_8859_1);
return Some(CharacterSetECI::ISO8859_1);
}
if can_be_shift_jis {
return Some(encoding::label::encoding_from_whatwg_label("SJIS").unwrap());
return Some(CharacterSetECI::SJIS);
}
if can_be_utf8 {
return Some(encoding::all::UTF_8);
return Some(CharacterSetECI::UTF8);
}
// Otherwise, we take a wild guess with platform encoding
Some(encoding::all::UTF_8)
Some(CharacterSetECI::UTF8)
}
}