From d5e6a5d0a73306d34dd16722493db65345bb767a Mon Sep 17 00:00:00 2001 From: Henry Schimke Date: Tue, 7 Mar 2023 17:09:42 -0600 Subject: [PATCH] update eci_string_builder This change modifies how the ECIStringBuilder works. Perviously it encoded data as it went, whenever the eci state changed. The new method only decodes data when it is requested, simply appending bytes as it goes otherwise. This should improve performance in a few situations. This release also uses the ECIStringBuilder in qrcodes, which previously handled their own decoding. --- src/common/eci_string_builder.rs | 166 ++++++++++-------- .../decoder/decoded_bit_stream_parser.rs | 8 +- .../decoder/decoded_bit_stream_parser.rs | 29 ++- .../decoder/decoded_bit_stream_parser.rs | 126 +++++++------ 4 files changed, 183 insertions(+), 146 deletions(-) diff --git a/src/common/eci_string_builder.rs b/src/common/eci_string_builder.rs index 4378d21..a43dd6a 100644 --- a/src/common/eci_string_builder.rs +++ b/src/common/eci_string_builder.rs @@ -23,8 +23,6 @@ use std::fmt; -use crate::common::Result; - use super::{CharacterSet, Eci}; /** @@ -32,35 +30,36 @@ use super::{CharacterSet, Eci}; * * @author Alex Geller */ +#[derive(Default)] pub struct ECIStringBuilder { - current_bytes: Vec, - result: String, - current_charset: CharacterSet, //= StandardCharsets.ISO_8859_1; + is_eci: bool, + eci_result: Option, + bytes: Vec, + eci_positions: Vec<(Eci, usize, usize)>, // (Eci, start, end) } impl ECIStringBuilder { - pub fn new() -> Self { - Self { - current_bytes: Vec::new(), - result: String::new(), - current_charset: CharacterSet::ISO8859_1, - } - } pub fn with_capacity(initial_capacity: usize) -> Self { Self { - current_bytes: Vec::with_capacity(initial_capacity), - result: String::with_capacity(initial_capacity), - current_charset: CharacterSet::ISO8859_1, + eci_result: None, + bytes: Vec::with_capacity(initial_capacity), + eci_positions: Vec::default(), + is_eci: false, } } + pub fn bytes(&self) -> &[u8] { + &self.bytes + } + /** * Appends {@code value} as a byte value * * @param value character whose lowest byte is to be appended */ pub fn append_char(&mut self, value: char) { - self.current_bytes.push(value as u8); + self.eci_result = None; + self.bytes.push(value as u8); } /** @@ -69,7 +68,13 @@ impl ECIStringBuilder { * @param value byte to append */ pub fn append_byte(&mut self, value: u8) { - self.current_bytes.push(value); + self.eci_result = None; + self.bytes.push(value) + } + + pub fn append_bytes(&mut self, value: &[u8]) { + self.eci_result = None; + self.bytes.extend_from_slice(value) } /** @@ -78,12 +83,11 @@ impl ECIStringBuilder { * @param value string to append */ pub fn append_string(&mut self, value: &str) { - value - .as_bytes() - .iter() - .map(|b| self.current_bytes.push(*b)) - .count(); - // self.current_bytes.push(value.as_bytes()); + if !value.is_ascii() { + self.append_eci(Eci::UTF8); + } + self.eci_result = None; + self.bytes.extend_from_slice(value.as_bytes()); } /** @@ -101,49 +105,78 @@ impl ECIStringBuilder { * @param value ECI value to append, as an int * @throws FormatException on invalid ECI value */ - pub fn appendECI(&mut self, eci: Eci) -> Result<()> { - self.encodeCurrentBytesIfAny(); + pub fn append_eci(&mut self, eci: Eci) { + self.eci_result = None; + if !self.is_eci && eci != Eci::ISO8859_1 { + self.is_eci = true; + } - self.current_charset = eci.into(); //CharacterSet::get_character_set_by_eci(value).ok(); + if self.is_eci { + if let Some(last) = self.eci_positions.last_mut() { + last.2 = self.bytes.len() + } - // if let Ok(character_set_eci) = CharacterSetECI::getCharacterSetECIByValue(value) { - // // dbg!( - // // character_set_eci, - // // CharacterSetECI::getCharset(&character_set_eci).name(), - // // CharacterSetECI::getCharset(&character_set_eci).whatwg_name() - // // ); - // self.current_charset = Some(character_set_eci); - // } else { - // self.current_charset = None - // } - - // self.current_charset = CharacterSetECI::getCharset(&character_set_eci); - Ok(()) + self.eci_positions.push((eci, self.bytes.len(), 0)); + } } /// Finishes encoding anything in the buffer using the current ECI and resets. /// /// This function can panic - pub fn encodeCurrentBytesIfAny(&mut self) { - if ![CharacterSet::Binary, CharacterSet::Unknown].contains(&self.current_charset) { - if self.current_charset == CharacterSet::UTF8 { - if !self.current_bytes.is_empty() { - self.result.push_str( - &String::from_utf8(std::mem::take(&mut self.current_bytes)).unwrap(), - ); - self.current_bytes.clear(); + pub fn encodeCurrentBytesIfAny(&self) -> String { + let mut encoded_string = String::with_capacity(self.bytes.len()); + // First encode the first set + let (_, end, _) = + *self + .eci_positions + .first() + .unwrap_or(&(Eci::ISO8859_1, self.bytes.len(), 0)); + + encoded_string.push_str( + &Self::encode_segment(&self.bytes[0..end], Eci::ISO8859_1).unwrap_or_default(), + ); + + // If there are more sets, encode each of them in turn + for (eci, eci_start, eci_end) in &self.eci_positions { + // let (_,end) = *self.eci_positions.first().unwrap_or(&(*eci, self.bytes.len())); + let end = if *eci_end == 0 { + self.bytes.len() + } else { + *eci_end + }; + encoded_string.push_str( + &Self::encode_segment(&self.bytes[*eci_start..end], *eci).unwrap_or_default(), + ); + } + + // Return the result + encoded_string + } + + fn encode_segment(bytes: &[u8], eci: Eci) -> Option { + let mut encoded_string = String::with_capacity(bytes.len()); + if ![Eci::Binary, Eci::Unknown].contains(&eci) { + if eci == Eci::UTF8 { + if !bytes.is_empty() { + encoded_string.push_str(&CharacterSet::UTF8.decode(bytes).ok()?); + } else { + return None; } - } else if !self.current_bytes.is_empty() { - let bytes = std::mem::take(&mut self.current_bytes); - self.current_bytes.clear(); - let encoded_value = self.current_charset.decode(&bytes).unwrap(); - self.result.push_str(&encoded_value); + } else if !bytes.is_empty() { + encoded_string.push_str(&CharacterSet::from(eci).decode(bytes).ok()?); + } else { + return None; } } else { - for byte in &self.current_bytes { - self.result.push(char::from(*byte)) + for byte in bytes { + encoded_string.push(char::from(*byte)) } - self.current_bytes.clear(); + } + + if encoded_string.is_empty() { + None + } else { + Some(encoded_string) } } @@ -153,8 +186,7 @@ impl ECIStringBuilder { * @param value characters to append */ pub fn appendCharacters(&mut self, value: &str) { - self.encodeCurrentBytesIfAny(); - self.result.push_str(value); + self.append_string(value); } /** @@ -163,19 +195,18 @@ impl ECIStringBuilder { * @return length of string representation in characters */ pub fn len(&mut self) -> usize { - self.encodeCurrentBytesIfAny(); //return toString().length(); - self.result.chars().count() + self.bytes.len() } /** * @return true iff nothing has been appended */ pub fn is_empty(&mut self) -> bool { - self.current_bytes.is_empty() && self.result.is_empty() + self.bytes.is_empty() } pub fn build_result(mut self) -> Self { - self.encodeCurrentBytesIfAny(); + self.eci_result = Some(self.encodeCurrentBytesIfAny()); self } @@ -183,13 +214,10 @@ impl ECIStringBuilder { impl fmt::Display for ECIStringBuilder { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - //self.encodeCurrentBytesIfAny(); - write!(f, "{}", self.result) - } -} - -impl Default for ECIStringBuilder { - fn default() -> Self { - Self::new() + if let Some(res) = &self.eci_result { + write!(f, "{res}") + } else { + write!(f, "{}", self.encodeCurrentBytesIfAny()) + } } } diff --git a/src/datamatrix/decoder/decoded_bit_stream_parser.rs b/src/datamatrix/decoder/decoded_bit_stream_parser.rs index 7f3bb88..9642470 100644 --- a/src/datamatrix/decoder/decoded_bit_stream_parser.rs +++ b/src/datamatrix/decoder/decoded_bit_stream_parser.rs @@ -739,21 +739,21 @@ fn decodeBase256Segment( fn decodeECISegment(bits: &mut BitSource, result: &mut ECIStringBuilder) -> Result { let firstByte = bits.readBits(8)?; if firstByte <= 127 { - result.appendECI(Eci::from(firstByte - 1))?; + result.append_eci(Eci::from(firstByte - 1)); return Ok(true); } let secondByte = bits.readBits(8)?; if firstByte <= 191 { - result.appendECI(Eci::from((firstByte - 128) * 254 + 127 + secondByte - 1))?; + result.append_eci(Eci::from((firstByte - 128) * 254 + 127 + secondByte - 1)); return Ok((firstByte - 128) * 254 + 127 + secondByte - 1 > 900); } let thirdByte = bits.readBits(8)?; - result.appendECI(Eci::from( + result.append_eci(Eci::from( (firstByte - 192) * 64516 + 16383 + (secondByte - 1) * 254 + thirdByte - 1, - ))?; + )); Ok((firstByte - 192) * 64516 + 16383 + (secondByte - 1) * 254 + thirdByte - 1 > 900) } diff --git a/src/pdf417/decoder/decoded_bit_stream_parser.rs b/src/pdf417/decoder/decoded_bit_stream_parser.rs index 6b4b0cf..4ef124b 100644 --- a/src/pdf417/decoder/decoded_bit_stream_parser.rs +++ b/src/pdf417/decoder/decoded_bit_stream_parser.rs @@ -128,7 +128,7 @@ pub fn decode(codewords: &[u32], ecLevel: &str) -> Result { codeIndex = numericCompaction(codewords, codeIndex, &mut result)? } ECI_CHARSET => { - result.appendECI(Eci::from(codewords[codeIndex]))?; + result.append_eci(Eci::from(codewords[codeIndex])); codeIndex += 1; } ECI_GENERAL_PURPOSE => @@ -234,54 +234,47 @@ pub fn decodeMacroBlock( codeIndex += 1; match codewords[codeIndex] { MACRO_PDF417_OPTIONAL_FIELD_FILE_NAME => { - let mut fileName = ECIStringBuilder::new(); + let mut fileName = ECIStringBuilder::default(); codeIndex = textCompaction(codewords, codeIndex + 1, &mut fileName)?; - fileName = fileName.build_result(); resultMetadata.setFileName(fileName.to_string()); } MACRO_PDF417_OPTIONAL_FIELD_SENDER => { - let mut sender = ECIStringBuilder::new(); + let mut sender = ECIStringBuilder::default(); codeIndex = textCompaction(codewords, codeIndex + 1, &mut sender)?; - sender = sender.build_result(); resultMetadata.setSender(sender.to_string()); } MACRO_PDF417_OPTIONAL_FIELD_ADDRESSEE => { - let mut addressee = ECIStringBuilder::new(); + let mut addressee = ECIStringBuilder::default(); codeIndex = textCompaction(codewords, codeIndex + 1, &mut addressee)?; - addressee = addressee.build_result(); resultMetadata.setAddressee(addressee.to_string()); } MACRO_PDF417_OPTIONAL_FIELD_SEGMENT_COUNT => { - let mut segmentCount = ECIStringBuilder::new(); + let mut segmentCount = ECIStringBuilder::default(); codeIndex = numericCompaction(codewords, codeIndex + 1, &mut segmentCount)?; - segmentCount = segmentCount.build_result(); let Ok(parsed_segment_count) = segmentCount.to_string().parse() else { return Err(Exceptions::FORMAT); }; resultMetadata.setSegmentCount(parsed_segment_count); } MACRO_PDF417_OPTIONAL_FIELD_TIME_STAMP => { - let mut timestamp = ECIStringBuilder::new(); + let mut timestamp = ECIStringBuilder::default(); codeIndex = numericCompaction(codewords, codeIndex + 1, &mut timestamp)?; - timestamp = timestamp.build_result(); let Ok(parsed_timestamp) = timestamp.to_string().parse() else { return Err(Exceptions::FORMAT); }; resultMetadata.setTimestamp(parsed_timestamp); } MACRO_PDF417_OPTIONAL_FIELD_CHECKSUM => { - let mut checksum = ECIStringBuilder::new(); + let mut checksum = ECIStringBuilder::default(); codeIndex = numericCompaction(codewords, codeIndex + 1, &mut checksum)?; - checksum = checksum.build_result(); let Ok(parsed_checksum ) = checksum.to_string().parse() else { return Err(Exceptions::FORMAT); }; resultMetadata.setChecksum(parsed_checksum); } MACRO_PDF417_OPTIONAL_FIELD_FILE_SIZE => { - let mut fileSize = ECIStringBuilder::new(); + let mut fileSize = ECIStringBuilder::default(); codeIndex = numericCompaction(codewords, codeIndex + 1, &mut fileSize)?; - fileSize = fileSize.build_result(); let Ok(parsed_file_size)= fileSize.to_string().parse() else { return Err(Exceptions::FORMAT); }; @@ -387,7 +380,7 @@ fn textCompaction( subMode, ) .ok_or(Exceptions::ILLEGAL_STATE)?; - result.appendECI(Eci::from(codewords[codeIndex]))?; + result.append_eci(Eci::from(codewords[codeIndex])); codeIndex += 1; textCompactionData = vec![0; (codewords[0] as usize - codeIndex) * 2]; byteCompactionData = vec![0; (codewords[0] as usize - codeIndex) * 2]; @@ -616,7 +609,7 @@ fn byteCompaction( //handle leading ECIs while codeIndex < codewords[0] as usize && codewords[codeIndex] == ECI_CHARSET { codeIndex += 1; - result.appendECI(Eci::from(codewords[codeIndex]))?; + result.append_eci(Eci::from(codewords[codeIndex])); codeIndex += 1; } @@ -654,7 +647,7 @@ fn byteCompaction( if code < TEXT_COMPACTION_MODE_LATCH { result.append_byte(code as u8); } else if code == ECI_CHARSET { - result.appendECI(Eci::from(codewords[codeIndex]))?; + result.append_eci(Eci::from(codewords[codeIndex])); codeIndex += 1; } else { codeIndex -= 1; diff --git a/src/qrcode/decoder/decoded_bit_stream_parser.rs b/src/qrcode/decoder/decoded_bit_stream_parser.rs index 5924d37..d30a122 100644 --- a/src/qrcode/decoder/decoded_bit_stream_parser.rs +++ b/src/qrcode/decoder/decoded_bit_stream_parser.rs @@ -15,7 +15,9 @@ */ use crate::{ - common::{BitSource, CharacterSet, DecoderRXingResult, Eci, Result, StringUtils}, + common::{ + BitSource, CharacterSet, DecoderRXingResult, ECIStringBuilder, Eci, Result, StringUtils, + }, DecodingHintDictionary, Exceptions, }; @@ -46,7 +48,7 @@ pub fn decode( hints: &DecodingHintDictionary, ) -> Result { let mut bits = BitSource::new(bytes.to_owned()); - let mut result = String::with_capacity(50); + let mut result = ECIStringBuilder::with_capacity(50); //String::with_capacity(50); let mut byteSegments = vec![vec![0u8; 0]; 0]; let mut symbolSequence = -1; let mut parityData = -1; @@ -91,7 +93,7 @@ pub fn decode( Mode::ECI => { // Count doesn't apply to ECI let value = parseECIValue(&mut bits)?; - currentCharacterSetECI = CharacterSet::from(Eci::from(value)).into(); //CharacterSet::get_character_set_by_eci(value).ok(); + currentCharacterSetECI = CharacterSet::from(value).into(); //CharacterSet::get_character_set_by_eci(value).ok(); if currentCharacterSetECI.is_none() { return Err(Exceptions::format_with(format!( "Value of {value} not valid" @@ -142,7 +144,27 @@ pub fn decode( } } - let symbologyModifier = if currentCharacterSetECI.is_some() { + let symbologyModifier = get_symbology_identifier( + currentCharacterSetECI.is_some(), + hasFNC1first, + hasFNC1second, + ); + + Ok(DecoderRXingResult::with_all( + bytes.to_owned(), + result.build_result().to_string(), + byteSegments.to_vec(), + format!("{}", u8::from(ecLevel)), + symbolSequence, + parityData, + symbologyModifier, + String::default(), + false, + )) +} + +fn get_symbology_identifier(has_charset: bool, hasFNC1first: bool, hasFNC1second: bool) -> u32 { + if has_charset { if hasFNC1first { 4 } else if hasFNC1second { @@ -156,25 +178,17 @@ pub fn decode( 5 } else { 1 - }; - - Ok(DecoderRXingResult::with_all( - bytes.to_owned(), - result, - byteSegments.to_vec(), - format!("{}", u8::from(ecLevel)), - symbolSequence, - parityData, - symbologyModifier, - String::default(), - false, - )) + } } /** * See specification GBT 18284-2000 */ -fn decodeHanziSegment(bits: &mut BitSource, result: &mut String, count: usize) -> Result<()> { +fn decodeHanziSegment( + bits: &mut BitSource, + result: &mut ECIStringBuilder, + count: usize, +) -> Result<()> { // Don't crash trying to read more bits than we have available. if count * 13 > bits.available() { return Err(Exceptions::FORMAT); @@ -203,17 +217,15 @@ fn decodeHanziSegment(bits: &mut BitSource, result: &mut String, count: usize) - count -= 1; } - let gb_encoder = CharacterSet::GB18030; - let encode_string = gb_encoder - .decode(&buffer) - .map_err(|e| Exceptions::parse_with(format!("unable to decode buffer {buffer:?}: {e}")))?; - result.push_str(&encode_string); + result.append_eci(Eci::GB18030); + result.append_bytes(&buffer); + Ok(()) } fn decodeKanjiSegment( bits: &mut BitSource, - result: &mut String, + result: &mut ECIStringBuilder, count: usize, currentCharacterSetECI: Option, hints: &DecodingHintDictionary, @@ -265,18 +277,15 @@ fn decodeKanjiSegment( CharacterSet::Shift_JIS }; - let encode_string = encoder - .decode(&buffer) - .map_err(|e| Exceptions::parse_with(format!("unable to decode buffer {buffer:?}: {e}")))?; - - result.push_str(&encode_string); + result.append_eci(Eci::from(encoder)); + result.append_bytes(&buffer); Ok(()) } fn decodeByteSegment( bits: &mut BitSource, - result: &mut String, + result: &mut ECIStringBuilder, count: usize, currentCharacterSetECI: Option, byteSegments: &mut Vec>, @@ -320,11 +329,9 @@ fn decodeByteSegment( // ) }; - let encode_string = encoding.decode(&readBytes).map_err(|e| { - Exceptions::parse_with(format!("unable to decode buffer {readBytes:?}: {e}")) - })?; + result.append_eci(Eci::from(encoding)); + result.append_bytes(&readBytes); - result.push_str(&encode_string); byteSegments.push(readBytes); Ok(()) @@ -343,20 +350,21 @@ fn toAlphaNumericChar(value: u32) -> Result { fn decodeAlphanumericSegment( bits: &mut BitSource, - result: &mut String, + result: &mut ECIStringBuilder, count: usize, fc1InEffect: bool, ) -> Result<()> { + let mut r_hld = String::with_capacity(count); // Read two characters at a time - let start = result.len(); + let start = 0; let mut count = count; while count > 1 { if bits.available() < 11 { return Err(Exceptions::FORMAT); } let nextTwoCharsBits = bits.readBits(11)?; - result.push(toAlphaNumericChar(nextTwoCharsBits / 45)?); - result.push(toAlphaNumericChar(nextTwoCharsBits % 45)?); + r_hld.push(toAlphaNumericChar(nextTwoCharsBits / 45)?); + r_hld.push(toAlphaNumericChar(nextTwoCharsBits % 45)?); count -= 2; } if count == 1 { @@ -364,39 +372,47 @@ fn decodeAlphanumericSegment( if bits.available() < 6 { return Err(Exceptions::FORMAT); } - result.push(toAlphaNumericChar(bits.readBits(6)?)?); + r_hld.push(toAlphaNumericChar(bits.readBits(6)?)?); } // See section 6.4.8.1, 6.4.8.2 if fc1InEffect { // We need to massage the result a bit if in an FNC1 mode: - for i in start..result.len() { - if result + for i in start..r_hld.len() { + if r_hld .chars() .nth(i) .ok_or(Exceptions::INDEX_OUT_OF_BOUNDS)? == '%' { if i < result.len() - 1 - && result + && r_hld .chars() .nth(i + 1) .ok_or(Exceptions::INDEX_OUT_OF_BOUNDS)? == '%' { // %% is rendered as % - result.remove(i + 1); + r_hld.remove(i + 1); } else { // In alpha mode, % should be converted to FNC1 separator 0x1D - result.replace_range(i..i + 1, "\u{1D}"); + r_hld.replace_range(i..i + 1, "\u{1D}"); } } } } + result.append_eci(Eci::ISO8859_1); + result.append_string(&r_hld); + Ok(()) } -fn decodeNumericSegment(bits: &mut BitSource, result: &mut String, count: usize) -> Result<()> { +fn decodeNumericSegment( + bits: &mut BitSource, + result: &mut ECIStringBuilder, + count: usize, +) -> Result<()> { + result.append_eci(Eci::ISO8859_1); let mut count = count; // Read three digits at a time while count >= 3 { @@ -408,9 +424,9 @@ fn decodeNumericSegment(bits: &mut BitSource, result: &mut String, count: usize) if threeDigitsBits >= 1000 { return Err(Exceptions::FORMAT); } - result.push(toAlphaNumericChar(threeDigitsBits / 100)?); - result.push(toAlphaNumericChar((threeDigitsBits / 10) % 10)?); - result.push(toAlphaNumericChar(threeDigitsBits % 10)?); + result.append_char(toAlphaNumericChar(threeDigitsBits / 100)?); + result.append_char(toAlphaNumericChar((threeDigitsBits / 10) % 10)?); + result.append_char(toAlphaNumericChar(threeDigitsBits % 10)?); count -= 3; } if count == 2 { @@ -422,8 +438,8 @@ fn decodeNumericSegment(bits: &mut BitSource, result: &mut String, count: usize) if twoDigitsBits >= 100 { return Err(Exceptions::FORMAT); } - result.push(toAlphaNumericChar(twoDigitsBits / 10)?); - result.push(toAlphaNumericChar(twoDigitsBits % 10)?); + result.append_char(toAlphaNumericChar(twoDigitsBits / 10)?); + result.append_char(toAlphaNumericChar(twoDigitsBits % 10)?); } else if count == 1 { // One digit left over to read if bits.available() < 4 { @@ -433,27 +449,27 @@ fn decodeNumericSegment(bits: &mut BitSource, result: &mut String, count: usize) if digitBits >= 10 { return Err(Exceptions::FORMAT); } - result.push(toAlphaNumericChar(digitBits)?); + result.append_char(toAlphaNumericChar(digitBits)?); } Ok(()) } -fn parseECIValue(bits: &mut BitSource) -> Result { +fn parseECIValue(bits: &mut BitSource) -> Result { let firstByte = bits.readBits(8)?; if (firstByte & 0x80) == 0 { // just one byte - return Ok(firstByte & 0x7F); + return Ok(Eci::from(firstByte & 0x7F)); } if (firstByte & 0xC0) == 0x80 { // two bytes let secondByte = bits.readBits(8)?; - return Ok(((firstByte & 0x3F) << 8) | secondByte); + return Ok(Eci::from(((firstByte & 0x3F) << 8) | secondByte)); } if (firstByte & 0xE0) == 0xC0 { // three bytes let secondThirdBytes = bits.readBits(16)?; - return Ok(((firstByte & 0x1F) << 16) | secondThirdBytes); + return Ok(Eci::from(((firstByte & 0x1F) << 16) | secondThirdBytes)); } Err(Exceptions::FORMAT)