utf8conv

Parser for reading and converting between UTF8 / UTF32 data without requiring heap memory, targeting no_std environments

Utf8conv can operate based on a single input buffer, or a series of input buffers. The outputs are produced one at a time, directly delivered to client caller with minimum latency.

Includes an adapter iterator to filter out Byte Order Mark at the beginning of a stream, and substituting carriage returns with newlines.

Utf8conv is dual licensed under the MIT License, and Apache 2.0 License.

Source Repository: link

Credits attribution of utf8conv is located in source directory doc/utf8conv-credits.md.

Single buffer iterator based parsing

```rust use utf8conv::*;

/// Single buffer iterator based UTF8 parsing converting to char fn utf8tocharsinglebufferiterator() { let mybuffer = "abc".asbytes(); let mut utf8refiter = mybuffer.iter(); let mut parser = FromUtf8::new(); let mut iterator = parser.utf8reftocharwithiter(& mut utf8refiter); while let Some(charval) = iterator.next() { println!("{}", charval); println!("{}", iterator.hasinvalid_sequence()); } }

/// Single buffer iterator based char parsing converting to UTF8 fn chartoutf8singlebufferiterator() { let mybuffer = [ '\u{7F}', '\u{80}', '\u{81}', '\u{82}' ]; let mut charrefiter = mybuffer.iter(); let mut parser = FromUnicode::new(); let mut iterator = parser.charreftoutf8withiter(& mut charrefiter); while let Some(utf8val) = iterator.next() { println!("{:#02x}", utf8val); println!("{}", iterator.hasinvalidsequence()); } } ```

Multi-buffer iterator based parsing

```rust use utf8conv::*;

/// Multi-buffer iterator based UTF8 parsing converting to char fn utf8tocharmultibufferiterator() { let mybuffers = ["ab".asbytes(), "".asbytes(), "cde".asbytes()]; let mut parser = FromUtf8::new(); for indx in 0 .. mybuffers.len() { parser.setislastbuffer(indx == mybuffers.len() - 1); let mut utf8refiter = mybuffers[indx].iter(); let mut iterator = parser.utf8reftocharwithiter(& mut utf8refiter); while let Some(charval) = iterator.next() { println!("{}", charval); println!("{}", iterator.hasinvalidsequence()); } } }

/// Multi-buffer iterator based char parsing converting to UTF8 fn chartoutf8multibufferiterator() { let mybuffers = [[ '\u{7F}', '\u{80}' ] , [ '\u{81}', '\u{82}' ]]; let mut parser = FromUnicode::new(); for indx in 0 .. mybuffers.len() { parser.setislastbuffer(indx == mybuffers.len() - 1); let mut charrefiter = mybuffers[indx].iter(); let mut iterator = parser.charreftoutf8withiter(& mut charrefiter); while let Some(utf8val) = iterator.next() { println!("{:#02x}", utf8val); println!("{}", iterator.hasinvalid_sequence()); } } } ```

Single-buffer slice based parsing

```rust use utf8conv::*;

/// Single-buffer slice reading based UTF8 parsing converting to char fn utf8tocharsinglebufferslicereading() { let mybuffer = "Wxyz".asbytes(); let mut parser = FromUtf8::new(); let mut curslice = mybuffer; loop { match parser.utf8tochar(curslice) { Result::Ok((slicepos, charval)) => { curslice = slicepos; println!("{}", charval); println!("{}", parser.hasinvalidsequence()); } Result::Err(MoreEnum::More(_amt)) => { // _amt equals to 0 when end of data break; } } } }

/// Single-buffer slice reading based UTF32 parsing converting to UTF8 fn utf32toutf8singlebufferslicereading() { let mybuffer = [0x7Fu32, 0x80u32, 0x81u32, 0x82u32]; let mut parser = FromUnicode::new(); let mut currentslice = & mybuffer[..]; loop { match parser.utf32toutf8(currentslice) { Result::Ok((slicepos, utf8val)) => { currentslice = slicepos; println!("{:02x}", utf8val); println!("{}", parser.hasinvalidsequence()); } Result::Err(MoreEnum::More(amt)) => { // _amt equals to 0 when end of data break; } } } } ```

Multi-buffer slice based parsing

```rust use utf8conv::*;

/// Multi-buffer slice reading based UTF8 parsing converting to char fn utf8tocharmultibufferslicereading() { let mybuffers = ["Wx".asbytes(), "".asbytes(), "yz".asbytes()]; let mut parser = FromUtf8::new(); for indx in 0 .. mybuffers.len() { parser.setislastbuffer(indx == mybuffers.len() - 1); let mut curslice = mybuffers[indx]; loop { match parser.utf8tochar(curslice) { Result::Ok((slicepos, charval)) => { curslice = slicepos; println!("{}", charval); println!("{}", parser.hasinvalidsequence()); } Result::Err(MoreEnum::More(amt)) => { // _amt equals to 0 when end of data break; } } } } }

/// Multi-buffer slice reading based UTF32 parsing converting to UTF8 fn utf32toutf8multibufferslicereading() { let mybuffers = [[0x7Fu32, 0x80u32], [0x81u32, 0x82u32]]; let mut parser = FromUnicode::new(); for indx in 0 .. mybuffers.len() { parser.setislastbuffer(indx == mybuffers.len() - 1); let currentarray = mybuffers[indx]; let mut currentslice = & currentarray[..]; loop { match parser.utf32toutf8(currentslice) { Result::Ok((slicepos, utf8val)) => { currentslice = slicepos; println!("{:02x}", utf8val); println!("{}", parser.hasinvalidsequence()); } Result::Err(MoreEnum::More(_amt)) => { // _amt equals to 0 when end of data break; } } } } } ```