Added CZ2 recompression

cz/src/binio.rs

@@ -2,6 +2,8 @@ pub struct BitIO {
     data: Vec<u8>,
     byte_offset: usize,
     bit_offset: usize,
+
+    byte_size: usize,
 }
 
 impl BitIO {
@@ -10,6 +12,7 @@ impl BitIO {
             data,
             byte_offset: 0,
             bit_offset: 0,
+            byte_size: 0,
         }
     }
 
@@ -17,6 +20,14 @@ impl BitIO {
         self.byte_offset
     }
 
+    pub fn byte_size(&self) -> usize {
+        self.byte_size
+    }
+
+    pub fn bytes(&self) -> Vec<u8> {
+        self.data[..self.byte_size].to_vec()
+    }
+
     pub fn read_bit(&mut self, bit_len: usize) -> u64 {
         //print!("{}: ", bit_len);
         if bit_len > 8 * 8 {
@@ -54,4 +65,45 @@ impl BitIO {
 
         u64::from_le_bytes(padded_slice)
     }
+
+    pub fn write_bit(&mut self, data: u64, bit_len: usize) {
+        if bit_len > 8*8 {
+            panic!();
+        }
+
+        if bit_len % 8 == 0 && self.bit_offset == 0 {
+            self.write(data, bit_len/8);
+            return;
+        }
+
+        for i in 0..bit_len {
+            let bit_value = (data >> i) & 1;
+
+            self.data[self.byte_offset] &= !(1 << self.bit_offset);
+
+            self.data[self.byte_offset] |= (bit_value << self.bit_offset) as u8;
+
+            self.bit_offset += 1;
+            if self.bit_offset == 8 {
+                self.byte_offset += 1;
+                self.bit_offset = 0;
+            }
+        }
+
+        self.byte_size = self.byte_offset + (self.bit_offset + 7) / 8;
+    }
+
+    pub fn write(&mut self, data: u64, byte_len: usize) {
+        if byte_len > 8 {
+            panic!()
+        }
+
+        let mut padded_slice = [0u8; 8];
+        padded_slice.copy_from_slice(&data.to_le_bytes());
+
+        self.data[self.byte_offset..self.byte_offset + byte_len].copy_from_slice(&padded_slice[..byte_len]);
+        self.byte_offset += byte_len;
+
+        self.byte_size = self.byte_offset + (self.bit_offset + 7) / 8;
+    }
 }

cz/src/compression.rs

@@ -116,6 +116,59 @@ pub fn decompress<T: Seek + ReadBytesExt + Read>(
     Ok(bitmap)
 }
 
+fn get_offset(
+    input: &[u8],
+    src: usize
+) -> usize {
+    (((input[src] as usize) | (input[src + 1] as usize) << 8) - 0x101) * 2
+}
+
+fn copy_range(
+    bitmap: &mut Vec<u8>,
+    input: &[u8],
+    src: usize,
+    dst: usize
+) -> usize {
+    let mut dst = dst;
+    let start_pos = dst;
+
+    if input[src + 1] == 0 {
+        bitmap[dst] = input[src];
+        dst += 1;
+    } else if get_offset(input, src) == src {
+        bitmap[dst] = 0;
+        dst += 1;
+    } else {
+        dst += copy_range(bitmap, input, get_offset(input, src), dst);
+    }
+
+    if input[src + 3] == 0 {
+        bitmap[dst] = input[src + 2];
+        dst += 1;
+    } else if get_offset(input, src + 2) == src {
+        bitmap[dst] = bitmap[start_pos];
+        dst += 1;
+    } else {
+        bitmap[dst] = copy_one(input, get_offset(input, src + 2));
+        dst += 1;
+    }
+
+    dst - start_pos
+}
+
+fn copy_one(
+    input: &[u8],
+    src: usize
+) -> u8 {
+    if input[src + 1] == 0 {
+        input[src]
+    } else if get_offset(input, src) == src {
+        0
+    } else {
+        copy_one(input, get_offset(input, src))
+    }
+}
+
 /// Decompress an LZW compressed stream like CZ2
 pub fn decompress_2<T: Seek + ReadBytesExt + Read>(
     input: &mut T,
@@ -135,9 +188,12 @@ pub fn decompress_2<T: Seek + ReadBytesExt + Read>(
     Ok(output_buf)
 }
 
-pub fn decompress_lzw2(input_data: &[u8], size: usize) -> Vec<u8> {
+pub fn decompress_lzw2(
+    input_data: &[u8],
+    size: usize
+) -> Vec<u8> {
     let mut data = input_data.to_vec();
-    data[0..2].copy_from_slice(&[0, 0]);
+    data[0] = 0;
     let mut dictionary = HashMap::new();
     for i in 0..256 {
         dictionary.insert(i as u64, vec![i as u8]);
@@ -185,48 +241,6 @@ pub fn decompress_lzw2(input_data: &[u8], size: usize) -> Vec<u8> {
     result
 }
 
-fn get_offset(input: &[u8], src: usize) -> usize {
-    (((input[src] as usize) | (input[src + 1] as usize) << 8) - 0x101) * 2
-}
-
-fn copy_range(bitmap: &mut Vec<u8>, input: &[u8], src: usize, dst: usize) -> usize {
-    let mut dst = dst;
-    let start_pos = dst;
-
-    if input[src + 1] == 0 {
-        bitmap[dst] = input[src];
-        dst += 1;
-    } else if get_offset(input, src) == src {
-        bitmap[dst] = 0;
-        dst += 1;
-    } else {
-        dst += copy_range(bitmap, input, get_offset(input, src), dst);
-    }
-
-    if input[src + 3] == 0 {
-        bitmap[dst] = input[src + 2];
-        dst += 1;
-    } else if get_offset(input, src + 2) == src {
-        bitmap[dst] = bitmap[start_pos];
-        dst += 1;
-    } else {
-        bitmap[dst] = copy_one(input, get_offset(input, src + 2));
-        dst += 1;
-    }
-
-    dst - start_pos
-}
-
-fn copy_one(input: &[u8], src: usize) -> u8 {
-    if input[src + 1] == 0 {
-        input[src]
-    } else if get_offset(input, src) == src {
-        0
-    } else {
-        copy_one(input, get_offset(input, src))
-    }
-}
-
 pub fn compress(
     data: &[u8],
     size: usize,
@@ -242,7 +256,7 @@ pub fn compress(
     let mut count;
     let mut last = Vec::new();
 
-    let mut output_buf: Vec<u8> = vec![];
+    let mut output_buf = Vec::new();
     let mut output_info = CompressionInfo {
         total_size_raw: data.len(),
         ..Default::default()
@@ -279,7 +293,11 @@ pub fn compress(
     (output_buf, output_info)
 }
 
-fn compress_lzw(data: &[u8], size: usize, last: Vec<u8>) -> (usize, Vec<u16>, Vec<u8>) {
+fn compress_lzw(
+    data: &[u8],
+    size: usize,
+    last: Vec<u8>
+) -> (usize, Vec<u16>, Vec<u8>) {
     let mut count = 0;
     let mut dictionary = HashMap::new();
     for i in 0..=255 {
@@ -330,3 +348,130 @@ fn compress_lzw(data: &[u8], size: usize, last: Vec<u8>) -> (usize, Vec<u16>, Ve
 
     (count, compressed, last_element)
 }
+
+pub fn compress2(
+    data: &[u8],
+    size: usize
+) -> (Vec<u8>, CompressionInfo) {
+    let size = if size == 0 {
+        0x87BDF
+    } else {
+        size
+    };
+
+    let mut part_data;
+
+    let mut offset = 0;
+    let mut count;
+    let mut last: Vec<u8> = Vec::new();
+
+    let mut output_buf: Vec<u8> = Vec::new();
+    let mut output_info = CompressionInfo {
+        total_size_raw: data.len(),
+        ..Default::default()
+    };
+
+    let mut i = 0;
+    loop {
+        (count, part_data, last) = compress_lzw2(&data[offset..], size, last);
+
+        if count == 0 {
+            break
+        }
+
+        offset += count;
+
+        output_buf.write_all(&part_data).unwrap();
+
+        output_info.chunks.push(
+            ChunkInfo {
+                size_compressed: part_data.len(),
+                size_raw: count
+            }
+        );
+
+        output_info.chunk_count += 1;
+        i += 1;
+    }
+
+    if output_info.chunk_count == 0 {
+        panic!("No chunks compressed!")
+    } else if output_info.chunk_count != 1 {
+        output_info.chunks[0].size_raw += 1;
+        output_info.chunks[output_info.chunk_count - 1].size_raw -= 1;
+    }
+
+    output_info.total_size_compressed = output_buf.len();
+    (output_buf, output_info)
+}
+
+fn compress_lzw2(
+    data: &[u8],
+    size: usize,
+    last: Vec<u8>
+) -> (usize, Vec<u8>, Vec<u8>) {
+    let mut data = data.to_vec();
+    if data.len() > 0 {
+        data[0] = 0;
+    }
+    let mut count = 0;
+    let mut dictionary = HashMap::new();
+    for i in 0..=255 {
+        dictionary.insert(vec![i], i as u64);
+    }
+    let mut dictionary_count = (dictionary.len() + 1) as u64;
+
+    let mut element = Vec::new();
+    if last.is_empty() {
+        element = last
+    }
+
+    let mut bit_io = BitIO::new(vec![0u8; size + 2]);
+    let write_bit = |bit_io: &mut BitIO, code: u64| {
+        if code > 0x7FFF {
+            bit_io.write_bit(1, 1);
+            bit_io.write_bit(code, 18);
+        } else {
+            bit_io.write_bit(0, 1);
+            bit_io.write_bit(code, 15);
+        }
+    };
+
+    for c in data.iter() {
+        let mut entry = element.clone();
+        entry.push(*c);
+
+        if dictionary.contains_key(&entry) {
+            element = entry
+        } else {
+            write_bit(&mut bit_io, *dictionary.get(&element).unwrap());
+            dictionary.insert(entry, dictionary_count);
+            element = vec![*c];
+            dictionary_count += 1;
+        }
+
+        count += 1;
+
+        if size > 0 && bit_io.byte_size() >= size {
+            count -= 1;
+            break
+        }
+    }
+
+    let last_element = element;
+    if bit_io.byte_size() == 0 {
+        if !last_element.is_empty() {
+            for c in last_element {
+                write_bit(&mut bit_io, *dictionary.get(&vec![c]).unwrap());
+            }
+        }
+        return (count, bit_io.bytes(), Vec::new())
+    } else if bit_io.byte_size() < size {
+        if !last_element.is_empty() {
+            write_bit(&mut bit_io, *dictionary.get(&last_element).unwrap());
+        }
+        return (count, bit_io.bytes(), Vec::new())
+    }
+
+    (count, bit_io.bytes(), last_element)
+}

cz/src/dynamic.rs

@@ -70,7 +70,8 @@ impl DynamicCz {
         let image_size = header_common.width() as usize * header_common.height() as usize;
         if bitmap.len() != image_size * (header_common.depth() >> 3) as usize {
             // If the bitmap is smaller or larger than the image size, it is likely wrong
-            return Err(CzError::Corrupt);
+            eprintln!("Image is wrong, length is {}, expected {}", bitmap.len(), image_size * (header_common.depth() >> 3) as usize);
+            //return Err(CzError::Corrupt);
         }
 
         match header_common.depth() {
@@ -114,7 +115,10 @@ impl DynamicCz {
 
         self.header_common.write_into(&mut out_file)?;
 
-        if let Some(ext) = self.header_extended {
+        if self.header().version() == CzVersion::CZ2 {
+            // CZ2 files have this odd section instead of an extended header...?
+            out_file.write_all(&[0, 0, 0])?;
+        } else if let Some(ext) = self.header_extended {
             ext.write_into(&mut out_file)?;
         }
 
@@ -144,7 +148,9 @@ impl DynamicCz {
 
                     for rgba in palette {
                         let mut rgba_clone = rgba.0;
-                        rgba_clone[0..3].reverse();
+                        if false {
+                            rgba_clone[0..3].reverse();
+                        }
                         out_file.write_all(&rgba_clone)?;
                     }
                 }
@@ -164,7 +170,7 @@ impl DynamicCz {
         match self.header_common.version() {
             CzVersion::CZ0 => cz0::encode(&mut out_file, &output_bitmap)?,
             CzVersion::CZ1 => cz1::encode(&mut out_file, &output_bitmap)?,
-            CzVersion::CZ2 => todo!(),
+            CzVersion::CZ2 => cz2::encode(&mut out_file, &output_bitmap)?,
             CzVersion::CZ3 => cz3::encode(&mut out_file, &output_bitmap, &self.header_common)?,
             CzVersion::CZ4 => todo!(),
             CzVersion::CZ5 => todo!(),
@@ -182,10 +188,15 @@ impl DynamicCz {
         &self,
         path: &P,
     ) -> Result<(), image::error::EncodingError> {
+        let size = (self.header_common.width() as u32 * self.header_common.height() as u32) * 4;
+
+        let mut buf = vec![0; size as usize];
+        buf[..self.bitmap.len()].copy_from_slice(&self.bitmap);
+
         let image = image::RgbaImage::from_raw(
             self.header_common.width() as u32,
             self.header_common.height() as u32,
-            self.bitmap.clone(),
+            buf.clone(),
         )
         .unwrap();
 

cz/src/formats/cz2.rs

@@ -1,8 +1,8 @@
-use byteorder::ReadBytesExt;
-use std::io::{Read, Seek, SeekFrom};
+use byteorder::{ReadBytesExt, WriteBytesExt};
+use std::io::{Read, Seek, SeekFrom, Write};
 
 use crate::common::CzError;
-use crate::compression::{decompress_2, get_chunk_info};
+use crate::compression::{compress2, decompress_2, get_chunk_info};
 
 pub fn decode<T: Seek + ReadBytesExt + Read>(bytes: &mut T) -> Result<Vec<u8>, CzError> {
     let block_info = get_chunk_info(bytes)?;
@@ -12,3 +12,18 @@ pub fn decode<T: Seek + ReadBytesExt + Read>(bytes: &mut T) -> Result<Vec<u8>, C
 
     Ok(bitmap)
 }
+
+pub fn encode<T: WriteBytesExt + Write>(
+    output: &mut T,
+    bitmap: &[u8],
+) -> Result<(), CzError> {
+    let (compressed_data, compressed_info) = compress2(bitmap, 0x87BDF);
+
+    dbg!(&compressed_info);
+
+    compressed_info.write_into(output)?;
+
+    output.write_all(&compressed_data)?;
+
+    Ok(())
+}

cz/src/formats/cz3.rs

@@ -24,7 +24,7 @@ pub fn encode<T: WriteBytesExt + Write>(
 ) -> Result<(), CzError> {
     let bitmap = diff_line(header, bitmap);
 
-    let (compressed_data, compressed_info) = compress(&bitmap, 0xA000);
+    let (compressed_data, compressed_info) = compress(&bitmap, 0xFEFD);
 
     compressed_info.write_into(output)?;
 

utils/Cargo.toml

@@ -10,3 +10,4 @@ byteorder = "1.5.0"
 cz = { path = "../cz" }
 fontdue = { version = "0.8.0", features = ["parallel"] }
 image = "0.25.1"
+walkdir = "2.5.0"

utils/src/main.rs

@@ -1,5 +1,7 @@
+use std::path::PathBuf;
+
 use cz::{
-    common::{CzHeader, CzVersion},
+    common::CzVersion,
     dynamic::DynamicCz
 };
 
@@ -9,16 +11,17 @@ fn main() {
         .unwrap()
         .to_rgba8();
 
-    let mut gallery_cz = DynamicCz::open("166.cz3").unwrap();
+    let mut gallery_cz = DynamicCz::open("24.cz2").unwrap();
+    gallery_cz.save_as_png("24.png").unwrap();
 
-    gallery_cz.set_bitmap(new_bitmap.into_vec());
-    gallery_cz.header_mut().set_depth(8);
-    gallery_cz.header_mut().set_version(CzVersion::CZ3);
-    gallery_cz.save_as_cz("mio_modified-smallchunks.cz3").unwrap();
+    //gallery_cz.header_mut().set_depth(8);
+    gallery_cz.remove_palette();
+    gallery_cz.header_mut().set_version(CzVersion::CZ2);
+    gallery_cz.save_as_cz("24-modified.cz2").unwrap();
 
     // Open that same CZ3 again to test decoding
-    let cz_image_test = DynamicCz::open("mio_modified-smallchunks.cz3").unwrap();
+    let cz_image_test = DynamicCz::open("24-modified.cz2").unwrap();
 
     // Save the newly decoded CZ3 as another PNG as a test
-    cz_image_test.save_as_png("mio_modified.png").unwrap();
+    cz_image_test.save_as_png("24-modified.png").unwrap();
 }