Loading page…
Rust walkthroughs
Loading page…
zstd::stream::Encoder::auto_finish ensure data integrity when the encoder is dropped?zstd::stream::Encoder::auto_finish returns a wrapper that implements Drop to automatically call finish() when the encoder goes out of scope, ensuring all compressed data is flushed to the underlying writer even if you forget to explicitly complete the compression. Without this mechanism, dropping an encoder without finishing it would silently lose data—the encoder's internal buffers contain compressed but unwritten data that must be flushed to complete the compressed stream. The auto_finish wrapper is essential for writing correct code in the presence of early returns, errors, or other control flow that might bypass an explicit finish() call, as it guarantees the compressor's final block marker and any remaining buffered data are written to the output.
use zstd::stream::Encoder;
use std::io::Write;
fn compress_data_bad(data: &[u8]) -> Vec<u8> {
let mut output = Vec::new();
{
let mut encoder = Encoder::new(&mut output, 3).unwrap();
// Write some data
encoder.write_all(data).unwrap();
// PROBLEM: Encoder is dropped without calling finish()
// The compressed data in internal buffers is LOST
}
// output may be incomplete or corrupted
output
}
fn main() {
let data = b"Hello, World! This is a test of zstd compression.";
let compressed = compress_data_bad(data);
// Attempting to decompress will fail
let result = zstd::stream::decode_all(&compressed[..]);
match result {
Ok(_) => println!("Unexpected success - data was complete"),
Err(e) => println!("Expected failure: {}", e),
}
}Dropping an encoder without finish() leaves compressed data in internal buffers.
use zstd::stream::Encoder;
use std::io::Write;
fn compress_data_explicit(data: &[u8]) -> Vec<u8> {
let mut output = Vec::new();
{
let mut encoder = Encoder::new(&mut output, 3).unwrap();
encoder.write_all(data).unwrap();
// MUST call finish() to write final data
encoder.finish().unwrap();
}
output
}
fn main() {
let data = b"Hello, World! This is a test of zstd compression.";
let compressed = compress_data_explicit(data);
// Decompression now works
let decompressed = zstd::stream::decode_all(&compressed[..]).unwrap();
assert_eq!(decompressed, data);
println!("Successfully compressed and decompressed {} bytes", data.len());
}finish() flushes internal buffers and writes the stream end marker.
use zstd::stream::Encoder;
use std::io::Write;
// Risk: Early return or error path might skip finish()
fn compress_with_risk(data: &[u8]) -> Result<Vec<u8>, String> {
let mut output = Vec::new();
let mut encoder = Encoder::new(&mut output, 3)
.map_err(|e| format!("Encoder creation failed: {}", e))?;
// What if this fails?
encoder.write_all(data)
.map_err(|e| format!("Write failed: {}", e))?;
// What if we return early here?
if data.len() < 10 {
return Err("Data too short".to_string());
// BUG: encoder dropped without finish()!
}
// What about exceptions in subsequent code?
some_operation()?; // If this fails, finish() is skipped
encoder.finish()
.map_err(|e| format!("Finish failed: {}", e))?;
Ok(output)
}
fn some_operation() -> Result<(), String> {
Err("Some error".to_string())
}Early returns and errors can skip finish(), causing data loss.
use zstd::stream::Encoder;
use std::io::Write;
fn compress_with_auto_finish(data: &[u8]) -> Result<Vec<u8>, String> {
let mut output = Vec::new();
{
// auto_finish() wraps the encoder
// The wrapper calls finish() on drop
let mut encoder = Encoder::new(&mut output, 3)
.map_err(|e| format!("Encoder creation failed: {}", e))?
.auto_finish();
encoder.write_all(data)
.map_err(|e| format!("Write failed: {}", e))?;
// Early return is now safe!
if data.len() < 10 {
return Err("Data too short".to_string());
// encoder dropped, but auto_finish calls finish()
}
// Errors after this point are also safe
// The Drop implementation handles finishing
}
Ok(output)
}
fn main() {
// Test with early return
let result = compress_with_auto_finish(b"short");
match result {
Ok(_) => println!("Unexpected success"),
Err(e) => {
println!("Error: {}", e);
// But data was still properly finished
}
}
// Test normal case
let result = compress_with_auto_finish(b"This is longer data that should compress well.");
match result {
Ok(compressed) => {
let decompressed = zstd::stream::decode_all(&compressed[..]).unwrap();
println!("Successfully compressed {} bytes", decompressed.len());
}
Err(e) => println!("Error: {}", e),
}
}auto_finish() wraps the encoder so finish() is called automatically on drop.
use zstd::stream::Encoder;
use std::io::Write;
// Conceptual implementation of auto_finish:
//
// pub struct AutoFinishEncoder<W: Write> {
// encoder: Option<Encoder<W>>,
// }
//
// impl<W: Write> Drop for AutoFinishEncoder<W> {
// fn drop(&mut self) {
// // Take ownership of the encoder
// if let Some(encoder) = self.encoder.take() {
// // Call finish() to flush data
// match encoder.finish() {
// Ok(_) => {} // Success
// Err(e) => { /* Log or handle error */ }
// }
// }
// }
// }
fn main() {
let mut output = Vec::new();
// Create encoder
let encoder = Encoder::new(&mut output, 3).unwrap();
// auto_finish() creates the wrapper
let mut encoder = encoder.auto_finish();
// Write data
encoder.write_all(b"test data").unwrap();
// When encoder goes out of scope, Drop::drop() is called
// which internally calls finish()
}AutoFinishEncoder implements Drop to call finish() when destroyed.
use zstd::stream::Encoder;
use std::io::Write;
fn main() {
let mut output = Vec::new();
// auto_finish() returns an AutoFinishEncoder
// which wraps the original Encoder
let encoder = Encoder::new(&mut output, 3).unwrap();
let auto_encoder = encoder.auto_finish();
// AutoFinishEncoder implements Write, so it can be used like Encoder
// The signature is: pub fn auto_finish(self) -> AutoFinishEncoder<W>
// This consumes the original encoder
// You cannot call finish() on auto_encoder manually
// (AutoFinishEncoder doesn't expose finish())
}
// If you need to call finish() manually, don't use auto_finish:
fn manual_finish_example() -> Vec<u8> {
let mut output = Vec::new();
let mut encoder = Encoder::new(&mut output, 3).unwrap();
encoder.write_all(b"data").unwrap();
// Manual finish() - you control when it happens
encoder.finish().unwrap();
output
}
// If you want automatic safety, use auto_finish:
fn auto_finish_example() -> Vec<u8> {
let mut output = Vec::new();
{
let mut encoder = Encoder::new(&mut output, 3).unwrap().auto_finish();
encoder.write_all(b"data").unwrap();
// finish() called automatically
}
output
}auto_finish() consumes the encoder and returns a wrapper that handles finish() on drop.
use zstd::stream::Encoder;
use std::io::Write;
fn compress_with_error_handling(data: &[u8]) -> Result<Vec<u8>, String> {
let mut output = Vec::new();
{
let mut encoder = Encoder::new(&mut output, 3)
.map_err(|e| format!("Failed to create encoder: {}", e))?
.auto_finish();
// Write might fail (e.g., if output is full or disk error)
encoder.write_all(data)
.map_err(|e| format!("Failed to write: {}", e))?;
// If write failed and we return early, auto_finish still runs
}
// Note: If finish() fails in Drop, the error is typically swallowed
// because Drop::drop cannot return Result
//
// This is a trade-off: auto_finish guarantees finish() is called,
// but you lose the ability to handle finish() errors
Ok(output)
}
// For critical error handling, use explicit finish:
fn compress_explicit_error_handling(data: &[u8]) -> Result<Vec<u8>, String> {
let mut output = Vec::new();
let mut encoder = Encoder::new(&mut output, 3)
.map_err(|e| format!("Failed to create encoder: {}", e))?;
encoder.write_all(data)
.map_err(|e| format!("Failed to write: {}", e))?;
// Explicit finish lets you handle errors
encoder.finish()
.map_err(|e| format!("Failed to finish: {}", e))?;
Ok(output)
}
fn main() {
let data = b"Test data for compression";
let result = compress_with_error_handling(data);
println!("Result: {:?}", result);
let result = compress_explicit_error_handling(data);
println!("Result: {:?}", result);
}auto_finish trades explicit error handling for guaranteed finish() execution.
use zstd::stream::Encoder;
use std::fs::File;
use std::io::{BufWriter, Write};
fn compress_file_auto(input_path: &str, output_path: &str) -> Result<(), std::io::Error> {
let input = std::fs::read(input_path)?;
// Use BufWriter for better performance
let file = File::create(output_path)?;
let buf_writer = BufWriter::new(file);
// auto_finish ensures file is properly written even on panic
let mut encoder = Encoder::new(buf_writer, 3)?.auto_finish();
encoder.write_all(&input)?;
// finish() called automatically when encoder is dropped
// This flushes to BufWriter, which flushes to File
Ok(())
}
// For comparison, explicit finish:
fn compress_file_explicit(input_path: &str, output_path: &str) -> Result<(), std::io::Error> {
let input = std::fs::read(input_path)?;
let file = File::create(output_path)?;
let buf_writer = BufWriter::new(file);
let mut encoder = Encoder::new(buf_writer, 3)?;
encoder.write_all(&input)?;
// Explicit finish - must remember this
encoder.finish()?;
Ok(())
}
fn main() {
// Create a test file
std::fs::write("test_input.txt", b"This is test content for compression.").unwrap();
// Compress with auto_finish
compress_file_auto("test_input.txt", "test_output.zst").unwrap();
// Verify by decompressing
let compressed = std::fs::read("test_output.zst").unwrap();
let decompressed = zstd::stream::decode_all(&compressed[..]).unwrap();
println!("Decompressed: {}", String::from_utf8_lossy(&decompressed));
}auto_finish ensures file data is properly written even in complex control flow.
use zstd::stream::Encoder;
use std::io::Write;
fn stream_compress(chunks: &[&[u8]]) -> Vec<u8> {
let mut output = Vec::new();
{
let mut encoder = Encoder::new(&mut output, 3).unwrap().auto_finish();
// Write multiple chunks
for chunk in chunks {
encoder.write_all(chunk).unwrap();
}
// No need to remember finish() - auto_finish handles it
}
output
}
fn main() {
let chunks: Vec<&[u8]> = vec![
b"First chunk of data. ",
b"Second chunk of data. ",
b"Third chunk of data.",
];
let compressed = stream_compress(&chunks);
// Verify
let decompressed = zstd::stream::decode_all(&compressed[..]).unwrap();
let expected = b"First chunk of data. Second chunk of data. Third chunk of data.";
assert_eq!(&decompressed[..], expected);
println!("Streamed compression successful: {} bytes -> {} bytes",
expected.len(), compressed.len());
}auto_finish is especially useful for streaming where multiple writes occur.
use zstd::stream::Encoder;
use std::io::Write;
// Use auto_finish when:
// 1. You want guaranteed data integrity without thinking about it
// 2. You have complex control flow with early returns
// 3. You're writing library code that shouldn't surprise users
// 4. Performance of finish error handling isn't critical
fn safe_compress(data: &[u8]) -> Vec<u8> {
let mut output = Vec::new();
// auto_finish: set and forget
let mut encoder = Encoder::new(&mut output, 3).unwrap().auto_finish();
encoder.write_all(data).unwrap();
// Done - finish() will be called
output
}
// Use explicit finish() when:
// 1. You need to handle finish errors
// 2. You want to measure compression statistics
// 3. You need precise control over when finish happens
// 4. You're in a performance-critical path and want explicit control
fn explicit_compress(data: &[u8]) -> Result<Vec<u8>, String> {
let mut output = Vec::new();
let mut encoder = Encoder::new(&mut output, 3)
.map_err(|e| format!("Create error: {}", e))?;
encoder.write_all(data)
.map_err(|e| format!("Write error: {}", e))?;
// Explicit finish with error handling
let encoder = encoder.finish()
.map_err(|e| format!("Finish error: {}", e))?;
// You can also get statistics from the returned encoder
// (depends on zstd crate version)
Ok(output)
}
fn main() {
let data = b"Test data for compression comparison";
// Safe but no error handling
let compressed = safe_compress(data);
println!("Safe compression: {} bytes", compressed.len());
// Explicit with error handling
match explicit_compress(data) {
Ok(compressed) => println!("Explicit compression: {} bytes", compressed.len()),
Err(e) => println!("Error: {}", e),
}
}Choose based on whether you need error handling from finish().
// | Feature | auto_finish | Explicit finish() |
// |----------------------|-----------------------|--------------------|
// | Guaranteed finish | Yes (Drop) | No (must remember) |
// | Error handling | Swallowed in Drop | Full control |
// | Code simplicity | Simpler | More explicit |
// | Control flow safety | Safe | Manual management |
// | Performance control | Automatic | Manual |
// | Statistics access | Not available | Available on return|auto_finish behavior:
| Situation | Without auto_finish | With auto_finish | |-----------|--------------------|--------------------| | Normal completion | Must call finish() | Automatic | | Early return | Data lost | Automatic finish | | Panic/unwrap | Data lost | Automatic finish | | Error propagation | Must handle before drop | Automatic finish |
When to use each:
| Use auto_finish | Use explicit finish() | |----------------|-----------------------| | Complex control flow | Need finish error handling | | Library code | Need compression statistics | | Quick scripts | Performance-critical paths | | Defensive programming | Precise timing control |
Key insight: auto_finish embodies the Rust philosophy of making correct behavior automatic. The compression process has an inherent requirement: you must call finish() to flush internal buffers and write the stream end marker. Forgetting this produces corrupt or incomplete output. By wrapping the encoder in a type that implements Drop to call finish(), the zstd crate makes the correct behavior automatic—any path that causes the encoder to go out of scope, whether it's normal completion, early return, or panic, will properly finalize the compressed stream. The trade-off is that finish() can fail (the underlying writer might error), and Drop::drop cannot propagate errors. If you need to handle finish errors, you must use explicit finish() and handle its Result. For most use cases—especially when writing to memory (Vec<u8>) or files—auto_finish is the safer default because it eliminates an entire class of bugs where data is silently lost due to forgotten finish() calls.