How do I work with bytes buffers in Rust?
Walkthrough
The bytes crate provides efficient byte manipulation types for network programming and other binary data scenarios. It offers Bytes (an immutable, reference-counted byte slice) and BytesMut (a mutable byte buffer). These types are designed for zero-copy operations, efficient cloning, and seamless integration with async I/O. The crate is fundamental to many async networking libraries like Tokio, hyper, and Tower.
Key types:
Bytes— immutable, cloneable byte buffer with reference countingBytesMut— mutable byte buffer for building dataBuftrait — read bytes from a bufferBufMuttrait — write bytes to a buffer- Zero-copy slicing — cheap views into existing buffers
Bytes enables efficient data sharing without copying, ideal for network protocols and parsing.
Code Example
# Cargo.toml
[dependencies]
bytes = "1"use bytes::Bytes;
fn main() {
// Create Bytes from various sources
let b = Bytes::from("Hello, World!");
println!("{}", String::from_utf8_lossy(&b));
// Clone is O(1) - reference counted
let b2 = b.clone();
println!("Cloned: {:?}", b2);
// Slice is zero-copy
let slice = b.slice(0..5);
println!("Sliced: {:?}", String::from_utf8_lossy(&slice));
}Creating Bytes
use bytes::Bytes;
fn main() {
// From static bytes (no allocation)
let b1 = Bytes::from_static(b"Hello");
println!("Static: {:?}", b1);
// From Vec<u8>
let vec = vec![1, 2, 3, 4, 5];
let b2 = Bytes::from(vec);
println!("From Vec: {:?}", b2);
// From slice (copies)
let slice: &[u8] = &[10, 20, 30];
let b3 = Bytes::copy_from_slice(slice);
println!("From slice: {:?}", b3);
// From String
let b4 = Bytes::from("Hello, World!");
println!("From string: {:?}", b4);
// Empty bytes
let empty = Bytes::new();
println!("Empty: {:?}", empty);
// With capacity
let b5 = Bytes::with_capacity(1024);
println!("Capacity: {}", b5.capacity());
// From iterator
let b6: Bytes = (0..5).collect();
println!("From iterator: {:?}", b6);
// Default
let b7: Bytes = Default::default();
assert!(b7.is_empty());
}Bytes Operations
use bytes::Bytes;
fn main() {
let b = Bytes::from("Hello, World!");
// Length
println!("Length: {}", b.len());
// Check if empty
println!("Is empty: {}", b.is_empty());
// Access by index
println!("First byte: {}", b[0]);
// Iterate
for byte in &b {
print!("{} ", byte);
}
println!();
// Slice (zero-copy)
let slice = b.slice(7..12);
println!("Sliced: {:?}", String::from_utf8_lossy(&slice));
// Split at index
let (left, right) = b.split_at(5);
println!("Left: {:?}", String::from_utf8_lossy(&left));
println!("Right: {:?}", String::from_utf8_lossy(&right));
// Clone (O(1) - reference counting)
let clone = b.clone();
println!("Clone: {:?}", clone);
// Convert to Vec
let vec: Vec<u8> = b.to_vec();
println!("Vec: {:?}", vec);
// Convert to slice
let slice: &[u8] = &b;
println!("Slice: {:?}", slice);
// Check equality
let b2 = Bytes::from("Hello, World!");
println!("Equal: {}", b == b2);
}
// Slicing and splitting preserve the underlying data
fn demonstrate_zero_copy() {
let original = Bytes::from("Hello, World!");
println!("Original: {:?}", original);
// Split off the front
let (hello, rest) = original.split_at(5);
let (comma, world) = rest.split_at(2);
println!("Hello: {:?}", String::from_utf8_lossy(&hello));
println!("Comma: {:?}", String::from_utf8_lossy(&comma));
println!("World: {:?}", String::from_utf8_lossy(&world));
// All three share the same underlying memory!
}BytesMut - Mutable Buffer
use bytes::{BytesMut, BufMut};
fn main() {
// Create mutable buffer
let mut buf = BytesMut::with_capacity(64);
println!("Capacity: {}, Len: {}", buf.capacity(), buf.len());
// Write bytes
buf.put_slice(b"Hello");
println!("After put_slice: {:?}", buf);
// Put single byte
buf.put_u8(b' ');
println!("After put_u8: {:?}", buf);
// Put multiple bytes
buf.put(&b"World"[..]);
println!("After put: {:?}", buf);
// Convert to Bytes (immutable)
let bytes = buf.freeze();
println!("Frozen: {:?}", bytes);
// Create with default capacity
let mut buf2 = BytesMut::new();
buf2.extend_from_slice(b"test");
println!("Buf2: {:?}", buf2);
// From Vec
let vec = vec![1, 2, 3, 4];
let mut buf3 = BytesMut::from(vec.as_slice());
println!("From Vec: {:?}", buf3);
}
// Building buffers incrementally
fn build_buffer() {
let mut buf = BytesMut::with_capacity(1024);
for i in 0..10 {
buf.put_slice(format!("Line {}\n", i).as_bytes());
}
println!("Buffer contents: {:?}", String::from_utf8_lossy(&buf));
println!("Len: {}, Capacity: {}", buf.len(), buf.capacity());
}The Buf Trait - Reading
use bytes::{Bytes, Buf};
fn main() {
// Reading from Bytes
let mut buf = Bytes::from(&[1, 2, 3, 4, 5, 6, 7, 8][..]);
// Get remaining length
println!("Remaining: {}", buf.remaining());
// Read single byte
let byte = buf.get_u8();
println!("Byte: {}", byte);
println!("Remaining: {}", buf.remaining());
// Read u16 (big-endian by default)
let val = buf.get_u16();
println!("u16: {}", val);
// Read u32
let val = buf.get_u32();
println!("u32: {}", val);
// Check if has remaining
println!("Has remaining: {}", buf.has_remaining());
// Copy to slice
let mut buf = Bytes::from(&[1, 2, 3, 4, 5][..]);
let mut dest = [0u8; 3];
buf.copy_to_slice(&mut dest);
println!("Copied: {:?}", dest);
println!("Remaining: {}", buf.remaining());
// Little-endian
let mut buf = Bytes::from(&[0x01, 0x02, 0x03, 0x04][..]);
let val_le = buf.get_u32_le();
println!("u32 LE: {}", val_le);
// Various integer types
let mut buf = Bytes::from(&[
0x01, 0x02, // u16
0x03, 0x04, 0x05, 0x06, // u32
0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, // u128
][..]);
let v16 = buf.get_u16();
let v32 = buf.get_u32();
let v64 = buf.get_u64();
println!("Values: {}, {}, {}", v16, v32, v64);
}
// Advanced Buf operations
fn advanced_buf_ops() {
use bytes::Buf;
let mut buf = Bytes::from(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10][..]);
// Peek without advancing
let first_byte = buf.chunk()[0];
println!("First byte (peek): {}", first_byte);
println!("Position unchanged, remaining: {}", buf.remaining());
// Get a slice of remaining bytes
let chunk = buf.chunk();
println!("Chunk: {:?}", chunk);
// Advance position manually
buf.advance(2);
println!("After advance: {:?}", buf.chunk());
// Split the buffer
let mut buf = Bytes::from(&[1, 2, 3, 4, 5, 6][..]);
let front = buf.copy_to_bytes(3);
println!("Front: {:?}", front);
println!("Back: {:?}", buf.chunk());
}The BufMut Trait - Writing
use bytes::{BytesMut, BufMut};
fn main() {
let mut buf = BytesMut::with_capacity(64);
// Write single byte
buf.put_u8(0x01);
println!("After u8: {:?}", buf);
// Write u16 (big-endian)
buf.put_u16(0x0203);
println!("After u16: {:?}", buf);
// Write u32
buf.put_u32(0x04050607);
println!("After u32: {:?}", buf);
// Write u64
buf.put_u64(0x08090A0B0C0D0E0F);
println!("After u64: {:?}", buf);
// Little-endian
let mut buf_le = BytesMut::new();
buf_le.put_u16_le(0x0102);
buf_le.put_u32_le(0x03040506);
println!("LE buffer: {:?}", buf_le);
// Write slice
let mut buf2 = BytesMut::new();
buf2.put_slice(b"Hello");
println!("After slice: {:?}", buf2);
// Write from another Buf
let source = Bytes::from("World");
buf2.put(source);
println!("After put: {:?}", String::from_utf8_lossy(&buf2));
}
// Building a protocol message
fn build_message() {
let mut buf = BytesMut::new();
// Message type (1 byte)
buf.put_u8(0x01);
// Message length (4 bytes)
let payload = b"Hello, World!";
buf.put_u32(payload.len() as u32);
// Payload
buf.put_slice(payload);
// Checksum (simple)
let checksum: u8 = payload.iter().fold(0u8, |acc, &b| acc.wrapping_add(b));
buf.put_u8(checksum);
println!("Message: {:?}", buf);
println!("Length: {}", buf.len());
}
// Variable-length integers
fn write_varint() {
use bytes::BufMut;
let mut buf = BytesMut::new();
// Write variable-length integer (like protobuf)
let mut write_varint = |mut val: u64| {
while val >= 0x80 {
buf.put_u8((val as u8) | 0x80);
val >>= 7;
}
buf.put_u8(val as u8);
};
write_varint(1); // 1 byte
write_varint(127); // 1 byte
write_varint(128); // 2 bytes
write_varint(300); // 2 bytes
write_varint(16383); // 2 bytes
write_varint(16384); // 3 bytes
println!("Varint buffer: {:?}", buf);
}Slicing and Splitting
use bytes::{Bytes, BytesMut};
fn main() {
// Slicing Bytes
let data = Bytes::from("Hello, World!");
// slice returns a new Bytes view
let hello = data.slice(0..5);
let world = data.slice(7..12);
println!("Hello: {:?}", String::from_utf8_lossy(&hello));
println!("World: {:?}", String::from_utf8_lossy(&world));
// Original still valid
println!("Original: {:?}", String::from_utf8_lossy(&data));
// Split at position
let data = Bytes::from("Hello, World!");
let (left, right) = data.split_at(5);
println!("Split left: {:?}", String::from_utf8_lossy(&left));
println!("Split right: {:?}", String::from_utf8_lossy(&right));
// Split off from BytesMut
let mut buf = BytesMut::from("Hello, World!");
let hello = buf.split_to(5);
println!("Split off: {:?}", String::from_utf8_lossy(&hello));
println!("Remaining: {:?}", String::from_utf8_lossy(&buf));
// Split off the end
let mut buf = BytesMut::from("Hello, World!");
let world = buf.split_off(7);
println!("Kept: {:?}", String::from_utf8_lossy(&buf));
println!("Split off: {:?}", String::from_utf8_lossy(&world));
// Truncate
let mut buf = BytesMut::from("Hello, World!");
buf.truncate(5);
println!("Truncated: {:?}", String::from_utf8_lossy(&buf));
// Clear
let mut buf = BytesMut::from("Hello");
buf.clear();
println!("After clear: {}", buf.len());
}
// Reference counting demonstration
fn reference_counting() {
let original = Bytes::from("Hello, World!");
println!("Original: {:?}", original);
// Clone is O(1) - shares underlying data
let clone1 = original.clone();
let clone2 = original.clone();
// Slices also share data
let slice1 = original.slice(0..5);
let slice2 = original.slice(7..12);
// All point to same memory!
println!("All share the same underlying buffer");
println!("Clone1: {:?}", String::from_utf8_lossy(&clone1));
println!("Slice1: {:?}", String::from_utf8_lossy(&slice1));
}Capacity Management
use bytes::{BytesMut, BufMut};
fn main() {
let mut buf = BytesMut::new();
println!("Initial capacity: {}", buf.capacity());
// Reserve capacity
buf.reserve(100);
println!("After reserve: {}", buf.capacity());
// Write data
buf.put_slice(&[1, 2, 3, 4, 5]);
println!("Len: {}, Capacity: {}", buf.len(), buf.capacity());
// Automatic growth
for i in 0..100 {
buf.put_u8(i as u8);
}
println!("After writes - Len: {}, Capacity: {}", buf.len(), buf.capacity());
// Reserve additional space
buf.reserve(1000);
println!("After reserve - Capacity: {}", buf.capacity());
// With capacity constructor
let mut buf2 = BytesMut::with_capacity(1024);
println!("Pre-allocated capacity: {}", buf2.capacity());
// Shrink to fit (if supported)
let mut buf3 = BytesMut::with_capacity(1000);
buf3.put_slice(b"small");
println!("Before shrink - Len: {}, Capacity: {}", buf3.len(), buf3.capacity());
}
// Efficient buffer reuse
fn reuse_buffer() {
let mut buf = BytesMut::with_capacity(1024);
for _ in 0..3 {
buf.clear();
buf.put_slice(b"Hello, World!");
println!("Buffer: {:?}", String::from_utf8_lossy(&buf));
}
// Capacity is preserved across clears
println!("Capacity: {}", buf.capacity());
}Iteration and Conversion
use bytes::{Bytes, BytesMut};
fn main() {
let data = Bytes::from("Hello, World!");
// Iterate over bytes
for byte in &data {
print!("{} ", byte);
}
println!();
// Iterate with indices
for (i, byte) in data.iter().enumerate() {
println!("{}: {}", i, byte);
}
// Convert to Vec
let vec: Vec<u8> = data.to_vec();
println!("Vec: {:?}", vec);
// Convert to slice
let slice: &[u8] = &data;
println!("Slice: {:?}", slice);
// Convert BytesMut to Vec
let mut buf = BytesMut::from("Test");
let vec: Vec<u8> = buf.to_vec();
println!("From mut: {:?}", vec);
// Convert BytesMut to Bytes
let bytes = buf.freeze();
println!("Frozen: {:?}", bytes);
// Bytes to BytesMut
let bytes = Bytes::from("Test");
let mut buf = BytesMut::from(&bytes[..]);
println!("To mut: {:?}", buf);
// From various types
let from_vec = Bytes::from(vec![1, 2, 3]);
let from_slice = Bytes::copy_from_slice(&[4, 5, 6]);
let from_static = Bytes::from_static(b"static");
println!("From vec: {:?}", from_vec);
println!("From slice: {:?}", from_slice);
println!("From static: {:?}", from_static);
}
// Pattern matching with Bytes
fn pattern_matching() {
let data = Bytes::from("Hello");
// Check prefix
if data.starts_with(b"Hel") {
println!("Starts with 'Hel'");
}
// Check suffix
if data.ends_with(b"lo") {
println!("Ends with 'lo'");
}
}Working with Strings
use bytes::{Bytes, BytesMut};
fn main() {
// From String
let s = String::from("Hello, World!");
let bytes = Bytes::from(s);
println!("From String: {:?}", bytes);
// To String (lossy)
let bytes = Bytes::from("Hello, World!");
let s = String::from_utf8_lossy(&bytes);
println!("Lossy string: {}", s);
// To String (exact)
let bytes = Bytes::from("Hello, World!");
let s = String::from_utf8(bytes.to_vec()).unwrap();
println!("Exact string: {}", s);
// From &str
let bytes = Bytes::from("text");
println!("From &str: {:?}", bytes);
// Append string to BytesMut
let mut buf = BytesMut::new();
buf.extend_from_slice(b"Hello, ");
buf.extend_from_slice(b"World!");
let s = String::from_utf8_lossy(&buf);
println!("Built: {}", s);
}
// Parse string from bytes
fn parse_string() {
let bytes = Bytes::from("Hello, World!");
// Find delimiter
if let Some(pos) = bytes.iter().position(|&b| b == b',') {
let (first, rest) = bytes.split_at(pos);
println!("First: {:?}", String::from_utf8_lossy(&first));
println!("Rest: {:?}", String::from_utf8_lossy(&rest));
}
}Bytes in Async I/O
use bytes::{Bytes, BytesMut, Buf, BufMut};
// Simulating async read/write operations
struct AsyncStream {
buffer: BytesMut,
}
impl AsyncStream {
fn new() -> Self {
Self {
buffer: BytesMut::with_capacity(4096),
}
}
// Simulate reading data into buffer
fn read(&mut self, data: &[u8]) {
self.buffer.put_slice(data);
}
// Try to read a complete message
fn try_read_message(&mut self) -> Option<Bytes> {
// Message format: [length: 4 bytes][payload]
if self.buffer.len() < 4 {
return None;
}
// Peek length
let length = u32::from_be_bytes([
self.buffer[0],
self.buffer[1],
self.buffer[2],
self.buffer[3],
]) as usize;
// Check if complete message is available
if self.buffer.len() < 4 + length {
return None;
}
// Split off the message
self.buffer.advance(4); // Skip length
let message = self.buffer.split_to(length).freeze();
Some(message)
}
// Write a message
fn write_message(&self, payload: &[u8]) -> Bytes {
let mut buf = BytesMut::with_capacity(4 + payload.len());
buf.put_u32(payload.len() as u32);
buf.put_slice(payload);
buf.freeze()
}
}
fn main() {
let mut stream = AsyncStream::new();
// Write a message
let msg = stream.write_message(b"Hello, World!");
println!("Encoded message: {:?}", msg);
// Read into buffer
stream.read(&msg);
// Try to decode
if let Some(decoded) = stream.try_read_message() {
println!("Decoded: {:?}", String::from_utf8_lossy(&decoded));
}
// Partial read
let mut stream2 = AsyncStream::new();
stream2.read(&msg[..6]); // Only partial message
match stream2.try_read_message() {
Some(msg) => println!("Got: {:?}", msg),
None => println!("Incomplete message, waiting for more data"),
}
}
// Framing for network protocols
struct LineCodec {
buffer: BytesMut,
}
impl LineCodec {
fn new() -> Self {
Self {
buffer: BytesMut::with_capacity(8192),
}
}
fn feed(&mut self, data: &[u8]) {
self.buffer.extend_from_slice(data);
}
fn next_line(&mut self) -> Option<Bytes> {
// Find newline
let pos = self.buffer.iter().position(|&b| b == b'\n')?;
// Get line (without newline)
let line = self.buffer.split_to(pos).freeze();
self.buffer.advance(1); // Skip newline
Some(line)
}
}
fn line_codec_example() {
let mut codec = LineCodec::new();
// Feed partial data
codec.feed(b"Hello, ");
assert!(codec.next_line().is_none());
// Feed rest with newline
codec.feed(b"World!\nSecond line\n");
// Get lines
let line1 = codec.next_line().unwrap();
println!("Line 1: {:?}", String::from_utf8_lossy(&line1));
let line2 = codec.next_line().unwrap();
println!("Line 2: {:?}", String::from_utf8_lossy(&line2));
}Comparing and Ordering
use bytes::Bytes;
fn main() {
let a = Bytes::from("hello");
let b = Bytes::from("hello");
let c = Bytes::from("world");
// Equality
println!("a == b: {}", a == b);
println!("a == c: {}", a == c);
// Comparison
println!("a < c: {}", a < c);
// Compare with slices
println!("a == b"hello": {}", a == &b"hello"[..]);
println!("a == "hello": {}", a.as_ref() == b"hello");
// Hash (can use in HashMap/HashSet)
use std::collections::HashSet;
let mut set = HashSet::new();
set.insert(Bytes::from("key1"));
set.insert(Bytes::from("key2"));
set.insert(Bytes::from("key1")); // Duplicate
println!("Set size: {}", set.len()); // 2
// Order (can use in BTreeMap/BTreeSet)
use std::collections::BTreeMap;
let mut map = BTreeMap::new();
map.insert(Bytes::from("b"), 2);
map.insert(Bytes::from("a"), 1);
map.insert(Bytes::from("c"), 3);
for (k, v) in &map {
println!("{}: {}", String::from_utf8_lossy(k), v);
}
}Real-World Example: Binary Protocol
use bytes::{Bytes, BytesMut, Buf, BufMut};
#[derive(Debug, Clone)]
enum Message {
Ping,
Pong,
Data { id: u32, payload: Bytes },
Error { code: u16, message: String },
}
impl Message {
// Encode message to bytes
fn encode(&self) -> Bytes {
let mut buf = BytesMut::new();
match self {
Message::Ping => {
buf.put_u8(0x01); // Message type
}
Message::Pong => {
buf.put_u8(0x02);
}
Message::Data { id, payload } => {
buf.put_u8(0x03);
buf.put_u32(*id);
buf.put_u32(payload.len() as u32);
buf.put(payload.clone());
}
Message::Error { code, message } => {
buf.put_u8(0x04);
buf.put_u16(*code);
buf.put_u16(message.len() as u16);
buf.put_slice(message.as_bytes());
}
}
buf.freeze()
}
// Decode message from bytes
fn decode(mut buf: Bytes) -> Result<Self, &'static str> {
if buf.is_empty() {
return Err("empty buffer");
}
let msg_type = buf.get_u8();
match msg_type {
0x01 => Ok(Message::Ping),
0x02 => Ok(Message::Pong),
0x03 => {
if buf.remaining() < 8 {
return Err("incomplete data message");
}
let id = buf.get_u32();
let len = buf.get_u32() as usize;
if buf.remaining() < len {
return Err("incomplete payload");
}
let payload = buf.copy_to_bytes(len);
Ok(Message::Data { id, payload })
}
0x04 => {
if buf.remaining() < 4 {
return Err("incomplete error message");
}
let code = buf.get_u16();
let len = buf.get_u16() as usize;
if buf.remaining() < len {
return Err("incomplete error message string");
}
let message_bytes = buf.copy_to_bytes(len);
let message = String::from_utf8_lossy(&message_bytes).to_string();
Ok(Message::Error { code, message })
}
_ => Err("unknown message type"),
}
}
}
fn main() {
// Encode and decode Ping
let ping = Message::Ping;
let encoded = ping.encode();
let decoded = Message::decode(encoded).unwrap();
println!("Decoded: {:?}", decoded);
// Encode and decode Data
let data = Message::Data {
id: 42,
payload: Bytes::from("Hello, World!"),
};
let encoded = data.encode();
println!("Encoded data: {:?}", encoded);
let decoded = Message::decode(encoded).unwrap();
println!("Decoded: {:?}", decoded);
// Encode and decode Error
let error = Message::Error {
code: 404,
message: "Not Found".to_string(),
};
let encoded = error.encode();
let decoded = Message::decode(encoded).unwrap();
println!("Decoded: {:?}", decoded);
// Protocol usage
let messages = vec![
Message::Ping,
Message::Data { id: 1, payload: Bytes::from("test") },
Message::Error { code: 500, message: "Server error".to_string() },
Message::Pong,
];
// Encode all messages into one buffer
let mut combined = BytesMut::new();
for msg in &messages {
let encoded = msg.encode();
combined.put(encoded);
}
println!("Combined buffer length: {}", combined.len());
// Decode messages
let mut buffer = combined.freeze();
while !buffer.is_empty() {
// Need to determine message length first
let msg_type = buffer[0];
let msg_len = match msg_type {
0x01 | 0x02 => 1,
0x03 => {
let id_len = 1 + 4 + 4;
let payload_len = u32::from_be_bytes([
buffer[5], buffer[6], buffer[7], buffer[8]
]);
id_len + payload_len as usize
}
0x04 => {
let header_len = 1 + 2 + 2;
let msg_len = u16::from_be_bytes([buffer[3], buffer[4]]);
header_len + msg_len as usize
}
_ => break,
};
let msg_bytes = buffer.split_to(msg_len);
match Message::decode(msg_bytes) {
Ok(msg) => println!("Decoded: {:?}", msg),
Err(e) => println!("Error: {}", e),
}
}
}Summary
- Use
Bytesfor immutable, reference-counted byte buffers - Use
BytesMutfor building byte buffers incrementally - Clone
Bytesis O(1) — shares underlying data via reference counting - Slice
Bytesis zero-copy — creates a view into existing data - Use
Bytes::from_static(b"...")for static byte data (no allocation) - Use
BytesMut::freeze()to convert mutable to immutableBytes - Use
Buftrait methods (get_u8,get_u32, etc.) for reading binary data - Use
BufMuttrait methods (put_u8,put_u32, etc.) for writing binary data - Big-endian is default; use
_lesuffix for little-endian (get_u32_le,put_u32_le) - Use
slice(range)for zero-copy slicing - Use
split_at(n)to split at position into twoBytes - Use
split_to(n)onBytesMutto split off front - Use
split_off(n)onBytesMutto split off back - Use
reserve(n)to pre-allocate capacity inBytesMut - Use
chunk()to access the contiguous slice of remaining bytes - Use
advance(n)to skip bytes in aBuf - Use
copy_to_bytes(n)to extract bytes and advance position - Ideal for network protocols, parsing, and zero-copy scenarios
- Integrates seamlessly with Tokio, hyper, and async I/O