Advanced I/O Patterns
Streaming, buffering, memory mapping, and I/O optimization.
Overview
Master advanced I/O techniques.
Streaming
python
def stream_large_file(filename, chunk_size=8192):
with open(filename, 'rb') as f:
while True:
chunk = f.read(chunk_size)
if not chunk:
break
yield chunk
# Process large file without loading into memory
total_size = 0
for chunk in stream_large_file('large_file.bin'):
total_size += len(chunk)
print(f"Total size: {total_size} bytes")
Buffered I/O
python
from io import BytesIO, StringIO
# In-memory file
buffer = BytesIO()
buffer.write(b"Hello, ")
buffer.write(b"World!")
buffer.seek(0)
print(buffer.read()) # b'Hello, World!'
# String buffer
text_buffer = StringIO()
text_buffer.write("Hello, ")
text_buffer.write("World!")
text_buffer.seek(0)
print(text_buffer.read()) # Hello, World!
Memory Mapping
python
import mmap
def read_mmap(filename):
with open(filename, 'r+b') as f:
mm = mmap.mmap(f.fileno(), 0)
data = mm.read()
mm.close()
return data
# Efficient random access
def access_mmap(filename, offset, size):
with open(filename, 'r+b') as f:
mm = mmap.mmap(f.fileno(), 0)
mm.seek(offset)
data = mm.read(size)
mm.close()
return data
Practice
Implement a file processor that handles large files efficiently.