Advanced I/O Patterns

Streaming, buffering, memory mapping, and I/O optimization.

Overview

Master advanced I/O techniques.

Streaming

def stream_large_file(filename, chunk_size=8192):
    with open(filename, 'rb') as f:
        while True:
            chunk = f.read(chunk_size)
            if not chunk:
                break
            yield chunk

# Process large file without loading into memory
total_size = 0
for chunk in stream_large_file('large_file.bin'):
    total_size += len(chunk)
print(f"Total size: {total_size} bytes")

Buffered I/O

from io import BytesIO, StringIO

# In-memory file
buffer = BytesIO()
buffer.write(b"Hello, ")
buffer.write(b"World!")
buffer.seek(0)
print(buffer.read())  # b'Hello, World!'

# String buffer
text_buffer = StringIO()
text_buffer.write("Hello, ")
text_buffer.write("World!")
text_buffer.seek(0)
print(text_buffer.read())  # Hello, World!

Memory Mapping

import mmap

def read_mmap(filename):
    with open(filename, 'r+b') as f:
        mm = mmap.mmap(f.fileno(), 0)
        data = mm.read()
        mm.close()
        return data

# Efficient random access
def access_mmap(filename, offset, size):
    with open(filename, 'r+b') as f:
        mm = mmap.mmap(f.fileno(), 0)
        mm.seek(offset)
        data = mm.read(size)
        mm.close()
        return data

Practice

Implement a file processor that handles large files efficiently.