I/O Optimization

Async I/O, streaming, and efficient data processing.

Overview

Master I/O optimization patterns.

Async File Operations

import asyncio
import aiofiles

async def async_read(filename):
    async with aiofiles.open(filename, 'r') as f:
        content = await f.read()
    return content

async def async_write(filename, content):
    async with aiofiles.open(filename, 'w') as f:
        await f.write(content)

async def main():
    content = await async_read('large_file.txt')
    print(f"Read {len(content)} characters")

asyncio.run(main())

Streaming Processing

def stream_lines(filename):
    with open(filename) as f:
        for line in f:
            yield line.strip()

def process_large_file(filename):
    for line in stream_lines(filename):
        if 'error' in line.lower():
            yield line

# Memory efficient
errors = list(process_large_file('huge_log.txt'))

Buffered I/O

from io import BytesIO

def write_large_data(filename, data_chunks):
    with open(filename, 'wb') as f:
        buffer = BytesIO()
        for chunk in data_chunks:
            buffer.write(chunk)
            if buffer.tell() > 8192:  # 8KB buffer
                f.write(buffer.getvalue())
                buffer = BytesIO()
        f.write(buffer.getvalue())

# Practice

Practice

Build a streaming data processor for large files.