Generator Patterns

Generator pipelines, coroutines, and advanced generator patterns.

Overview

Master generator patterns.

Generator Pipelines

def read_lines(filename):
    with open(filename) as f:
        for line in f:
            yield line.strip()

def filter_comments(lines):
    for line in lines:
        if not line.startswith('#'):
            yield line

def parse_csv(lines):
    for line in lines:
        yield line.split(',')

def extract_names(rows):
    for row in rows:
        if len(row) >= 2:
            yield row[1]

# Pipeline
pipeline = extract_names(
    parse_csv(
        filter_comments(
            read_lines('data.csv')
        )
    )
)

for name in pipeline:
    print(name)

Coroutine Pipelines

def coroutine(func):
    def wrapper(*args, **kwargs):
        gen = func(*args, **kwargs)
        next(gen)
        return gen
    return wrapper

@coroutine
def averager():
    total = 0.0
    count = 0
    average = None
    while True:
        term = yield
        if term is None:
            break
        total += term
        count += 1
        average = total / count
    return (count, average)

avg = averager()
avg.send(10)
avg.send(20)
avg.send(30)
try:
    avg.send(None)
except StopIteration as e:
    print(e.value)  # (3, 20.0)

Practice

Implement a generator pipeline for log file analysis.