Advanced Generators

Generator pipelines, yield from, send(), and advanced patterns.

Overview

Master advanced generator techniques.

Generator Pipelines

def read_data(filename):
    with open(filename) as f:
        for line in f:
            yield line.strip()

def filter_comments(lines):
    for line in lines:
        if not line.startswith('#'):
            yield line

def parse_csv(lines):
    for line in lines:
        yield line.split(',')

def transform(data):
    for row in data:
        yield {
            'name': row[0],
            'value': int(row[1])
        }

# Pipeline
pipeline = transform(
    filter_comments(
        read_data('data.txt')
    )
)

for item in pipeline:
    print(item)

yield from

def flatten(nested):
    for item in nested:
        if isinstance(item, (list, tuple)):
            yield from flatten(item)
        else:
            yield item

nested = [1, [2, 3], [4, [5, 6]], 7]
print(list(flatten(nested)))  # [1, 2, 3, 4, 5, 6, 7]

# Chain multiple generators
def generator1():
    yield 1
    yield 2

def generator2():
    yield 3
    yield 4

def combined():
    yield from generator1()
    yield from generator2()

print(list(combined()))  # [1, 2, 3, 4]

send() and throw()

def accumulator():
    total = 0
    while True:
        value = yield total
        if value is None:
            break
        total += value

acc = accumulator()
next(acc)  # Initialize
print(acc.send(10))   # 10
print(acc.send(20))   # 30
print(acc.send(5))    # 35

# throw() to send exceptions
def controlled_generator():
    while True:
        try:
            value = yield
            print(f"Received: {value}")
        except ValueError:
            print("Invalid value!")
        except GeneratorExit:
            print("Closing generator")
            break

gen = controlled_generator()
next(gen)
gen.send("hello")     # Received: hello
gen.throw(ValueError) # Invalid value!
gen.close()           # Closing generator

Practice

Create a generator pipeline that reads a CSV file and filters/transforms data.