Concurrency Patterns

Producer-consumer, worker pools, and async patterns.

Overview

Master concurrency patterns.

Producer-Consumer

import asyncio
import random

async def producer(queue):
    for i in range(10):
        item = f"item_{i}"
        await queue.put(item)
        print(f"Produced: {item}")
        await asyncio.sleep(random.uniform(0.1, 0.5))
    await queue.put(None)

async def consumer(queue):
    while True:
        item = await queue.get()
        if item is None:
            break
        print(f"Consumed: {item}")
        await asyncio.sleep(random.uniform(0.1, 0.3))
        queue.task_done()

async def main():
    queue = asyncio.Queue()
    producer_task = asyncio.create_task(producer(queue))
    consumer_task = asyncio.create_task(consumer(queue))
    
    await producer_task
    await consumer_task

asyncio.run(main())

Worker Pool

from concurrent.futures import ProcessPoolExecutor
import os

def worker(task):
    return f"Processed {task}"

def main():
    tasks = range(10)
    
    with ProcessPoolExecutor(max_workers=os.cpu_count()) as executor:
        results = list(executor.map(worker, tasks))
    
    print(results)

if __name__ == "__main__":
    main()

Practice

Build a web crawler using async producer-consumer pattern.