A simple for loop over Order.objects.all() works fine until it meets 5 million rows. Then you get memory spikes, timeouts, a slow database, and no idea where the job stopped.

Here is the pattern that turns that into a restartable pipeline.

The core idea

Don't run one big job. Run thousands of small, independent Celery tasks, each handling one ID range.

CHUNK_SIZE = 2000

@shared_task def dispatch_order_recalculation (): bounds = Order . objects . aggregate ( lo = Min ( " id " ), hi = Max ( " id " )) if bounds [ " lo " ] is None : return 0 start = bounds [ " lo " ] while start <= bounds [ " hi " ]: end = start + CHUNK_SIZE - 1 process_order_chunk . delay ( start , end ) start = end + 1

@shared_task ( bind = True , max_retries = 3 , acks_late = True ) def process_order_chunk ( self , start_id , end_id ): try : orders = list ( Order . objects . filter ( id__gte = start_id , id__lte = end_id ) . only ( " id " , " subtotal " , " tax_rate " , " discount " ) ) for order in orders : order .…