Grouping items in a list Python [duplicate] - python

I have a Python script which takes as input a list of integers, which I need to work with four integers at a time. Unfortunately, I don't have control of the input, or I'd have it passed in as a list of four-element tuples. Currently, I'm iterating over it this way:
for i in range(0, len(ints), 4):
# dummy op for example code
foo += ints[i] * ints[i + 1] + ints[i + 2] * ints[i + 3]
It looks a lot like "C-think", though, which makes me suspect there's a more pythonic way of dealing with this situation. The list is discarded after iterating, so it needn't be preserved. Perhaps something like this would be better?
while ints:
foo += ints[0] * ints[1] + ints[2] * ints[3]
ints[0:4] = []
Still doesn't quite "feel" right, though. :-/
Related question: How do you split a list into evenly sized chunks in Python?

def chunker(seq, size):
return (seq[pos:pos + size] for pos in range(0, len(seq), size))
Works with any sequence:
text = "I am a very, very helpful text"
for group in chunker(text, 7):
print(repr(group),)
# 'I am a ' 'very, v' 'ery hel' 'pful te' 'xt'
print('|'.join(chunker(text, 10)))
# I am a ver|y, very he|lpful text
animals = ['cat', 'dog', 'rabbit', 'duck', 'bird', 'cow', 'gnu', 'fish']
for group in chunker(animals, 3):
print(group)
# ['cat', 'dog', 'rabbit']
# ['duck', 'bird', 'cow']
# ['gnu', 'fish']

Modified from the Recipes section of Python's itertools docs:
from itertools import zip_longest
def grouper(iterable, n, fillvalue=None):
args = [iter(iterable)] * n
return zip_longest(*args, fillvalue=fillvalue)
Example
grouper('ABCDEFG', 3, 'x') # --> 'ABC' 'DEF' 'Gxx'
Note: on Python 2 use izip_longest instead of zip_longest.

chunk_size = 4
for i in range(0, len(ints), chunk_size):
chunk = ints[i:i+chunk_size]
# process chunk of size <= chunk_size

import itertools
def chunks(iterable,size):
it = iter(iterable)
chunk = tuple(itertools.islice(it,size))
while chunk:
yield chunk
chunk = tuple(itertools.islice(it,size))
# though this will throw ValueError if the length of ints
# isn't a multiple of four:
for x1,x2,x3,x4 in chunks(ints,4):
foo += x1 + x2 + x3 + x4
for chunk in chunks(ints,4):
foo += sum(chunk)
Another way:
import itertools
def chunks2(iterable,size,filler=None):
it = itertools.chain(iterable,itertools.repeat(filler,size-1))
chunk = tuple(itertools.islice(it,size))
while len(chunk) == size:
yield chunk
chunk = tuple(itertools.islice(it,size))
# x2, x3 and x4 could get the value 0 if the length is not
# a multiple of 4.
for x1,x2,x3,x4 in chunks2(ints,4,0):
foo += x1 + x2 + x3 + x4

If you don't mind using an external package you could use iteration_utilities.grouper from iteration_utilties 1. It supports all iterables (not just sequences):
from iteration_utilities import grouper
seq = list(range(20))
for group in grouper(seq, 4):
print(group)
which prints:
(0, 1, 2, 3)
(4, 5, 6, 7)
(8, 9, 10, 11)
(12, 13, 14, 15)
(16, 17, 18, 19)
In case the length isn't a multiple of the groupsize it also supports filling (the incomplete last group) or truncating (discarding the incomplete last group) the last one:
from iteration_utilities import grouper
seq = list(range(17))
for group in grouper(seq, 4):
print(group)
# (0, 1, 2, 3)
# (4, 5, 6, 7)
# (8, 9, 10, 11)
# (12, 13, 14, 15)
# (16,)
for group in grouper(seq, 4, fillvalue=None):
print(group)
# (0, 1, 2, 3)
# (4, 5, 6, 7)
# (8, 9, 10, 11)
# (12, 13, 14, 15)
# (16, None, None, None)
for group in grouper(seq, 4, truncate=True):
print(group)
# (0, 1, 2, 3)
# (4, 5, 6, 7)
# (8, 9, 10, 11)
# (12, 13, 14, 15)
Benchmarks
I also decided to compare the run-time of a few of the mentioned approaches. It's a log-log plot grouping into groups of "10" elements based on a list of varying size. For qualitative results: Lower means faster:
At least in this benchmark the iteration_utilities.grouper performs best. Followed by the approach of Craz.
The benchmark was created with simple_benchmark1. The code used to run this benchmark was:
import iteration_utilities
import itertools
from itertools import zip_longest
def consume_all(it):
return iteration_utilities.consume(it, None)
import simple_benchmark
b = simple_benchmark.BenchmarkBuilder()
#b.add_function()
def grouper(l, n):
return consume_all(iteration_utilities.grouper(l, n))
def Craz_inner(iterable, n, fillvalue=None):
args = [iter(iterable)] * n
return zip_longest(*args, fillvalue=fillvalue)
#b.add_function()
def Craz(iterable, n, fillvalue=None):
return consume_all(Craz_inner(iterable, n, fillvalue))
def nosklo_inner(seq, size):
return (seq[pos:pos + size] for pos in range(0, len(seq), size))
#b.add_function()
def nosklo(seq, size):
return consume_all(nosklo_inner(seq, size))
def SLott_inner(ints, chunk_size):
for i in range(0, len(ints), chunk_size):
yield ints[i:i+chunk_size]
#b.add_function()
def SLott(ints, chunk_size):
return consume_all(SLott_inner(ints, chunk_size))
def MarkusJarderot1_inner(iterable,size):
it = iter(iterable)
chunk = tuple(itertools.islice(it,size))
while chunk:
yield chunk
chunk = tuple(itertools.islice(it,size))
#b.add_function()
def MarkusJarderot1(iterable,size):
return consume_all(MarkusJarderot1_inner(iterable,size))
def MarkusJarderot2_inner(iterable,size,filler=None):
it = itertools.chain(iterable,itertools.repeat(filler,size-1))
chunk = tuple(itertools.islice(it,size))
while len(chunk) == size:
yield chunk
chunk = tuple(itertools.islice(it,size))
#b.add_function()
def MarkusJarderot2(iterable,size):
return consume_all(MarkusJarderot2_inner(iterable,size))
#b.add_arguments()
def argument_provider():
for exp in range(2, 20):
size = 2**exp
yield size, simple_benchmark.MultiArgument([[0] * size, 10])
r = b.run()
1 Disclaimer: I'm the author of the libraries iteration_utilities and simple_benchmark.

With Python 3.8 you can use the walrus operator and itertools.islice.
from itertools import islice
list_ = [i for i in range(10, 100)]
def chunker(it, size):
iterator = iter(it)
while chunk := list(islice(iterator, size)):
print(chunk)
In [2]: chunker(list_, 10)
[10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
[20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
[30, 31, 32, 33, 34, 35, 36, 37, 38, 39]
[40, 41, 42, 43, 44, 45, 46, 47, 48, 49]
[50, 51, 52, 53, 54, 55, 56, 57, 58, 59]
[60, 61, 62, 63, 64, 65, 66, 67, 68, 69]
[70, 71, 72, 73, 74, 75, 76, 77, 78, 79]
[80, 81, 82, 83, 84, 85, 86, 87, 88, 89]
[90, 91, 92, 93, 94, 95, 96, 97, 98, 99]

I needed a solution that would also work with sets and generators. I couldn't come up with anything very short and pretty, but it's quite readable at least.
def chunker(seq, size):
res = []
for el in seq:
res.append(el)
if len(res) == size:
yield res
res = []
if res:
yield res
List:
>>> list(chunker([i for i in range(10)], 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
Set:
>>> list(chunker(set([i for i in range(10)]), 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
Generator:
>>> list(chunker((i for i in range(10)), 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]

The more-itertools package has chunked method which does exactly that:
import more_itertools
for s in more_itertools.chunked(range(9), 4):
print(s)
Prints
[0, 1, 2, 3]
[4, 5, 6, 7]
[8]
chunked returns the items in a list. If you'd prefer iterables, use ichunked.

The ideal solution for this problem works with iterators (not just sequences). It should also be fast.
This is the solution provided by the documentation for itertools:
def grouper(n, iterable, fillvalue=None):
#"grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
args = [iter(iterable)] * n
return itertools.izip_longest(fillvalue=fillvalue, *args)
Using ipython's %timeit on my mac book air, I get 47.5 us per loop.
However, this really doesn't work for me since the results are padded to be even sized groups. A solution without the padding is slightly more complicated. The most naive solution might be:
def grouper(size, iterable):
i = iter(iterable)
while True:
out = []
try:
for _ in range(size):
out.append(i.next())
except StopIteration:
yield out
break
yield out
Simple, but pretty slow: 693 us per loop
The best solution I could come up with uses islice for the inner loop:
def grouper(size, iterable):
it = iter(iterable)
while True:
group = tuple(itertools.islice(it, None, size))
if not group:
break
yield group
With the same dataset, I get 305 us per loop.
Unable to get a pure solution any faster than that, I provide the following solution with an important caveat: If your input data has instances of filldata in it, you could get wrong answer.
def grouper(n, iterable, fillvalue=None):
#"grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
args = [iter(iterable)] * n
# itertools.zip_longest on Python 3
for x in itertools.izip_longest(*args, fillvalue=fillvalue):
if x[-1] is fillvalue:
yield tuple(v for v in x if v is not fillvalue)
else:
yield x
I really don't like this answer, but it is significantly faster. 124 us per loop

from itertools import izip_longest
def chunker(iterable, chunksize, filler):
return izip_longest(*[iter(iterable)]*chunksize, fillvalue=filler)

Similar to other proposals, but not exactly identical, I like doing it this way, because it's simple and easy to read:
it = iter([1, 2, 3, 4, 5, 6, 7, 8, 9])
for chunk in zip(it, it, it, it):
print chunk
>>> (1, 2, 3, 4)
>>> (5, 6, 7, 8)
This way you won't get the last partial chunk. If you want to get (9, None, None, None) as last chunk, just use izip_longest from itertools.

Since nobody's mentioned it yet here's a zip() solution:
>>> def chunker(iterable, chunksize):
... return zip(*[iter(iterable)]*chunksize)
It works only if your sequence's length is always divisible by the chunk size or you don't care about a trailing chunk if it isn't.
Example:
>>> s = '1234567890'
>>> chunker(s, 3)
[('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9')]
>>> chunker(s, 4)
[('1', '2', '3', '4'), ('5', '6', '7', '8')]
>>> chunker(s, 5)
[('1', '2', '3', '4', '5'), ('6', '7', '8', '9', '0')]
Or using itertools.izip to return an iterator instead of a list:
>>> from itertools import izip
>>> def chunker(iterable, chunksize):
... return izip(*[iter(iterable)]*chunksize)
Padding can be fixed using #ΤΖΩΤΖΙΟΥ's answer:
>>> from itertools import chain, izip, repeat
>>> def chunker(iterable, chunksize, fillvalue=None):
... it = chain(iterable, repeat(fillvalue, chunksize-1))
... args = [it] * chunksize
... return izip(*args)

Another approach would be to use the two-argument form of iter:
from itertools import islice
def group(it, size):
it = iter(it)
return iter(lambda: tuple(islice(it, size)), ())
This can be adapted easily to use padding (this is similar to Markus Jarderot’s answer):
from itertools import islice, chain, repeat
def group_pad(it, size, pad=None):
it = chain(iter(it), repeat(pad))
return iter(lambda: tuple(islice(it, size)), (pad,) * size)
These can even be combined for optional padding:
_no_pad = object()
def group(it, size, pad=_no_pad):
if pad == _no_pad:
it = iter(it)
sentinel = ()
else:
it = chain(iter(it), repeat(pad))
sentinel = (pad,) * size
return iter(lambda: tuple(islice(it, size)), sentinel)

If the list is large, the highest-performing way to do this will be to use a generator:
def get_chunk(iterable, chunk_size):
result = []
for item in iterable:
result.append(item)
if len(result) == chunk_size:
yield tuple(result)
result = []
if len(result) > 0:
yield tuple(result)
for x in get_chunk([1,2,3,4,5,6,7,8,9,10], 3):
print x
(1, 2, 3)
(4, 5, 6)
(7, 8, 9)
(10,)

Using little functions and things really doesn't appeal to me; I prefer to just use slices:
data = [...]
chunk_size = 10000 # or whatever
chunks = [data[i:i+chunk_size] for i in xrange(0,len(data),chunk_size)]
for chunk in chunks:
...

Using map() instead of zip() fixes the padding issue in J.F. Sebastian's answer:
>>> def chunker(iterable, chunksize):
... return map(None,*[iter(iterable)]*chunksize)
Example:
>>> s = '1234567890'
>>> chunker(s, 3)
[('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9'), ('0', None, None)]
>>> chunker(s, 4)
[('1', '2', '3', '4'), ('5', '6', '7', '8'), ('9', '0', None, None)]
>>> chunker(s, 5)
[('1', '2', '3', '4', '5'), ('6', '7', '8', '9', '0')]

One-liner, adhoc solution to iterate over a list x in chunks of size 4 -
for a, b, c, d in zip(x[0::4], x[1::4], x[2::4], x[3::4]):
... do something with a, b, c and d ...

To avoid all conversions to a list import itertools and:
>>> for k, g in itertools.groupby(xrange(35), lambda x: x/10):
... list(g)
Produces:
...
0 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
1 [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
2 [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
3 [30, 31, 32, 33, 34]
>>>
I checked groupby and it doesn't convert to list or use len so I (think) this will delay resolution of each value until it is actually used. Sadly none of the available answers (at this time) seemed to offer this variation.
Obviously if you need to handle each item in turn nest a for loop over g:
for k,g in itertools.groupby(xrange(35), lambda x: x/10):
for i in g:
# do what you need to do with individual items
# now do what you need to do with the whole group
My specific interest in this was the need to consume a generator to submit changes in batches of up to 1000 to the gmail API:
messages = a_generator_which_would_not_be_smart_as_a_list
for idx, batch in groupby(messages, lambda x: x/1000):
batch_request = BatchHttpRequest()
for message in batch:
batch_request.add(self.service.users().messages().modify(userId='me', id=message['id'], body=msg_labels))
http = httplib2.Http()
self.credentials.authorize(http)
batch_request.execute(http=http)

Unless I misses something, the following simple solution with generator expressions has not been mentioned. It assumes that both the size and the number of chunks are known (which is often the case), and that no padding is required:
def chunks(it, n, m):
"""Make an iterator over m first chunks of size n.
"""
it = iter(it)
# Chunks are presented as tuples.
return (tuple(next(it) for _ in range(n)) for _ in range(m))

In your second method, I would advance to the next group of 4 by doing this:
ints = ints[4:]
However, I haven't done any performance measurement so I don't know which one might be more efficient.
Having said that, I would usually choose the first method. It's not pretty, but that's often a consequence of interfacing with the outside world.

With NumPy it's simple:
ints = array([1, 2, 3, 4, 5, 6, 7, 8])
for int1, int2 in ints.reshape(-1, 2):
print(int1, int2)
output:
1 2
3 4
5 6
7 8

I never want my chunks padded, so that requirement is essential. I find that the ability to work on any iterable is also requirement. Given that, I decided to extend on the accepted answer, https://stackoverflow.com/a/434411/1074659.
Performance takes a slight hit in this approach if padding is not wanted due to the need to compare and filter the padded values. However, for large chunk sizes, this utility is very performant.
#!/usr/bin/env python3
from itertools import zip_longest
_UNDEFINED = object()
def chunker(iterable, chunksize, fillvalue=_UNDEFINED):
"""
Collect data into chunks and optionally pad it.
Performance worsens as `chunksize` approaches 1.
Inspired by:
https://docs.python.org/3/library/itertools.html#itertools-recipes
"""
args = [iter(iterable)] * chunksize
chunks = zip_longest(*args, fillvalue=fillvalue)
yield from (
filter(lambda val: val is not _UNDEFINED, chunk)
if chunk[-1] is _UNDEFINED
else chunk
for chunk in chunks
) if fillvalue is _UNDEFINED else chunks

def chunker(iterable, n):
"""Yield iterable in chunk sizes.
>>> chunks = chunker('ABCDEF', n=4)
>>> chunks.next()
['A', 'B', 'C', 'D']
>>> chunks.next()
['E', 'F']
"""
it = iter(iterable)
while True:
chunk = []
for i in range(n):
try:
chunk.append(next(it))
except StopIteration:
yield chunk
raise StopIteration
yield chunk
if __name__ == '__main__':
import doctest
doctest.testmod()

Yet another answer, the advantages of which are:
1) Easily understandable
2) Works on any iterable, not just sequences (some of the above answers will choke on filehandles)
3) Does not load the chunk into memory all at once
4) Does not make a chunk-long list of references to the same iterator in memory
5) No padding of fill values at the end of the list
That being said, I haven't timed it so it might be slower than some of the more clever methods, and some of the advantages may be irrelevant given the use case.
def chunkiter(iterable, size):
def inneriter(first, iterator, size):
yield first
for _ in xrange(size - 1):
yield iterator.next()
it = iter(iterable)
while True:
yield inneriter(it.next(), it, size)
In [2]: i = chunkiter('abcdefgh', 3)
In [3]: for ii in i:
for c in ii:
print c,
print ''
...:
a b c
d e f
g h
Update:
A couple of drawbacks due to the fact the inner and outer loops are pulling values from the same iterator:
1) continue doesn't work as expected in the outer loop - it just continues on to the next item rather than skipping a chunk. However, this doesn't seem like a problem as there's nothing to test in the outer loop.
2) break doesn't work as expected in the inner loop - control will wind up in the inner loop again with the next item in the iterator. To skip whole chunks, either wrap the inner iterator (ii above) in a tuple, e.g. for c in tuple(ii), or set a flag and exhaust the iterator.

def group_by(iterable, size):
"""Group an iterable into lists that don't exceed the size given.
>>> group_by([1,2,3,4,5], 2)
[[1, 2], [3, 4], [5]]
"""
sublist = []
for index, item in enumerate(iterable):
if index > 0 and index % size == 0:
yield sublist
sublist = []
sublist.append(item)
if sublist:
yield sublist

You can use partition or chunks function from funcy library:
from funcy import partition
for a, b, c, d in partition(4, ints):
foo += a * b * c * d
These functions also has iterator versions ipartition and ichunks, which will be more efficient in this case.
You can also peek at their implementation.

About solution gave by J.F. Sebastian here:
def chunker(iterable, chunksize):
return zip(*[iter(iterable)]*chunksize)
It's clever, but has one disadvantage - always return tuple. How to get string instead?
Of course you can write ''.join(chunker(...)), but the temporary tuple is constructed anyway.
You can get rid of the temporary tuple by writing own zip, like this:
class IteratorExhausted(Exception):
pass
def translate_StopIteration(iterable, to=IteratorExhausted):
for i in iterable:
yield i
raise to # StopIteration would get ignored because this is generator,
# but custom exception can leave the generator.
def custom_zip(*iterables, reductor=tuple):
iterators = tuple(map(translate_StopIteration, iterables))
while True:
try:
yield reductor(next(i) for i in iterators)
except IteratorExhausted: # when any of iterators get exhausted.
break
Then
def chunker(data, size, reductor=tuple):
return custom_zip(*[iter(data)]*size, reductor=reductor)
Example usage:
>>> for i in chunker('12345', 2):
... print(repr(i))
...
('1', '2')
('3', '4')
>>> for i in chunker('12345', 2, ''.join):
... print(repr(i))
...
'12'
'34'

I like this approach. It feels simple and not magical and supports all iterable types and doesn't require imports.
def chunk_iter(iterable, chunk_size):
it = iter(iterable)
while True:
chunk = tuple(next(it) for _ in range(chunk_size))
if not chunk:
break
yield chunk

Quite pythonic here (you may also inline the body of the split_groups function)
import itertools
def split_groups(iter_in, group_size):
return ((x for _, x in item) for _, item in itertools.groupby(enumerate(iter_in), key=lambda x: x[0] // group_size))
for x, y, z, w in split_groups(range(16), 4):
foo += x * y + z * w

Here is a chunker without imports that supports generators:
def chunks(seq, size):
it = iter(seq)
while True:
ret = tuple(next(it) for _ in range(size))
if len(ret) == size:
yield ret
else:
raise StopIteration()
Example of use:
>>> def foo():
... i = 0
... while True:
... i += 1
... yield i
...
>>> c = chunks(foo(), 3)
>>> c.next()
(1, 2, 3)
>>> c.next()
(4, 5, 6)
>>> list(chunks('abcdefg', 2))
[('a', 'b'), ('c', 'd'), ('e', 'f')]

Related

Is there a way to select a block of n elements from a list? [duplicate]

I have a Python script which takes as input a list of integers, which I need to work with four integers at a time. Unfortunately, I don't have control of the input, or I'd have it passed in as a list of four-element tuples. Currently, I'm iterating over it this way:
for i in range(0, len(ints), 4):
# dummy op for example code
foo += ints[i] * ints[i + 1] + ints[i + 2] * ints[i + 3]
It looks a lot like "C-think", though, which makes me suspect there's a more pythonic way of dealing with this situation. The list is discarded after iterating, so it needn't be preserved. Perhaps something like this would be better?
while ints:
foo += ints[0] * ints[1] + ints[2] * ints[3]
ints[0:4] = []
Still doesn't quite "feel" right, though. :-/
Related question: How do you split a list into evenly sized chunks in Python?
def chunker(seq, size):
return (seq[pos:pos + size] for pos in range(0, len(seq), size))
Works with any sequence:
text = "I am a very, very helpful text"
for group in chunker(text, 7):
print(repr(group),)
# 'I am a ' 'very, v' 'ery hel' 'pful te' 'xt'
print('|'.join(chunker(text, 10)))
# I am a ver|y, very he|lpful text
animals = ['cat', 'dog', 'rabbit', 'duck', 'bird', 'cow', 'gnu', 'fish']
for group in chunker(animals, 3):
print(group)
# ['cat', 'dog', 'rabbit']
# ['duck', 'bird', 'cow']
# ['gnu', 'fish']
Modified from the Recipes section of Python's itertools docs:
from itertools import zip_longest
def grouper(iterable, n, fillvalue=None):
args = [iter(iterable)] * n
return zip_longest(*args, fillvalue=fillvalue)
Example
grouper('ABCDEFG', 3, 'x') # --> 'ABC' 'DEF' 'Gxx'
Note: on Python 2 use izip_longest instead of zip_longest.
chunk_size = 4
for i in range(0, len(ints), chunk_size):
chunk = ints[i:i+chunk_size]
# process chunk of size <= chunk_size
import itertools
def chunks(iterable,size):
it = iter(iterable)
chunk = tuple(itertools.islice(it,size))
while chunk:
yield chunk
chunk = tuple(itertools.islice(it,size))
# though this will throw ValueError if the length of ints
# isn't a multiple of four:
for x1,x2,x3,x4 in chunks(ints,4):
foo += x1 + x2 + x3 + x4
for chunk in chunks(ints,4):
foo += sum(chunk)
Another way:
import itertools
def chunks2(iterable,size,filler=None):
it = itertools.chain(iterable,itertools.repeat(filler,size-1))
chunk = tuple(itertools.islice(it,size))
while len(chunk) == size:
yield chunk
chunk = tuple(itertools.islice(it,size))
# x2, x3 and x4 could get the value 0 if the length is not
# a multiple of 4.
for x1,x2,x3,x4 in chunks2(ints,4,0):
foo += x1 + x2 + x3 + x4
If you don't mind using an external package you could use iteration_utilities.grouper from iteration_utilties 1. It supports all iterables (not just sequences):
from iteration_utilities import grouper
seq = list(range(20))
for group in grouper(seq, 4):
print(group)
which prints:
(0, 1, 2, 3)
(4, 5, 6, 7)
(8, 9, 10, 11)
(12, 13, 14, 15)
(16, 17, 18, 19)
In case the length isn't a multiple of the groupsize it also supports filling (the incomplete last group) or truncating (discarding the incomplete last group) the last one:
from iteration_utilities import grouper
seq = list(range(17))
for group in grouper(seq, 4):
print(group)
# (0, 1, 2, 3)
# (4, 5, 6, 7)
# (8, 9, 10, 11)
# (12, 13, 14, 15)
# (16,)
for group in grouper(seq, 4, fillvalue=None):
print(group)
# (0, 1, 2, 3)
# (4, 5, 6, 7)
# (8, 9, 10, 11)
# (12, 13, 14, 15)
# (16, None, None, None)
for group in grouper(seq, 4, truncate=True):
print(group)
# (0, 1, 2, 3)
# (4, 5, 6, 7)
# (8, 9, 10, 11)
# (12, 13, 14, 15)
Benchmarks
I also decided to compare the run-time of a few of the mentioned approaches. It's a log-log plot grouping into groups of "10" elements based on a list of varying size. For qualitative results: Lower means faster:
At least in this benchmark the iteration_utilities.grouper performs best. Followed by the approach of Craz.
The benchmark was created with simple_benchmark1. The code used to run this benchmark was:
import iteration_utilities
import itertools
from itertools import zip_longest
def consume_all(it):
return iteration_utilities.consume(it, None)
import simple_benchmark
b = simple_benchmark.BenchmarkBuilder()
#b.add_function()
def grouper(l, n):
return consume_all(iteration_utilities.grouper(l, n))
def Craz_inner(iterable, n, fillvalue=None):
args = [iter(iterable)] * n
return zip_longest(*args, fillvalue=fillvalue)
#b.add_function()
def Craz(iterable, n, fillvalue=None):
return consume_all(Craz_inner(iterable, n, fillvalue))
def nosklo_inner(seq, size):
return (seq[pos:pos + size] for pos in range(0, len(seq), size))
#b.add_function()
def nosklo(seq, size):
return consume_all(nosklo_inner(seq, size))
def SLott_inner(ints, chunk_size):
for i in range(0, len(ints), chunk_size):
yield ints[i:i+chunk_size]
#b.add_function()
def SLott(ints, chunk_size):
return consume_all(SLott_inner(ints, chunk_size))
def MarkusJarderot1_inner(iterable,size):
it = iter(iterable)
chunk = tuple(itertools.islice(it,size))
while chunk:
yield chunk
chunk = tuple(itertools.islice(it,size))
#b.add_function()
def MarkusJarderot1(iterable,size):
return consume_all(MarkusJarderot1_inner(iterable,size))
def MarkusJarderot2_inner(iterable,size,filler=None):
it = itertools.chain(iterable,itertools.repeat(filler,size-1))
chunk = tuple(itertools.islice(it,size))
while len(chunk) == size:
yield chunk
chunk = tuple(itertools.islice(it,size))
#b.add_function()
def MarkusJarderot2(iterable,size):
return consume_all(MarkusJarderot2_inner(iterable,size))
#b.add_arguments()
def argument_provider():
for exp in range(2, 20):
size = 2**exp
yield size, simple_benchmark.MultiArgument([[0] * size, 10])
r = b.run()
1 Disclaimer: I'm the author of the libraries iteration_utilities and simple_benchmark.
With Python 3.8 you can use the walrus operator and itertools.islice.
from itertools import islice
list_ = [i for i in range(10, 100)]
def chunker(it, size):
iterator = iter(it)
while chunk := list(islice(iterator, size)):
print(chunk)
In [2]: chunker(list_, 10)
[10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
[20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
[30, 31, 32, 33, 34, 35, 36, 37, 38, 39]
[40, 41, 42, 43, 44, 45, 46, 47, 48, 49]
[50, 51, 52, 53, 54, 55, 56, 57, 58, 59]
[60, 61, 62, 63, 64, 65, 66, 67, 68, 69]
[70, 71, 72, 73, 74, 75, 76, 77, 78, 79]
[80, 81, 82, 83, 84, 85, 86, 87, 88, 89]
[90, 91, 92, 93, 94, 95, 96, 97, 98, 99]
I needed a solution that would also work with sets and generators. I couldn't come up with anything very short and pretty, but it's quite readable at least.
def chunker(seq, size):
res = []
for el in seq:
res.append(el)
if len(res) == size:
yield res
res = []
if res:
yield res
List:
>>> list(chunker([i for i in range(10)], 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
Set:
>>> list(chunker(set([i for i in range(10)]), 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
Generator:
>>> list(chunker((i for i in range(10)), 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
The more-itertools package has chunked method which does exactly that:
import more_itertools
for s in more_itertools.chunked(range(9), 4):
print(s)
Prints
[0, 1, 2, 3]
[4, 5, 6, 7]
[8]
chunked returns the items in a list. If you'd prefer iterables, use ichunked.
The ideal solution for this problem works with iterators (not just sequences). It should also be fast.
This is the solution provided by the documentation for itertools:
def grouper(n, iterable, fillvalue=None):
#"grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
args = [iter(iterable)] * n
return itertools.izip_longest(fillvalue=fillvalue, *args)
Using ipython's %timeit on my mac book air, I get 47.5 us per loop.
However, this really doesn't work for me since the results are padded to be even sized groups. A solution without the padding is slightly more complicated. The most naive solution might be:
def grouper(size, iterable):
i = iter(iterable)
while True:
out = []
try:
for _ in range(size):
out.append(i.next())
except StopIteration:
yield out
break
yield out
Simple, but pretty slow: 693 us per loop
The best solution I could come up with uses islice for the inner loop:
def grouper(size, iterable):
it = iter(iterable)
while True:
group = tuple(itertools.islice(it, None, size))
if not group:
break
yield group
With the same dataset, I get 305 us per loop.
Unable to get a pure solution any faster than that, I provide the following solution with an important caveat: If your input data has instances of filldata in it, you could get wrong answer.
def grouper(n, iterable, fillvalue=None):
#"grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
args = [iter(iterable)] * n
# itertools.zip_longest on Python 3
for x in itertools.izip_longest(*args, fillvalue=fillvalue):
if x[-1] is fillvalue:
yield tuple(v for v in x if v is not fillvalue)
else:
yield x
I really don't like this answer, but it is significantly faster. 124 us per loop
from itertools import izip_longest
def chunker(iterable, chunksize, filler):
return izip_longest(*[iter(iterable)]*chunksize, fillvalue=filler)
Similar to other proposals, but not exactly identical, I like doing it this way, because it's simple and easy to read:
it = iter([1, 2, 3, 4, 5, 6, 7, 8, 9])
for chunk in zip(it, it, it, it):
print chunk
>>> (1, 2, 3, 4)
>>> (5, 6, 7, 8)
This way you won't get the last partial chunk. If you want to get (9, None, None, None) as last chunk, just use izip_longest from itertools.
Since nobody's mentioned it yet here's a zip() solution:
>>> def chunker(iterable, chunksize):
... return zip(*[iter(iterable)]*chunksize)
It works only if your sequence's length is always divisible by the chunk size or you don't care about a trailing chunk if it isn't.
Example:
>>> s = '1234567890'
>>> chunker(s, 3)
[('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9')]
>>> chunker(s, 4)
[('1', '2', '3', '4'), ('5', '6', '7', '8')]
>>> chunker(s, 5)
[('1', '2', '3', '4', '5'), ('6', '7', '8', '9', '0')]
Or using itertools.izip to return an iterator instead of a list:
>>> from itertools import izip
>>> def chunker(iterable, chunksize):
... return izip(*[iter(iterable)]*chunksize)
Padding can be fixed using #ΤΖΩΤΖΙΟΥ's answer:
>>> from itertools import chain, izip, repeat
>>> def chunker(iterable, chunksize, fillvalue=None):
... it = chain(iterable, repeat(fillvalue, chunksize-1))
... args = [it] * chunksize
... return izip(*args)
Another approach would be to use the two-argument form of iter:
from itertools import islice
def group(it, size):
it = iter(it)
return iter(lambda: tuple(islice(it, size)), ())
This can be adapted easily to use padding (this is similar to Markus Jarderot’s answer):
from itertools import islice, chain, repeat
def group_pad(it, size, pad=None):
it = chain(iter(it), repeat(pad))
return iter(lambda: tuple(islice(it, size)), (pad,) * size)
These can even be combined for optional padding:
_no_pad = object()
def group(it, size, pad=_no_pad):
if pad == _no_pad:
it = iter(it)
sentinel = ()
else:
it = chain(iter(it), repeat(pad))
sentinel = (pad,) * size
return iter(lambda: tuple(islice(it, size)), sentinel)
If the list is large, the highest-performing way to do this will be to use a generator:
def get_chunk(iterable, chunk_size):
result = []
for item in iterable:
result.append(item)
if len(result) == chunk_size:
yield tuple(result)
result = []
if len(result) > 0:
yield tuple(result)
for x in get_chunk([1,2,3,4,5,6,7,8,9,10], 3):
print x
(1, 2, 3)
(4, 5, 6)
(7, 8, 9)
(10,)
Using little functions and things really doesn't appeal to me; I prefer to just use slices:
data = [...]
chunk_size = 10000 # or whatever
chunks = [data[i:i+chunk_size] for i in xrange(0,len(data),chunk_size)]
for chunk in chunks:
...
Using map() instead of zip() fixes the padding issue in J.F. Sebastian's answer:
>>> def chunker(iterable, chunksize):
... return map(None,*[iter(iterable)]*chunksize)
Example:
>>> s = '1234567890'
>>> chunker(s, 3)
[('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9'), ('0', None, None)]
>>> chunker(s, 4)
[('1', '2', '3', '4'), ('5', '6', '7', '8'), ('9', '0', None, None)]
>>> chunker(s, 5)
[('1', '2', '3', '4', '5'), ('6', '7', '8', '9', '0')]
One-liner, adhoc solution to iterate over a list x in chunks of size 4 -
for a, b, c, d in zip(x[0::4], x[1::4], x[2::4], x[3::4]):
... do something with a, b, c and d ...
To avoid all conversions to a list import itertools and:
>>> for k, g in itertools.groupby(xrange(35), lambda x: x/10):
... list(g)
Produces:
...
0 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
1 [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
2 [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
3 [30, 31, 32, 33, 34]
>>>
I checked groupby and it doesn't convert to list or use len so I (think) this will delay resolution of each value until it is actually used. Sadly none of the available answers (at this time) seemed to offer this variation.
Obviously if you need to handle each item in turn nest a for loop over g:
for k,g in itertools.groupby(xrange(35), lambda x: x/10):
for i in g:
# do what you need to do with individual items
# now do what you need to do with the whole group
My specific interest in this was the need to consume a generator to submit changes in batches of up to 1000 to the gmail API:
messages = a_generator_which_would_not_be_smart_as_a_list
for idx, batch in groupby(messages, lambda x: x/1000):
batch_request = BatchHttpRequest()
for message in batch:
batch_request.add(self.service.users().messages().modify(userId='me', id=message['id'], body=msg_labels))
http = httplib2.Http()
self.credentials.authorize(http)
batch_request.execute(http=http)
Unless I misses something, the following simple solution with generator expressions has not been mentioned. It assumes that both the size and the number of chunks are known (which is often the case), and that no padding is required:
def chunks(it, n, m):
"""Make an iterator over m first chunks of size n.
"""
it = iter(it)
# Chunks are presented as tuples.
return (tuple(next(it) for _ in range(n)) for _ in range(m))
In your second method, I would advance to the next group of 4 by doing this:
ints = ints[4:]
However, I haven't done any performance measurement so I don't know which one might be more efficient.
Having said that, I would usually choose the first method. It's not pretty, but that's often a consequence of interfacing with the outside world.
With NumPy it's simple:
ints = array([1, 2, 3, 4, 5, 6, 7, 8])
for int1, int2 in ints.reshape(-1, 2):
print(int1, int2)
output:
1 2
3 4
5 6
7 8
I never want my chunks padded, so that requirement is essential. I find that the ability to work on any iterable is also requirement. Given that, I decided to extend on the accepted answer, https://stackoverflow.com/a/434411/1074659.
Performance takes a slight hit in this approach if padding is not wanted due to the need to compare and filter the padded values. However, for large chunk sizes, this utility is very performant.
#!/usr/bin/env python3
from itertools import zip_longest
_UNDEFINED = object()
def chunker(iterable, chunksize, fillvalue=_UNDEFINED):
"""
Collect data into chunks and optionally pad it.
Performance worsens as `chunksize` approaches 1.
Inspired by:
https://docs.python.org/3/library/itertools.html#itertools-recipes
"""
args = [iter(iterable)] * chunksize
chunks = zip_longest(*args, fillvalue=fillvalue)
yield from (
filter(lambda val: val is not _UNDEFINED, chunk)
if chunk[-1] is _UNDEFINED
else chunk
for chunk in chunks
) if fillvalue is _UNDEFINED else chunks
def chunker(iterable, n):
"""Yield iterable in chunk sizes.
>>> chunks = chunker('ABCDEF', n=4)
>>> chunks.next()
['A', 'B', 'C', 'D']
>>> chunks.next()
['E', 'F']
"""
it = iter(iterable)
while True:
chunk = []
for i in range(n):
try:
chunk.append(next(it))
except StopIteration:
yield chunk
raise StopIteration
yield chunk
if __name__ == '__main__':
import doctest
doctest.testmod()
Yet another answer, the advantages of which are:
1) Easily understandable
2) Works on any iterable, not just sequences (some of the above answers will choke on filehandles)
3) Does not load the chunk into memory all at once
4) Does not make a chunk-long list of references to the same iterator in memory
5) No padding of fill values at the end of the list
That being said, I haven't timed it so it might be slower than some of the more clever methods, and some of the advantages may be irrelevant given the use case.
def chunkiter(iterable, size):
def inneriter(first, iterator, size):
yield first
for _ in xrange(size - 1):
yield iterator.next()
it = iter(iterable)
while True:
yield inneriter(it.next(), it, size)
In [2]: i = chunkiter('abcdefgh', 3)
In [3]: for ii in i:
for c in ii:
print c,
print ''
...:
a b c
d e f
g h
Update:
A couple of drawbacks due to the fact the inner and outer loops are pulling values from the same iterator:
1) continue doesn't work as expected in the outer loop - it just continues on to the next item rather than skipping a chunk. However, this doesn't seem like a problem as there's nothing to test in the outer loop.
2) break doesn't work as expected in the inner loop - control will wind up in the inner loop again with the next item in the iterator. To skip whole chunks, either wrap the inner iterator (ii above) in a tuple, e.g. for c in tuple(ii), or set a flag and exhaust the iterator.
def group_by(iterable, size):
"""Group an iterable into lists that don't exceed the size given.
>>> group_by([1,2,3,4,5], 2)
[[1, 2], [3, 4], [5]]
"""
sublist = []
for index, item in enumerate(iterable):
if index > 0 and index % size == 0:
yield sublist
sublist = []
sublist.append(item)
if sublist:
yield sublist
You can use partition or chunks function from funcy library:
from funcy import partition
for a, b, c, d in partition(4, ints):
foo += a * b * c * d
These functions also has iterator versions ipartition and ichunks, which will be more efficient in this case.
You can also peek at their implementation.
About solution gave by J.F. Sebastian here:
def chunker(iterable, chunksize):
return zip(*[iter(iterable)]*chunksize)
It's clever, but has one disadvantage - always return tuple. How to get string instead?
Of course you can write ''.join(chunker(...)), but the temporary tuple is constructed anyway.
You can get rid of the temporary tuple by writing own zip, like this:
class IteratorExhausted(Exception):
pass
def translate_StopIteration(iterable, to=IteratorExhausted):
for i in iterable:
yield i
raise to # StopIteration would get ignored because this is generator,
# but custom exception can leave the generator.
def custom_zip(*iterables, reductor=tuple):
iterators = tuple(map(translate_StopIteration, iterables))
while True:
try:
yield reductor(next(i) for i in iterators)
except IteratorExhausted: # when any of iterators get exhausted.
break
Then
def chunker(data, size, reductor=tuple):
return custom_zip(*[iter(data)]*size, reductor=reductor)
Example usage:
>>> for i in chunker('12345', 2):
... print(repr(i))
...
('1', '2')
('3', '4')
>>> for i in chunker('12345', 2, ''.join):
... print(repr(i))
...
'12'
'34'
I like this approach. It feels simple and not magical and supports all iterable types and doesn't require imports.
def chunk_iter(iterable, chunk_size):
it = iter(iterable)
while True:
chunk = tuple(next(it) for _ in range(chunk_size))
if not chunk:
break
yield chunk
Quite pythonic here (you may also inline the body of the split_groups function)
import itertools
def split_groups(iter_in, group_size):
return ((x for _, x in item) for _, item in itertools.groupby(enumerate(iter_in), key=lambda x: x[0] // group_size))
for x, y, z, w in split_groups(range(16), 4):
foo += x * y + z * w
Here is a chunker without imports that supports generators:
def chunks(seq, size):
it = iter(seq)
while True:
ret = tuple(next(it) for _ in range(size))
if len(ret) == size:
yield ret
else:
raise StopIteration()
Example of use:
>>> def foo():
... i = 0
... while True:
... i += 1
... yield i
...
>>> c = chunks(foo(), 3)
>>> c.next()
(1, 2, 3)
>>> c.next()
(4, 5, 6)
>>> list(chunks('abcdefg', 2))
[('a', 'b'), ('c', 'd'), ('e', 'f')]

Can you split a set into equal parts?

I want to do something like the following:
s = set()
s.add('a')
s.add('q')
s.add('x')
s.add('banana')
s1 = s(:1)
s2 = s(2:3)
Is the above a valid operation on sets? Is there a way to apply the above to a generator such as:
def Chunks(container, size):
for i in xrange(0, len(container), size):
yield container(i:i+size)
You can convert the set to a list, do the slicing, and then convert it back to sets:
In [13]: s = {1,2,3}
In [14]: s_l = list(s)
In [15]: print set(s_l[:1]), set(s_l[1:])
set([1]) set([2, 3])
Do note that sets do not support ordering, so any such ordering operation would have to be done within a list:
In [16]: s = {1,2,3,0}
In [17]: s
Out[17]: {0, 1, 2, 3}
In [18]: s_l = list(s)
In [19]: print set(s_l[:2]), set(s_l[2:])
set([0, 1]) set([2, 3])
Using itertools.islice it would be easy
>>> from itertools import islice
>>> def chunk(it, size):
... it = iter(it)
... return iter(lambda: tuple(islice(it, size)), ())
>>> data = {i for i in range(20)}
for j in chunk({i for i in range(20)}, 4):
... print(j)
...
(0, 1, 2, 3)
(4, 5, 6, 7)
(8, 9, 10, 11)
(12, 13, 14, 15)
(16, 17, 18, 19)
Sets are not indexable in the same way that lists are, but you can iterate over "chunks" or subgroups from them. Use the grouper recipe from itertools:
from itertools import izip_longest
def grouper(iterable, n, fillvalue=None):
"Collect data into fixed-length chunks or blocks"
# grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
args = [iter(iterable)] * n
return izip_longest(fillvalue=fillvalue, *args)
s2 = set('Andy Sandy Jack Jill Frank Fred Sally Sarah Bill Billy Bob'.split())
for g in grouper(s2, 3):
print g
Yields:
('Sarah', 'Frank', 'Bill')
('Fred', 'Billy', 'Jill')
('Andy', 'Jack', 'Bob')
('Sally', 'Sandy', None)
The initial order of the set initializer is not preserved because unlike lists,
sets don't preserve order.
you can use itertools.islice, though sets don't guarantee an ordering so be careful if you aren't looking at the whole set

Loop through 30K+ rows in groups/blocks of 10 or 100s [duplicate]

I have a Python script which takes as input a list of integers, which I need to work with four integers at a time. Unfortunately, I don't have control of the input, or I'd have it passed in as a list of four-element tuples. Currently, I'm iterating over it this way:
for i in range(0, len(ints), 4):
# dummy op for example code
foo += ints[i] * ints[i + 1] + ints[i + 2] * ints[i + 3]
It looks a lot like "C-think", though, which makes me suspect there's a more pythonic way of dealing with this situation. The list is discarded after iterating, so it needn't be preserved. Perhaps something like this would be better?
while ints:
foo += ints[0] * ints[1] + ints[2] * ints[3]
ints[0:4] = []
Still doesn't quite "feel" right, though. :-/
Related question: How do you split a list into evenly sized chunks in Python?
def chunker(seq, size):
return (seq[pos:pos + size] for pos in range(0, len(seq), size))
Works with any sequence:
text = "I am a very, very helpful text"
for group in chunker(text, 7):
print(repr(group),)
# 'I am a ' 'very, v' 'ery hel' 'pful te' 'xt'
print('|'.join(chunker(text, 10)))
# I am a ver|y, very he|lpful text
animals = ['cat', 'dog', 'rabbit', 'duck', 'bird', 'cow', 'gnu', 'fish']
for group in chunker(animals, 3):
print(group)
# ['cat', 'dog', 'rabbit']
# ['duck', 'bird', 'cow']
# ['gnu', 'fish']
Modified from the Recipes section of Python's itertools docs:
from itertools import zip_longest
def grouper(iterable, n, fillvalue=None):
args = [iter(iterable)] * n
return zip_longest(*args, fillvalue=fillvalue)
Example
grouper('ABCDEFG', 3, 'x') # --> 'ABC' 'DEF' 'Gxx'
Note: on Python 2 use izip_longest instead of zip_longest.
chunk_size = 4
for i in range(0, len(ints), chunk_size):
chunk = ints[i:i+chunk_size]
# process chunk of size <= chunk_size
import itertools
def chunks(iterable,size):
it = iter(iterable)
chunk = tuple(itertools.islice(it,size))
while chunk:
yield chunk
chunk = tuple(itertools.islice(it,size))
# though this will throw ValueError if the length of ints
# isn't a multiple of four:
for x1,x2,x3,x4 in chunks(ints,4):
foo += x1 + x2 + x3 + x4
for chunk in chunks(ints,4):
foo += sum(chunk)
Another way:
import itertools
def chunks2(iterable,size,filler=None):
it = itertools.chain(iterable,itertools.repeat(filler,size-1))
chunk = tuple(itertools.islice(it,size))
while len(chunk) == size:
yield chunk
chunk = tuple(itertools.islice(it,size))
# x2, x3 and x4 could get the value 0 if the length is not
# a multiple of 4.
for x1,x2,x3,x4 in chunks2(ints,4,0):
foo += x1 + x2 + x3 + x4
If you don't mind using an external package you could use iteration_utilities.grouper from iteration_utilties 1. It supports all iterables (not just sequences):
from iteration_utilities import grouper
seq = list(range(20))
for group in grouper(seq, 4):
print(group)
which prints:
(0, 1, 2, 3)
(4, 5, 6, 7)
(8, 9, 10, 11)
(12, 13, 14, 15)
(16, 17, 18, 19)
In case the length isn't a multiple of the groupsize it also supports filling (the incomplete last group) or truncating (discarding the incomplete last group) the last one:
from iteration_utilities import grouper
seq = list(range(17))
for group in grouper(seq, 4):
print(group)
# (0, 1, 2, 3)
# (4, 5, 6, 7)
# (8, 9, 10, 11)
# (12, 13, 14, 15)
# (16,)
for group in grouper(seq, 4, fillvalue=None):
print(group)
# (0, 1, 2, 3)
# (4, 5, 6, 7)
# (8, 9, 10, 11)
# (12, 13, 14, 15)
# (16, None, None, None)
for group in grouper(seq, 4, truncate=True):
print(group)
# (0, 1, 2, 3)
# (4, 5, 6, 7)
# (8, 9, 10, 11)
# (12, 13, 14, 15)
Benchmarks
I also decided to compare the run-time of a few of the mentioned approaches. It's a log-log plot grouping into groups of "10" elements based on a list of varying size. For qualitative results: Lower means faster:
At least in this benchmark the iteration_utilities.grouper performs best. Followed by the approach of Craz.
The benchmark was created with simple_benchmark1. The code used to run this benchmark was:
import iteration_utilities
import itertools
from itertools import zip_longest
def consume_all(it):
return iteration_utilities.consume(it, None)
import simple_benchmark
b = simple_benchmark.BenchmarkBuilder()
#b.add_function()
def grouper(l, n):
return consume_all(iteration_utilities.grouper(l, n))
def Craz_inner(iterable, n, fillvalue=None):
args = [iter(iterable)] * n
return zip_longest(*args, fillvalue=fillvalue)
#b.add_function()
def Craz(iterable, n, fillvalue=None):
return consume_all(Craz_inner(iterable, n, fillvalue))
def nosklo_inner(seq, size):
return (seq[pos:pos + size] for pos in range(0, len(seq), size))
#b.add_function()
def nosklo(seq, size):
return consume_all(nosklo_inner(seq, size))
def SLott_inner(ints, chunk_size):
for i in range(0, len(ints), chunk_size):
yield ints[i:i+chunk_size]
#b.add_function()
def SLott(ints, chunk_size):
return consume_all(SLott_inner(ints, chunk_size))
def MarkusJarderot1_inner(iterable,size):
it = iter(iterable)
chunk = tuple(itertools.islice(it,size))
while chunk:
yield chunk
chunk = tuple(itertools.islice(it,size))
#b.add_function()
def MarkusJarderot1(iterable,size):
return consume_all(MarkusJarderot1_inner(iterable,size))
def MarkusJarderot2_inner(iterable,size,filler=None):
it = itertools.chain(iterable,itertools.repeat(filler,size-1))
chunk = tuple(itertools.islice(it,size))
while len(chunk) == size:
yield chunk
chunk = tuple(itertools.islice(it,size))
#b.add_function()
def MarkusJarderot2(iterable,size):
return consume_all(MarkusJarderot2_inner(iterable,size))
#b.add_arguments()
def argument_provider():
for exp in range(2, 20):
size = 2**exp
yield size, simple_benchmark.MultiArgument([[0] * size, 10])
r = b.run()
1 Disclaimer: I'm the author of the libraries iteration_utilities and simple_benchmark.
With Python 3.8 you can use the walrus operator and itertools.islice.
from itertools import islice
list_ = [i for i in range(10, 100)]
def chunker(it, size):
iterator = iter(it)
while chunk := list(islice(iterator, size)):
print(chunk)
In [2]: chunker(list_, 10)
[10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
[20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
[30, 31, 32, 33, 34, 35, 36, 37, 38, 39]
[40, 41, 42, 43, 44, 45, 46, 47, 48, 49]
[50, 51, 52, 53, 54, 55, 56, 57, 58, 59]
[60, 61, 62, 63, 64, 65, 66, 67, 68, 69]
[70, 71, 72, 73, 74, 75, 76, 77, 78, 79]
[80, 81, 82, 83, 84, 85, 86, 87, 88, 89]
[90, 91, 92, 93, 94, 95, 96, 97, 98, 99]
I needed a solution that would also work with sets and generators. I couldn't come up with anything very short and pretty, but it's quite readable at least.
def chunker(seq, size):
res = []
for el in seq:
res.append(el)
if len(res) == size:
yield res
res = []
if res:
yield res
List:
>>> list(chunker([i for i in range(10)], 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
Set:
>>> list(chunker(set([i for i in range(10)]), 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
Generator:
>>> list(chunker((i for i in range(10)), 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
The more-itertools package has chunked method which does exactly that:
import more_itertools
for s in more_itertools.chunked(range(9), 4):
print(s)
Prints
[0, 1, 2, 3]
[4, 5, 6, 7]
[8]
chunked returns the items in a list. If you'd prefer iterables, use ichunked.
The ideal solution for this problem works with iterators (not just sequences). It should also be fast.
This is the solution provided by the documentation for itertools:
def grouper(n, iterable, fillvalue=None):
#"grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
args = [iter(iterable)] * n
return itertools.izip_longest(fillvalue=fillvalue, *args)
Using ipython's %timeit on my mac book air, I get 47.5 us per loop.
However, this really doesn't work for me since the results are padded to be even sized groups. A solution without the padding is slightly more complicated. The most naive solution might be:
def grouper(size, iterable):
i = iter(iterable)
while True:
out = []
try:
for _ in range(size):
out.append(i.next())
except StopIteration:
yield out
break
yield out
Simple, but pretty slow: 693 us per loop
The best solution I could come up with uses islice for the inner loop:
def grouper(size, iterable):
it = iter(iterable)
while True:
group = tuple(itertools.islice(it, None, size))
if not group:
break
yield group
With the same dataset, I get 305 us per loop.
Unable to get a pure solution any faster than that, I provide the following solution with an important caveat: If your input data has instances of filldata in it, you could get wrong answer.
def grouper(n, iterable, fillvalue=None):
#"grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
args = [iter(iterable)] * n
# itertools.zip_longest on Python 3
for x in itertools.izip_longest(*args, fillvalue=fillvalue):
if x[-1] is fillvalue:
yield tuple(v for v in x if v is not fillvalue)
else:
yield x
I really don't like this answer, but it is significantly faster. 124 us per loop
from itertools import izip_longest
def chunker(iterable, chunksize, filler):
return izip_longest(*[iter(iterable)]*chunksize, fillvalue=filler)
Similar to other proposals, but not exactly identical, I like doing it this way, because it's simple and easy to read:
it = iter([1, 2, 3, 4, 5, 6, 7, 8, 9])
for chunk in zip(it, it, it, it):
print chunk
>>> (1, 2, 3, 4)
>>> (5, 6, 7, 8)
This way you won't get the last partial chunk. If you want to get (9, None, None, None) as last chunk, just use izip_longest from itertools.
Since nobody's mentioned it yet here's a zip() solution:
>>> def chunker(iterable, chunksize):
... return zip(*[iter(iterable)]*chunksize)
It works only if your sequence's length is always divisible by the chunk size or you don't care about a trailing chunk if it isn't.
Example:
>>> s = '1234567890'
>>> chunker(s, 3)
[('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9')]
>>> chunker(s, 4)
[('1', '2', '3', '4'), ('5', '6', '7', '8')]
>>> chunker(s, 5)
[('1', '2', '3', '4', '5'), ('6', '7', '8', '9', '0')]
Or using itertools.izip to return an iterator instead of a list:
>>> from itertools import izip
>>> def chunker(iterable, chunksize):
... return izip(*[iter(iterable)]*chunksize)
Padding can be fixed using #ΤΖΩΤΖΙΟΥ's answer:
>>> from itertools import chain, izip, repeat
>>> def chunker(iterable, chunksize, fillvalue=None):
... it = chain(iterable, repeat(fillvalue, chunksize-1))
... args = [it] * chunksize
... return izip(*args)
Another approach would be to use the two-argument form of iter:
from itertools import islice
def group(it, size):
it = iter(it)
return iter(lambda: tuple(islice(it, size)), ())
This can be adapted easily to use padding (this is similar to Markus Jarderot’s answer):
from itertools import islice, chain, repeat
def group_pad(it, size, pad=None):
it = chain(iter(it), repeat(pad))
return iter(lambda: tuple(islice(it, size)), (pad,) * size)
These can even be combined for optional padding:
_no_pad = object()
def group(it, size, pad=_no_pad):
if pad == _no_pad:
it = iter(it)
sentinel = ()
else:
it = chain(iter(it), repeat(pad))
sentinel = (pad,) * size
return iter(lambda: tuple(islice(it, size)), sentinel)
If the list is large, the highest-performing way to do this will be to use a generator:
def get_chunk(iterable, chunk_size):
result = []
for item in iterable:
result.append(item)
if len(result) == chunk_size:
yield tuple(result)
result = []
if len(result) > 0:
yield tuple(result)
for x in get_chunk([1,2,3,4,5,6,7,8,9,10], 3):
print x
(1, 2, 3)
(4, 5, 6)
(7, 8, 9)
(10,)
Using little functions and things really doesn't appeal to me; I prefer to just use slices:
data = [...]
chunk_size = 10000 # or whatever
chunks = [data[i:i+chunk_size] for i in xrange(0,len(data),chunk_size)]
for chunk in chunks:
...
Using map() instead of zip() fixes the padding issue in J.F. Sebastian's answer:
>>> def chunker(iterable, chunksize):
... return map(None,*[iter(iterable)]*chunksize)
Example:
>>> s = '1234567890'
>>> chunker(s, 3)
[('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9'), ('0', None, None)]
>>> chunker(s, 4)
[('1', '2', '3', '4'), ('5', '6', '7', '8'), ('9', '0', None, None)]
>>> chunker(s, 5)
[('1', '2', '3', '4', '5'), ('6', '7', '8', '9', '0')]
One-liner, adhoc solution to iterate over a list x in chunks of size 4 -
for a, b, c, d in zip(x[0::4], x[1::4], x[2::4], x[3::4]):
... do something with a, b, c and d ...
To avoid all conversions to a list import itertools and:
>>> for k, g in itertools.groupby(xrange(35), lambda x: x/10):
... list(g)
Produces:
...
0 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
1 [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
2 [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
3 [30, 31, 32, 33, 34]
>>>
I checked groupby and it doesn't convert to list or use len so I (think) this will delay resolution of each value until it is actually used. Sadly none of the available answers (at this time) seemed to offer this variation.
Obviously if you need to handle each item in turn nest a for loop over g:
for k,g in itertools.groupby(xrange(35), lambda x: x/10):
for i in g:
# do what you need to do with individual items
# now do what you need to do with the whole group
My specific interest in this was the need to consume a generator to submit changes in batches of up to 1000 to the gmail API:
messages = a_generator_which_would_not_be_smart_as_a_list
for idx, batch in groupby(messages, lambda x: x/1000):
batch_request = BatchHttpRequest()
for message in batch:
batch_request.add(self.service.users().messages().modify(userId='me', id=message['id'], body=msg_labels))
http = httplib2.Http()
self.credentials.authorize(http)
batch_request.execute(http=http)
Unless I misses something, the following simple solution with generator expressions has not been mentioned. It assumes that both the size and the number of chunks are known (which is often the case), and that no padding is required:
def chunks(it, n, m):
"""Make an iterator over m first chunks of size n.
"""
it = iter(it)
# Chunks are presented as tuples.
return (tuple(next(it) for _ in range(n)) for _ in range(m))
In your second method, I would advance to the next group of 4 by doing this:
ints = ints[4:]
However, I haven't done any performance measurement so I don't know which one might be more efficient.
Having said that, I would usually choose the first method. It's not pretty, but that's often a consequence of interfacing with the outside world.
With NumPy it's simple:
ints = array([1, 2, 3, 4, 5, 6, 7, 8])
for int1, int2 in ints.reshape(-1, 2):
print(int1, int2)
output:
1 2
3 4
5 6
7 8
I never want my chunks padded, so that requirement is essential. I find that the ability to work on any iterable is also requirement. Given that, I decided to extend on the accepted answer, https://stackoverflow.com/a/434411/1074659.
Performance takes a slight hit in this approach if padding is not wanted due to the need to compare and filter the padded values. However, for large chunk sizes, this utility is very performant.
#!/usr/bin/env python3
from itertools import zip_longest
_UNDEFINED = object()
def chunker(iterable, chunksize, fillvalue=_UNDEFINED):
"""
Collect data into chunks and optionally pad it.
Performance worsens as `chunksize` approaches 1.
Inspired by:
https://docs.python.org/3/library/itertools.html#itertools-recipes
"""
args = [iter(iterable)] * chunksize
chunks = zip_longest(*args, fillvalue=fillvalue)
yield from (
filter(lambda val: val is not _UNDEFINED, chunk)
if chunk[-1] is _UNDEFINED
else chunk
for chunk in chunks
) if fillvalue is _UNDEFINED else chunks
def chunker(iterable, n):
"""Yield iterable in chunk sizes.
>>> chunks = chunker('ABCDEF', n=4)
>>> chunks.next()
['A', 'B', 'C', 'D']
>>> chunks.next()
['E', 'F']
"""
it = iter(iterable)
while True:
chunk = []
for i in range(n):
try:
chunk.append(next(it))
except StopIteration:
yield chunk
raise StopIteration
yield chunk
if __name__ == '__main__':
import doctest
doctest.testmod()
Yet another answer, the advantages of which are:
1) Easily understandable
2) Works on any iterable, not just sequences (some of the above answers will choke on filehandles)
3) Does not load the chunk into memory all at once
4) Does not make a chunk-long list of references to the same iterator in memory
5) No padding of fill values at the end of the list
That being said, I haven't timed it so it might be slower than some of the more clever methods, and some of the advantages may be irrelevant given the use case.
def chunkiter(iterable, size):
def inneriter(first, iterator, size):
yield first
for _ in xrange(size - 1):
yield iterator.next()
it = iter(iterable)
while True:
yield inneriter(it.next(), it, size)
In [2]: i = chunkiter('abcdefgh', 3)
In [3]: for ii in i:
for c in ii:
print c,
print ''
...:
a b c
d e f
g h
Update:
A couple of drawbacks due to the fact the inner and outer loops are pulling values from the same iterator:
1) continue doesn't work as expected in the outer loop - it just continues on to the next item rather than skipping a chunk. However, this doesn't seem like a problem as there's nothing to test in the outer loop.
2) break doesn't work as expected in the inner loop - control will wind up in the inner loop again with the next item in the iterator. To skip whole chunks, either wrap the inner iterator (ii above) in a tuple, e.g. for c in tuple(ii), or set a flag and exhaust the iterator.
def group_by(iterable, size):
"""Group an iterable into lists that don't exceed the size given.
>>> group_by([1,2,3,4,5], 2)
[[1, 2], [3, 4], [5]]
"""
sublist = []
for index, item in enumerate(iterable):
if index > 0 and index % size == 0:
yield sublist
sublist = []
sublist.append(item)
if sublist:
yield sublist
You can use partition or chunks function from funcy library:
from funcy import partition
for a, b, c, d in partition(4, ints):
foo += a * b * c * d
These functions also has iterator versions ipartition and ichunks, which will be more efficient in this case.
You can also peek at their implementation.
About solution gave by J.F. Sebastian here:
def chunker(iterable, chunksize):
return zip(*[iter(iterable)]*chunksize)
It's clever, but has one disadvantage - always return tuple. How to get string instead?
Of course you can write ''.join(chunker(...)), but the temporary tuple is constructed anyway.
You can get rid of the temporary tuple by writing own zip, like this:
class IteratorExhausted(Exception):
pass
def translate_StopIteration(iterable, to=IteratorExhausted):
for i in iterable:
yield i
raise to # StopIteration would get ignored because this is generator,
# but custom exception can leave the generator.
def custom_zip(*iterables, reductor=tuple):
iterators = tuple(map(translate_StopIteration, iterables))
while True:
try:
yield reductor(next(i) for i in iterators)
except IteratorExhausted: # when any of iterators get exhausted.
break
Then
def chunker(data, size, reductor=tuple):
return custom_zip(*[iter(data)]*size, reductor=reductor)
Example usage:
>>> for i in chunker('12345', 2):
... print(repr(i))
...
('1', '2')
('3', '4')
>>> for i in chunker('12345', 2, ''.join):
... print(repr(i))
...
'12'
'34'
I like this approach. It feels simple and not magical and supports all iterable types and doesn't require imports.
def chunk_iter(iterable, chunk_size):
it = iter(iterable)
while True:
chunk = tuple(next(it) for _ in range(chunk_size))
if not chunk:
break
yield chunk
Quite pythonic here (you may also inline the body of the split_groups function)
import itertools
def split_groups(iter_in, group_size):
return ((x for _, x in item) for _, item in itertools.groupby(enumerate(iter_in), key=lambda x: x[0] // group_size))
for x, y, z, w in split_groups(range(16), 4):
foo += x * y + z * w
Here is a chunker without imports that supports generators:
def chunks(seq, size):
it = iter(seq)
while True:
ret = tuple(next(it) for _ in range(size))
if len(ret) == size:
yield ret
else:
raise StopIteration()
Example of use:
>>> def foo():
... i = 0
... while True:
... i += 1
... yield i
...
>>> c = chunks(foo(), 3)
>>> c.next()
(1, 2, 3)
>>> c.next()
(4, 5, 6)
>>> list(chunks('abcdefg', 2))
[('a', 'b'), ('c', 'd'), ('e', 'f')]

how to split an iterable in constant-size chunks [duplicate]

This question already has answers here:
How do I split a list into equally-sized chunks?
(66 answers)
Closed 8 months ago.
I am surprised I could not find a "batch" function that would take as input an iterable and return an iterable of iterables.
For example:
for i in batch(range(0,10), 1): print i
[0]
[1]
...
[9]
or:
for i in batch(range(0,10), 3): print i
[0,1,2]
[3,4,5]
[6,7,8]
[9]
Now, I wrote what I thought was a pretty simple generator:
def batch(iterable, n = 1):
current_batch = []
for item in iterable:
current_batch.append(item)
if len(current_batch) == n:
yield current_batch
current_batch = []
if current_batch:
yield current_batch
But the above does not give me what I would have expected:
for x in batch(range(0,10),3): print x
[0]
[0, 1]
[0, 1, 2]
[3]
[3, 4]
[3, 4, 5]
[6]
[6, 7]
[6, 7, 8]
[9]
So, I have missed something and this probably shows my complete lack of understanding of python generators. Anyone would care to point me in the right direction ?
[Edit: I eventually realized that the above behavior happens only when I run this within ipython rather than python itself]
This is probably more efficient (faster)
def batch(iterable, n=1):
l = len(iterable)
for ndx in range(0, l, n):
yield iterable[ndx:min(ndx + n, l)]
for x in batch(range(0, 10), 3):
print x
Example using list
data = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] # list of data
for x in batch(data, 3):
print(x)
# Output
[0, 1, 2]
[3, 4, 5]
[6, 7, 8]
[9, 10]
It avoids building new lists.
The recipes in the itertools module provide two ways to do this depending on how you want to handle a final odd-sized lot (keep it, pad it with a fillvalue, ignore it, or raise an exception):
from itertools import islice, zip_longest
def batched(iterable, n):
"Batch data into lists of length n. The last batch may be shorter."
# batched('ABCDEFG', 3) --> ABC DEF G
it = iter(iterable)
while True:
batch = list(islice(it, n))
if not batch:
return
yield batch
def grouper(iterable, n, *, incomplete='fill', fillvalue=None):
"Collect data into non-overlapping fixed-length chunks or blocks"
# grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx
# grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError
# grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF
args = [iter(iterable)] * n
if incomplete == 'fill':
return zip_longest(*args, fillvalue=fillvalue)
if incomplete == 'strict':
return zip(*args, strict=True)
if incomplete == 'ignore':
return zip(*args)
else:
raise ValueError('Expected fill, strict, or ignore')
More-itertools includes two functions that do what you need:
chunked(iterable, n) returns an iterable of lists, each of length n (except the last one, which may be shorter);
ichunked(iterable, n) is similar, but returns an iterable of iterables instead.
As others have noted, the code you have given does exactly what you want. For another approach using itertools.islice you could see an example of following recipe:
from itertools import islice, chain
def batch(iterable, size):
sourceiter = iter(iterable)
while True:
batchiter = islice(sourceiter, size)
yield chain([batchiter.next()], batchiter)
Solution for Python 3.8 if you are working with iterables that don't define a len function, and get exhausted:
from itertools import islice
def batcher(iterable, batch_size):
iterator = iter(iterable)
while batch := list(islice(iterator, batch_size)):
yield batch
Example usage:
def my_gen():
yield from range(10)
for batch in batcher(my_gen(), 3):
print(batch)
>>> [0, 1, 2]
>>> [3, 4, 5]
>>> [6, 7, 8]
>>> [9]
Could of course be implemented without the walrus operator as well.
This is a very short code snippet I know that does not use len and works under both Python 2 and 3 (not my creation):
def chunks(iterable, size):
from itertools import chain, islice
iterator = iter(iterable)
for first in iterator:
yield list(chain([first], islice(iterator, size - 1)))
Weird, seems to work fine for me in Python 2.x
>>> def batch(iterable, n = 1):
... current_batch = []
... for item in iterable:
... current_batch.append(item)
... if len(current_batch) == n:
... yield current_batch
... current_batch = []
... if current_batch:
... yield current_batch
...
>>> for x in batch(range(0, 10), 3):
... print x
...
[0, 1, 2]
[3, 4, 5]
[6, 7, 8]
[9]
A workable version without new features in python 3.8, adapted from #Atra Azami's answer.
import itertools
def batch_generator(iterable, batch_size=1):
iterable = iter(iterable)
while True:
batch = list(itertools.islice(iterable, batch_size))
if len(batch) > 0:
yield batch
else:
break
for x in batch_generator(range(0, 10), 3):
print(x)
Output:
[0, 1, 2]
[3, 4, 5]
[6, 7, 8]
[9]
def batch(iterable, n):
iterable=iter(iterable)
while True:
chunk=[]
for i in range(n):
try:
chunk.append(next(iterable))
except StopIteration:
yield chunk
return
yield chunk
list(batch(range(10), 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
Moving as much into CPython as possible, by leveraging islice and iter(callable) behavior:
from itertools import islice
def chunked(generator, size):
"""Read parts of the generator, pause each time after a chunk"""
# islice returns results until 'size',
# make_chunk gets repeatedly called by iter(callable).
gen = iter(generator)
make_chunk = lambda: list(islice(gen, size))
return iter(make_chunk, [])
Inspired by more-itertools, and shortened to the essence of that code.
This is what I use in my project. It handles iterables or lists as efficiently as possible.
def chunker(iterable, size):
if not hasattr(iterable, "__len__"):
# generators don't have len, so fall back to slower
# method that works with generators
for chunk in chunker_gen(iterable, size):
yield chunk
return
it = iter(iterable)
for i in range(0, len(iterable), size):
yield [k for k in islice(it, size)]
def chunker_gen(generator, size):
iterator = iter(generator)
for first in iterator:
def chunk():
yield first
for more in islice(iterator, size - 1):
yield more
yield [k for k in chunk()]
I like this one,
def batch(x, bs):
return [x[i:i+bs] for i in range(0, len(x), bs)]
This returns a list of batches of size bs, you can make it a generator by using a generator expression (i for i in iterable) of course.
Here is an approach using reduce function.
Oneliner:
from functools import reduce
reduce(lambda cumulator,item: cumulator[-1].append(item) or cumulator if len(cumulator[-1]) < batch_size else cumulator + [[item]], input_array, [[]])
Or more readable version:
from functools import reduce
def batch(input_list, batch_size):
def reducer(cumulator, item):
if len(cumulator[-1]) < batch_size:
cumulator[-1].append(item)
return cumulator
else:
cumulator.append([item])
return cumulator
return reduce(reducer, input_list, [[]])
Test:
>>> batch([1,2,3,4,5,6,7], 3)
[[1, 2, 3], [4, 5, 6], [7]]
>>> batch(a, 8)
[[1, 2, 3, 4, 5, 6, 7]]
>>> batch([1,2,3,None,4], 3)
[[1, 2, 3], [None, 4]]
This would work for any iterable.
from itertools import zip_longest, filterfalse
def batch_iterable(iterable, batch_size=2):
args = [iter(iterable)] * batch_size
return (tuple(filterfalse(lambda x: x is None, group)) for group in zip_longest(fillvalue=None, *args))
It would work like this:
>>>list(batch_iterable(range(0,5)), 2)
[(0, 1), (2, 3), (4,)]
PS: It would not work if iterable has None values.
You can just group iterable items by their batch index.
def batch(items: Iterable, batch_size: int) -> Iterable[Iterable]:
# enumerate items and group them by batch index
enumerated_item_groups = itertools.groupby(enumerate(items), lambda t: t[0] // batch_size)
# extract items from enumeration tuples
item_batches = ((t[1] for t in enumerated_items) for key, enumerated_items in enumerated_item_groups)
return item_batches
It is often the case when you want to collect inner iterables so here is more advanced version.
def batch_advanced(items: Iterable, batch_size: int, batches_mapper: Callable[[Iterable], Any] = None) -> Iterable[Iterable]:
enumerated_item_groups = itertools.groupby(enumerate(items), lambda t: t[0] // batch_size)
if batches_mapper:
item_batches = (batches_mapper(t[1] for t in enumerated_items) for key, enumerated_items in enumerated_item_groups)
else:
item_batches = ((t[1] for t in enumerated_items) for key, enumerated_items in enumerated_item_groups)
return item_batches
Examples:
print(list(batch_advanced([1, 9, 3, 5, 2, 4, 2], 4, tuple)))
# [(1, 9, 3, 5), (2, 4, 2)]
print(list(batch_advanced([1, 9, 3, 5, 2, 4, 2], 4, list)))
# [[1, 9, 3, 5], [2, 4, 2]]
Related functionality you may need:
def batch(size, i):
""" Get the i'th batch of the given size """
return slice(size* i, size* i + size)
Usage:
>>> [1,2,3,4,5,6,7,8,9,10][batch(3, 1)]
>>> [4, 5, 6]
It gets the i'th batch from the sequence and it can work with other data structures as well, like pandas dataframes (df.iloc[batch(100,0)]) or numpy array (array[batch(100,0)]).
from itertools import *
class SENTINEL: pass
def batch(iterable, n):
return (tuple(filterfalse(lambda x: x is SENTINEL, group)) for group in zip_longest(fillvalue=SENTINEL, *[iter(iterable)] * n))
print(list(range(10), 3)))
# outputs: [(0, 1, 2), (3, 4, 5), (6, 7, 8), (9,)]
print(list(batch([None]*10, 3)))
# outputs: [(None, None, None), (None, None, None), (None, None, None), (None,)]
I use
def batchify(arr, batch_size):
num_batches = math.ceil(len(arr) / batch_size)
return [arr[i*batch_size:(i+1)*batch_size] for i in range(num_batches)]
Keep taking (at most) n elements until it runs out.
def chop(n, iterable):
iterator = iter(iterable)
while chunk := list(take(n, iterator)):
yield chunk
def take(n, iterable):
iterator = iter(iterable)
for i in range(n):
try:
yield next(iterator)
except StopIteration:
return
This code has the following features:
Can take lists or generators (no len()) as input
Does not require imports of other packages
No padding added to last batch
def batch_generator(items, batch_size):
itemid=0 # Keeps track of current position in items generator/list
batch = [] # Empty batch
for item in items:
batch.append(item) # Append items to batch
if len(batch)==batch_size:
yield batch
itemid += batch_size # Increment the position in items
batch = []
yield batch # yield last bit

How to iterate over a list in chunks

I have a Python script which takes as input a list of integers, which I need to work with four integers at a time. Unfortunately, I don't have control of the input, or I'd have it passed in as a list of four-element tuples. Currently, I'm iterating over it this way:
for i in range(0, len(ints), 4):
# dummy op for example code
foo += ints[i] * ints[i + 1] + ints[i + 2] * ints[i + 3]
It looks a lot like "C-think", though, which makes me suspect there's a more pythonic way of dealing with this situation. The list is discarded after iterating, so it needn't be preserved. Perhaps something like this would be better?
while ints:
foo += ints[0] * ints[1] + ints[2] * ints[3]
ints[0:4] = []
Still doesn't quite "feel" right, though. :-/
Related question: How do you split a list into evenly sized chunks in Python?
def chunker(seq, size):
return (seq[pos:pos + size] for pos in range(0, len(seq), size))
Works with any sequence:
text = "I am a very, very helpful text"
for group in chunker(text, 7):
print(repr(group),)
# 'I am a ' 'very, v' 'ery hel' 'pful te' 'xt'
print('|'.join(chunker(text, 10)))
# I am a ver|y, very he|lpful text
animals = ['cat', 'dog', 'rabbit', 'duck', 'bird', 'cow', 'gnu', 'fish']
for group in chunker(animals, 3):
print(group)
# ['cat', 'dog', 'rabbit']
# ['duck', 'bird', 'cow']
# ['gnu', 'fish']
Modified from the Recipes section of Python's itertools docs:
from itertools import zip_longest
def grouper(iterable, n, fillvalue=None):
args = [iter(iterable)] * n
return zip_longest(*args, fillvalue=fillvalue)
Example
grouper('ABCDEFG', 3, 'x') # --> 'ABC' 'DEF' 'Gxx'
Note: on Python 2 use izip_longest instead of zip_longest.
chunk_size = 4
for i in range(0, len(ints), chunk_size):
chunk = ints[i:i+chunk_size]
# process chunk of size <= chunk_size
import itertools
def chunks(iterable,size):
it = iter(iterable)
chunk = tuple(itertools.islice(it,size))
while chunk:
yield chunk
chunk = tuple(itertools.islice(it,size))
# though this will throw ValueError if the length of ints
# isn't a multiple of four:
for x1,x2,x3,x4 in chunks(ints,4):
foo += x1 + x2 + x3 + x4
for chunk in chunks(ints,4):
foo += sum(chunk)
Another way:
import itertools
def chunks2(iterable,size,filler=None):
it = itertools.chain(iterable,itertools.repeat(filler,size-1))
chunk = tuple(itertools.islice(it,size))
while len(chunk) == size:
yield chunk
chunk = tuple(itertools.islice(it,size))
# x2, x3 and x4 could get the value 0 if the length is not
# a multiple of 4.
for x1,x2,x3,x4 in chunks2(ints,4,0):
foo += x1 + x2 + x3 + x4
If you don't mind using an external package you could use iteration_utilities.grouper from iteration_utilties 1. It supports all iterables (not just sequences):
from iteration_utilities import grouper
seq = list(range(20))
for group in grouper(seq, 4):
print(group)
which prints:
(0, 1, 2, 3)
(4, 5, 6, 7)
(8, 9, 10, 11)
(12, 13, 14, 15)
(16, 17, 18, 19)
In case the length isn't a multiple of the groupsize it also supports filling (the incomplete last group) or truncating (discarding the incomplete last group) the last one:
from iteration_utilities import grouper
seq = list(range(17))
for group in grouper(seq, 4):
print(group)
# (0, 1, 2, 3)
# (4, 5, 6, 7)
# (8, 9, 10, 11)
# (12, 13, 14, 15)
# (16,)
for group in grouper(seq, 4, fillvalue=None):
print(group)
# (0, 1, 2, 3)
# (4, 5, 6, 7)
# (8, 9, 10, 11)
# (12, 13, 14, 15)
# (16, None, None, None)
for group in grouper(seq, 4, truncate=True):
print(group)
# (0, 1, 2, 3)
# (4, 5, 6, 7)
# (8, 9, 10, 11)
# (12, 13, 14, 15)
Benchmarks
I also decided to compare the run-time of a few of the mentioned approaches. It's a log-log plot grouping into groups of "10" elements based on a list of varying size. For qualitative results: Lower means faster:
At least in this benchmark the iteration_utilities.grouper performs best. Followed by the approach of Craz.
The benchmark was created with simple_benchmark1. The code used to run this benchmark was:
import iteration_utilities
import itertools
from itertools import zip_longest
def consume_all(it):
return iteration_utilities.consume(it, None)
import simple_benchmark
b = simple_benchmark.BenchmarkBuilder()
#b.add_function()
def grouper(l, n):
return consume_all(iteration_utilities.grouper(l, n))
def Craz_inner(iterable, n, fillvalue=None):
args = [iter(iterable)] * n
return zip_longest(*args, fillvalue=fillvalue)
#b.add_function()
def Craz(iterable, n, fillvalue=None):
return consume_all(Craz_inner(iterable, n, fillvalue))
def nosklo_inner(seq, size):
return (seq[pos:pos + size] for pos in range(0, len(seq), size))
#b.add_function()
def nosklo(seq, size):
return consume_all(nosklo_inner(seq, size))
def SLott_inner(ints, chunk_size):
for i in range(0, len(ints), chunk_size):
yield ints[i:i+chunk_size]
#b.add_function()
def SLott(ints, chunk_size):
return consume_all(SLott_inner(ints, chunk_size))
def MarkusJarderot1_inner(iterable,size):
it = iter(iterable)
chunk = tuple(itertools.islice(it,size))
while chunk:
yield chunk
chunk = tuple(itertools.islice(it,size))
#b.add_function()
def MarkusJarderot1(iterable,size):
return consume_all(MarkusJarderot1_inner(iterable,size))
def MarkusJarderot2_inner(iterable,size,filler=None):
it = itertools.chain(iterable,itertools.repeat(filler,size-1))
chunk = tuple(itertools.islice(it,size))
while len(chunk) == size:
yield chunk
chunk = tuple(itertools.islice(it,size))
#b.add_function()
def MarkusJarderot2(iterable,size):
return consume_all(MarkusJarderot2_inner(iterable,size))
#b.add_arguments()
def argument_provider():
for exp in range(2, 20):
size = 2**exp
yield size, simple_benchmark.MultiArgument([[0] * size, 10])
r = b.run()
1 Disclaimer: I'm the author of the libraries iteration_utilities and simple_benchmark.
With Python 3.8 you can use the walrus operator and itertools.islice.
from itertools import islice
list_ = [i for i in range(10, 100)]
def chunker(it, size):
iterator = iter(it)
while chunk := list(islice(iterator, size)):
print(chunk)
In [2]: chunker(list_, 10)
[10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
[20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
[30, 31, 32, 33, 34, 35, 36, 37, 38, 39]
[40, 41, 42, 43, 44, 45, 46, 47, 48, 49]
[50, 51, 52, 53, 54, 55, 56, 57, 58, 59]
[60, 61, 62, 63, 64, 65, 66, 67, 68, 69]
[70, 71, 72, 73, 74, 75, 76, 77, 78, 79]
[80, 81, 82, 83, 84, 85, 86, 87, 88, 89]
[90, 91, 92, 93, 94, 95, 96, 97, 98, 99]
I needed a solution that would also work with sets and generators. I couldn't come up with anything very short and pretty, but it's quite readable at least.
def chunker(seq, size):
res = []
for el in seq:
res.append(el)
if len(res) == size:
yield res
res = []
if res:
yield res
List:
>>> list(chunker([i for i in range(10)], 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
Set:
>>> list(chunker(set([i for i in range(10)]), 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
Generator:
>>> list(chunker((i for i in range(10)), 3))
[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
The more-itertools package has chunked method which does exactly that:
import more_itertools
for s in more_itertools.chunked(range(9), 4):
print(s)
Prints
[0, 1, 2, 3]
[4, 5, 6, 7]
[8]
chunked returns the items in a list. If you'd prefer iterables, use ichunked.
The ideal solution for this problem works with iterators (not just sequences). It should also be fast.
This is the solution provided by the documentation for itertools:
def grouper(n, iterable, fillvalue=None):
#"grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
args = [iter(iterable)] * n
return itertools.izip_longest(fillvalue=fillvalue, *args)
Using ipython's %timeit on my mac book air, I get 47.5 us per loop.
However, this really doesn't work for me since the results are padded to be even sized groups. A solution without the padding is slightly more complicated. The most naive solution might be:
def grouper(size, iterable):
i = iter(iterable)
while True:
out = []
try:
for _ in range(size):
out.append(i.next())
except StopIteration:
yield out
break
yield out
Simple, but pretty slow: 693 us per loop
The best solution I could come up with uses islice for the inner loop:
def grouper(size, iterable):
it = iter(iterable)
while True:
group = tuple(itertools.islice(it, None, size))
if not group:
break
yield group
With the same dataset, I get 305 us per loop.
Unable to get a pure solution any faster than that, I provide the following solution with an important caveat: If your input data has instances of filldata in it, you could get wrong answer.
def grouper(n, iterable, fillvalue=None):
#"grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
args = [iter(iterable)] * n
# itertools.zip_longest on Python 3
for x in itertools.izip_longest(*args, fillvalue=fillvalue):
if x[-1] is fillvalue:
yield tuple(v for v in x if v is not fillvalue)
else:
yield x
I really don't like this answer, but it is significantly faster. 124 us per loop
from itertools import izip_longest
def chunker(iterable, chunksize, filler):
return izip_longest(*[iter(iterable)]*chunksize, fillvalue=filler)
Similar to other proposals, but not exactly identical, I like doing it this way, because it's simple and easy to read:
it = iter([1, 2, 3, 4, 5, 6, 7, 8, 9])
for chunk in zip(it, it, it, it):
print chunk
>>> (1, 2, 3, 4)
>>> (5, 6, 7, 8)
This way you won't get the last partial chunk. If you want to get (9, None, None, None) as last chunk, just use izip_longest from itertools.
Since nobody's mentioned it yet here's a zip() solution:
>>> def chunker(iterable, chunksize):
... return zip(*[iter(iterable)]*chunksize)
It works only if your sequence's length is always divisible by the chunk size or you don't care about a trailing chunk if it isn't.
Example:
>>> s = '1234567890'
>>> chunker(s, 3)
[('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9')]
>>> chunker(s, 4)
[('1', '2', '3', '4'), ('5', '6', '7', '8')]
>>> chunker(s, 5)
[('1', '2', '3', '4', '5'), ('6', '7', '8', '9', '0')]
Or using itertools.izip to return an iterator instead of a list:
>>> from itertools import izip
>>> def chunker(iterable, chunksize):
... return izip(*[iter(iterable)]*chunksize)
Padding can be fixed using #ΤΖΩΤΖΙΟΥ's answer:
>>> from itertools import chain, izip, repeat
>>> def chunker(iterable, chunksize, fillvalue=None):
... it = chain(iterable, repeat(fillvalue, chunksize-1))
... args = [it] * chunksize
... return izip(*args)
Another approach would be to use the two-argument form of iter:
from itertools import islice
def group(it, size):
it = iter(it)
return iter(lambda: tuple(islice(it, size)), ())
This can be adapted easily to use padding (this is similar to Markus Jarderot’s answer):
from itertools import islice, chain, repeat
def group_pad(it, size, pad=None):
it = chain(iter(it), repeat(pad))
return iter(lambda: tuple(islice(it, size)), (pad,) * size)
These can even be combined for optional padding:
_no_pad = object()
def group(it, size, pad=_no_pad):
if pad == _no_pad:
it = iter(it)
sentinel = ()
else:
it = chain(iter(it), repeat(pad))
sentinel = (pad,) * size
return iter(lambda: tuple(islice(it, size)), sentinel)
If the list is large, the highest-performing way to do this will be to use a generator:
def get_chunk(iterable, chunk_size):
result = []
for item in iterable:
result.append(item)
if len(result) == chunk_size:
yield tuple(result)
result = []
if len(result) > 0:
yield tuple(result)
for x in get_chunk([1,2,3,4,5,6,7,8,9,10], 3):
print x
(1, 2, 3)
(4, 5, 6)
(7, 8, 9)
(10,)
Using little functions and things really doesn't appeal to me; I prefer to just use slices:
data = [...]
chunk_size = 10000 # or whatever
chunks = [data[i:i+chunk_size] for i in xrange(0,len(data),chunk_size)]
for chunk in chunks:
...
Using map() instead of zip() fixes the padding issue in J.F. Sebastian's answer:
>>> def chunker(iterable, chunksize):
... return map(None,*[iter(iterable)]*chunksize)
Example:
>>> s = '1234567890'
>>> chunker(s, 3)
[('1', '2', '3'), ('4', '5', '6'), ('7', '8', '9'), ('0', None, None)]
>>> chunker(s, 4)
[('1', '2', '3', '4'), ('5', '6', '7', '8'), ('9', '0', None, None)]
>>> chunker(s, 5)
[('1', '2', '3', '4', '5'), ('6', '7', '8', '9', '0')]
One-liner, adhoc solution to iterate over a list x in chunks of size 4 -
for a, b, c, d in zip(x[0::4], x[1::4], x[2::4], x[3::4]):
... do something with a, b, c and d ...
To avoid all conversions to a list import itertools and:
>>> for k, g in itertools.groupby(xrange(35), lambda x: x/10):
... list(g)
Produces:
...
0 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
1 [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
2 [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
3 [30, 31, 32, 33, 34]
>>>
I checked groupby and it doesn't convert to list or use len so I (think) this will delay resolution of each value until it is actually used. Sadly none of the available answers (at this time) seemed to offer this variation.
Obviously if you need to handle each item in turn nest a for loop over g:
for k,g in itertools.groupby(xrange(35), lambda x: x/10):
for i in g:
# do what you need to do with individual items
# now do what you need to do with the whole group
My specific interest in this was the need to consume a generator to submit changes in batches of up to 1000 to the gmail API:
messages = a_generator_which_would_not_be_smart_as_a_list
for idx, batch in groupby(messages, lambda x: x/1000):
batch_request = BatchHttpRequest()
for message in batch:
batch_request.add(self.service.users().messages().modify(userId='me', id=message['id'], body=msg_labels))
http = httplib2.Http()
self.credentials.authorize(http)
batch_request.execute(http=http)
Unless I misses something, the following simple solution with generator expressions has not been mentioned. It assumes that both the size and the number of chunks are known (which is often the case), and that no padding is required:
def chunks(it, n, m):
"""Make an iterator over m first chunks of size n.
"""
it = iter(it)
# Chunks are presented as tuples.
return (tuple(next(it) for _ in range(n)) for _ in range(m))
In your second method, I would advance to the next group of 4 by doing this:
ints = ints[4:]
However, I haven't done any performance measurement so I don't know which one might be more efficient.
Having said that, I would usually choose the first method. It's not pretty, but that's often a consequence of interfacing with the outside world.
With NumPy it's simple:
ints = array([1, 2, 3, 4, 5, 6, 7, 8])
for int1, int2 in ints.reshape(-1, 2):
print(int1, int2)
output:
1 2
3 4
5 6
7 8
I never want my chunks padded, so that requirement is essential. I find that the ability to work on any iterable is also requirement. Given that, I decided to extend on the accepted answer, https://stackoverflow.com/a/434411/1074659.
Performance takes a slight hit in this approach if padding is not wanted due to the need to compare and filter the padded values. However, for large chunk sizes, this utility is very performant.
#!/usr/bin/env python3
from itertools import zip_longest
_UNDEFINED = object()
def chunker(iterable, chunksize, fillvalue=_UNDEFINED):
"""
Collect data into chunks and optionally pad it.
Performance worsens as `chunksize` approaches 1.
Inspired by:
https://docs.python.org/3/library/itertools.html#itertools-recipes
"""
args = [iter(iterable)] * chunksize
chunks = zip_longest(*args, fillvalue=fillvalue)
yield from (
filter(lambda val: val is not _UNDEFINED, chunk)
if chunk[-1] is _UNDEFINED
else chunk
for chunk in chunks
) if fillvalue is _UNDEFINED else chunks
def chunker(iterable, n):
"""Yield iterable in chunk sizes.
>>> chunks = chunker('ABCDEF', n=4)
>>> chunks.next()
['A', 'B', 'C', 'D']
>>> chunks.next()
['E', 'F']
"""
it = iter(iterable)
while True:
chunk = []
for i in range(n):
try:
chunk.append(next(it))
except StopIteration:
yield chunk
raise StopIteration
yield chunk
if __name__ == '__main__':
import doctest
doctest.testmod()
Yet another answer, the advantages of which are:
1) Easily understandable
2) Works on any iterable, not just sequences (some of the above answers will choke on filehandles)
3) Does not load the chunk into memory all at once
4) Does not make a chunk-long list of references to the same iterator in memory
5) No padding of fill values at the end of the list
That being said, I haven't timed it so it might be slower than some of the more clever methods, and some of the advantages may be irrelevant given the use case.
def chunkiter(iterable, size):
def inneriter(first, iterator, size):
yield first
for _ in xrange(size - 1):
yield iterator.next()
it = iter(iterable)
while True:
yield inneriter(it.next(), it, size)
In [2]: i = chunkiter('abcdefgh', 3)
In [3]: for ii in i:
for c in ii:
print c,
print ''
...:
a b c
d e f
g h
Update:
A couple of drawbacks due to the fact the inner and outer loops are pulling values from the same iterator:
1) continue doesn't work as expected in the outer loop - it just continues on to the next item rather than skipping a chunk. However, this doesn't seem like a problem as there's nothing to test in the outer loop.
2) break doesn't work as expected in the inner loop - control will wind up in the inner loop again with the next item in the iterator. To skip whole chunks, either wrap the inner iterator (ii above) in a tuple, e.g. for c in tuple(ii), or set a flag and exhaust the iterator.
def group_by(iterable, size):
"""Group an iterable into lists that don't exceed the size given.
>>> group_by([1,2,3,4,5], 2)
[[1, 2], [3, 4], [5]]
"""
sublist = []
for index, item in enumerate(iterable):
if index > 0 and index % size == 0:
yield sublist
sublist = []
sublist.append(item)
if sublist:
yield sublist
You can use partition or chunks function from funcy library:
from funcy import partition
for a, b, c, d in partition(4, ints):
foo += a * b * c * d
These functions also has iterator versions ipartition and ichunks, which will be more efficient in this case.
You can also peek at their implementation.
About solution gave by J.F. Sebastian here:
def chunker(iterable, chunksize):
return zip(*[iter(iterable)]*chunksize)
It's clever, but has one disadvantage - always return tuple. How to get string instead?
Of course you can write ''.join(chunker(...)), but the temporary tuple is constructed anyway.
You can get rid of the temporary tuple by writing own zip, like this:
class IteratorExhausted(Exception):
pass
def translate_StopIteration(iterable, to=IteratorExhausted):
for i in iterable:
yield i
raise to # StopIteration would get ignored because this is generator,
# but custom exception can leave the generator.
def custom_zip(*iterables, reductor=tuple):
iterators = tuple(map(translate_StopIteration, iterables))
while True:
try:
yield reductor(next(i) for i in iterators)
except IteratorExhausted: # when any of iterators get exhausted.
break
Then
def chunker(data, size, reductor=tuple):
return custom_zip(*[iter(data)]*size, reductor=reductor)
Example usage:
>>> for i in chunker('12345', 2):
... print(repr(i))
...
('1', '2')
('3', '4')
>>> for i in chunker('12345', 2, ''.join):
... print(repr(i))
...
'12'
'34'
I like this approach. It feels simple and not magical and supports all iterable types and doesn't require imports.
def chunk_iter(iterable, chunk_size):
it = iter(iterable)
while True:
chunk = tuple(next(it) for _ in range(chunk_size))
if not chunk:
break
yield chunk
Quite pythonic here (you may also inline the body of the split_groups function)
import itertools
def split_groups(iter_in, group_size):
return ((x for _, x in item) for _, item in itertools.groupby(enumerate(iter_in), key=lambda x: x[0] // group_size))
for x, y, z, w in split_groups(range(16), 4):
foo += x * y + z * w
Here is a chunker without imports that supports generators:
def chunks(seq, size):
it = iter(seq)
while True:
ret = tuple(next(it) for _ in range(size))
if len(ret) == size:
yield ret
else:
raise StopIteration()
Example of use:
>>> def foo():
... i = 0
... while True:
... i += 1
... yield i
...
>>> c = chunks(foo(), 3)
>>> c.next()
(1, 2, 3)
>>> c.next()
(4, 5, 6)
>>> list(chunks('abcdefg', 2))
[('a', 'b'), ('c', 'd'), ('e', 'f')]

Categories