use itertools.imap in streaming_bulk rather than map about elasticsearch-py HOT 3 CLOSED

def parse_line(line):
    """Turn a raw data file line into es document. """
    ...
    return processed_line

# as opposed to open("foo").readlines() which would read the entire file into memory
# all at once
f = open("some_raw_file_that_wont_fit_in_memory.dat")
data_ready_to_be_indexed = itertools.imap(parse_line, f)

for ok, result in streaming_bulk(client, data_ready_to_be_indexed ...):
    ...