We Keep Coding

How do I include both data and state dependencies in a Prefect flow?

This seems like it should be simple, but I can't figure out how to include both state and data dependencies in a single flow. Here is what I attempted (simplified):
def main():
with Flow("load_data") as flow:
test_results = prepare_file1()
load_file1(test_results)
participants = prepare_file2()
load_file2(participants)
email = flow.add_task(EmailTask(name='email', subject='Flow succeeded!', msg='flow succeeded', email_to='xxx', email_from='xxx', smtp_server='xxx',smtp_port=25, smtp_type='INSECURE',))
flow.set_dependencies(task=email, upstream_tasks=[load_file1,load_file2])
flow.visualize()
I get the following graph:
Which means that load_file1 and load_file2 run twice. Can I just set up an additional dependency so that email runs when the two load tasks finish?

The issue is how you add the task to your Flow. When using tasks from the Prefect task library, it's best to first initialize those and then call those in your Flow as follows:
send_email = EmailTask(name='email', subject='Flow succeeded!', msg='flow succeeded', email_to='xxx', email_from='xxx', smtp_server='xxx', smtp_port=25, smtp_type='INSECURE')
with Flow("load_data") as flow:
send_email()
Or alternatively, do it in one step with double round brackets EmailTask(init_kwargs)(run_kwargs). The first pair of brackets will initialize the task and the second one will call the task by invoking the task's .run() method.
with Flow("load_data") as flow:
EmailTask(name='email', subject='Flow succeeded!', msg='flow succeeded', email_to='xxx', email_from='xxx', smtp_server='xxx', smtp_port=25, smtp_type='INSECURE')()
The full flow example could look as follows:
from prefect import task, Flow
from prefect.tasks.notifications import EmailTask
from prefect.triggers import always_run
#task(log_stdout=True)
def prepare_file1():
print("File1 prepared!")
return "file1"
#task(log_stdout=True)
def prepare_file2():
print("File2 prepared!")
return "file2"
#task(log_stdout=True)
def load_file1(file: str):
print(f"{file} loaded!")
#task(log_stdout=True)
def load_file2(file: str):
print(f"{file} loaded!")
send_email = EmailTask(
name="email",
subject="Flow succeeded!",
msg="flow succeeded",
email_to="xxx",
email_from="xxx",
smtp_server="xxx",
smtp_port=25,
smtp_type="INSECURE",
trigger=always_run,
)
with Flow("load_data") as flow:
test_results = prepare_file1()
load1_task = load_file1(test_results)
participants = prepare_file2()
load2_task = load_file2(participants)
send_email(upstream_tasks=[load1_task, load2_task])
if __name__ == "__main__":
flow.visualize()

Writing a dataset to multiple directories with modin and Ray pauses unexplainably

Problem
I am trying to perform IO operations with multiple directories using ray, modin(with ray backend) and python. The file writes pause and the memory and disk usages do not change at all and the program is blocked.
Setup
I have a ray actor set up as this
import os
os.environ["MODIN_ENGINE"] = "ray" # Modin will use Ray
import ray
import modin.pandas as mpd
from numpy.core import numeric
from tqdm import tqdm
#ray.remote
class DatasetHelper:
# Class Variables (static) are to be written here
#ray.method(num_returns=1)
def get_dataset(self):
return self.dataset
#ray.method(num_returns=1)
def generate_dataset(self):
# generates some dataset and returns a dictionary.
return {'status': 1,
'data_dir': self.data_dir}
#ray.method(num_returns=1)
def get_config(self):
return {
"data_dir": self.data_dir,
"data_map_dir": self.data_map_dir,
"out_dir": self.out_dir
}
def _validate_initialization(self):
# Logic here isnt relevant
if self.data_dir == "" or self.data_map == "" or self.nRows == 42:
return False
return True
def __init__(self, data_dir, data_map_dir, nRows, out_dir):
self.data = {}
self.data_map = {}
self.dataset = mpd.DataFrame()
self.timestamp = []
self.first = True
self.out_dir = out_dir
self.data_dir = data_dir
self.data_map_dir = data_map_dir
self.nRows = nRows
def _extract_data(self):
print('Reading data ...')
for each in os.listdir(self.data_dir):
self.data[each.split('.')[0]] = mpd.read_csv(os.path.join(self.data_dir, each),
header=None,
nrows=self.nRows)
print('Data read successfully ...')
print('Validating times for monotonicity and uniqueness ... ')
for each in tqdm(self.data):
if mpd.to_datetime(self.data[each][0]).is_monotonic and mpd.to_datetime(self.data[each][0]).is_unique:
pass
else:
print('Validation failed for uuid: {}'.format(each))
return
def _extract_data_maps(self):
self.data_map = mpd.read_pickle(self.data_map_dir)
print('Data-Map unpickled successfully ...')
The main logic is structured as shown below,
from functools import cached_property
import os
import threading
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm
from DatasetHelper import DatasetHelper
import gc
import json
import logging
from multiprocessing import Process
import asyncio
import ray
ray.init(
# Limiting the object memory store used by ray.put()
# object_store_memory=20000000000,
# Limiting the memory usage of each worker.
# _memory = (1024.0 * 3) * 0.5,
# Specifiying custom directories for temp and object spilling
_temp_dir=os.path.join("/project/bhavaraj/Anaheim/ray_tmp"),
_system_config={
"object_spilling_config": json.dumps(
{"type": "filesystem", "params": {
"directory_path": "/project/bhavaraj/Anaheim/ray_plasma"}},
)
},
logging_level=logging.DEBUG,
ignore_reinit_error=True,
num_gpus=1,
num_cpus=40,
dashboard_port=8265
)
write_lock = threading.Lock()
def cache_dataset(loc):
from datetime import datetime
params = ray.get(loc.get_config.remote())
params['out_dir'] = os.getcwd() if params['out_dir'] is None else params['out_dir']
if os.path.exists(params['out_dir']) is False:
os.mkdir(params['out_dir'])
dataset_name = datetime.now().strftime("%H:%M:%S") + \
"_{}_Cache.csv".format(id(params['out_dir']))
print("Writing to file in {}".format(params['out_dir']))
print("Acquiring Lock")
with write_lock:
print("Lock acquired ...")
ray.get(loc.get_dataset.remote()).to_csv(os.path.join(params['out_dir'], dataset_name))
print("Writing to file finished at {}".format(params['out_dir']))
R_DATA_DIR: str = '/data/intermediate/R/'
R_DATA_MAP: str = '/data/external/DataMap/R.pkl'
G_DATA_DIR: str = '/data/intermediate/G/'
G_DATA_MAP: str = 'data/external/DataMap/G.pkl'
B_DATA_DIR: str = '/data/intermediate/B/'
B_DATA_MAP: str = '/data/external/DataMap/B.pkl'
C_DATA_DIR: str = '/data/intermediate/C/'
C_DATA_MAP: str = '/data/external/DataMap/C.pkl'
Z_DATA_DIR: str = '/data/intermediate/Z/'
Z_DATA_MAP: str = '/data/external/DataMap/Z.pkl'
objs_refs = []
n = 50000
b = DatasetHelper.remote(B_DATA_DIR, B_DATA_MAP, n,"./CB")
r = DatasetHelper.remote(R_DATA_DIR, R_DATA_MAP, n, "./LR")
c = DatasetHelper.remote(C_DATA_DIR, C_DATA_MAP, n, "./CC")
g = DatasetHelper.remote(G_DATA_DIR, G_DATA_MAP, n, "./AG")
objs_refs.append(b.generate_dataset.remote())
objs_refs.append(r.generate_dataset.remote())
objs_refs.append(c.generate_dataset.remote())
objs_refs.append(r.generate_dataset.remote())
objs_refs.append(g.generate_dataset.remote())
generate_outs = ray.get([x for x in objs_refs])
print("Printing dataset generation results...")
for each in generate_outs:
print(each)
# I also tried placing these methods inside the actor but the same issue persists
cache_dataset(b)
cache_dataset(r)
cache_dataset(c)
cache_dataset(g)
I tried decorating the cache_dataset() method with #remote and calling the method as below,
locs = [b, r, c, g]
ray.get([cache_dataset.remote(each) for each in locs])
Output
There are no errors with file writes but the programs pauses execution.
2021-09-20 08:32:53,024 DEBUG node.py:890 -- Process STDOUT and STDERR is being redirected to /project/bhavaraj/Anaheim/ray_tmp/session_2021-09-20_08-32-53_008570_36561/logs.
2021-09-20 08:32:53,172 DEBUG services.py:652 -- Waiting for redis server at 127.0.0.1:6379 to respond...
2021-09-20 08:32:53,334 DEBUG services.py:652 -- Waiting for redis server at 127.0.0.1:44291 to respond...
2021-09-20 08:32:53,340 DEBUG services.py:1043 -- Starting Redis shard with 10.0 GB max memory.
2021-09-20 08:33:01,212 INFO services.py:1263 -- View the Ray dashboard at http://127.0.0.1:8265
2021-09-20 08:33:01,216 DEBUG node.py:911 -- Process STDOUT and STDERR is being redirected to /project/bhavaraj/Anaheim/ray_tmp/session_2021-09-20_08-32-53_008570_36561/logs.
2021-09-20 08:33:01,221 DEBUG services.py:1788 -- Determine to start the Plasma object store with 76.48 GB memory using /dev/shm.
2021-09-20 08:33:01,314 DEBUG services.py:652 -- Waiting for redis server at 10.2.1.35:6379 to respond...
(pid=36906) Dataset shape: (100340, 41)
(pid=36913) Dataset shape: (150692, 40)
(pid=36902) Dataset shape: (103949, 41)
(pid=36910) Dataset shape: (420269, 41)
Printing dataset generation results... # prints the results correctly
Writing to file in ./CB
Acquiring Lock
Lock acquired ...
Writing to file finished at ./CB
Writing to file in ./LR
Acquiring Lock
Lock acquired ...
2021-09-20 08:43:02,612 DEBUG (unknown file):0 -- gc.collect() freed 115 refs in 0.23721289704553783 seconds
Hypothesis
I am thinking that the ray engine is stopping before all of the tasks have finished execution. I do not know how to prove or validate this hypothesis.
I also know that ray.get is supposed to block execution till all the tasks have finished executing.
There is a deadlock "like" situation somewhere.
References
https://docs.ray.io/en/latest/actors.html
https://towardsdatascience.com/writing-your-first-distributed-python-application-with-ray-4248ebc07f41

For any future readers,
modin.DataFrame.to_csv() pauses unexplainably for unknown reasons, but modin.Dataframe.to pickle() doesnt with the same logic.
There is also a significant performance increase in terms of read/write times, when data is stored as .pkl files.

How to detect SSTP traffic?

I'm writing a program which should detect a VPN traffic.
As far as I read about the subject, it seems the detection of the tunneling protocol is easy as firewall rules, using their dedicated ports:
PPTP: port 1723/TCP
OpenVPN: port 1194
L2TP: port 1701/UDP
My problem is with the SSTP, because it is using port 443, which is widely used.
So I have 2 questions:
Am I too naive to think I can detect those VPNs tunneling protocols only by their ports?
Does anyone has any idea how to detect SSTP and differ its traffic from any other type / app which uses TLS/SSL (even using DPI)?
I'm attaching piece of Python code which detects the communication in the above ports
import dpkt
import socket
# -------------------------- Globals
# VPN PORTS
import common
import dal
protocols_strs = {"pp2e_gre": "1723/TCP PP2P_GRE_PORT",
"openvpn": "1194 OPENVPN_PORT",
"ike": "500/UDP IKE_PORT",
"l2tp_ipsec": "1701/UDP L2TP_IPSEC_PORT"
}
port_protocols = {1723: 'pp2e_gre',
1194: 'openvpn',
500: 'ike',
1701: 'l2tp_ipsec'
}
# Dict of sets holding the protocols sessions
protocol_sessions = {"pp2e_gre": [],
"openvpn": [],
"ike": [],
"l2tp_ipsec": []}
# -------------------------- Functions
def is_bidirectional(five_tuple, protocol):
"""
Given a tuple and protocol check if the connection is bidirectional in the protocol
:param five_tuple:
:return: True of the connection is bidirectional False otherwise
"""
src_ip = five_tuple['src_ip']
dest_ip = five_tuple['dest_ip']
# Filter the sessions the five tuple's ips spoke in
ike_sessions = filter(lambda session: (session['src_ip'] == src_ip and session['dest_ip'] == dest_ip)
or
(session['dest_ip'] == src_ip and session['src_ip'] == dest_ip),
protocol_sessions[protocol])
# Return true if 2 session (1 for each direction) were found
return len(ike_sessions) == 2
def print_alert(timestamp, protocol, five_tuple):
"""
Print alert description to std
:param timestamp:
:param protocol:
:param five_tuple:
:return:
"""
print timestamp, ":\t detected port %s communication (%s:%s ---> %s:%s)" % \
(protocol, five_tuple['src_ip'], five_tuple['src_port'], five_tuple['dest_ip'],
five_tuple['dest_port'])
def pp2e_gre_openvpn_ike_handler(five_tuple):
# Get protocol
protocol = five_tuple['protocol']
# Clear old sessions in db
dal.remove_old_sessions(five_tuple['timestamp'], 'vpn_sessions')
# Clear old sessions in cache
protocol_sessions[protocol] = common.clear_old_sessions(five_tuple, protocol_sessions[protocol])
# If session already exists - return
if common.check_if_session_exists(five_tuple, protocol_sessions[protocol]):
session_to_update = common.get_session(five_tuple, protocol_sessions[protocol])
session_to_update['timestamp'] = five_tuple['timestamp']
return
# Update DB
dal.upsert_vpn_session(five_tuple)
# Add to cache
protocol_sessions[protocol].append(five_tuple)
# Print alert
print_alert(five_tuple['timestamp'], protocols_strs[protocol], five_tuple)
def l2tp_ipsec_handler(five_tuple):
if five_tuple in protocol_sessions['l2tp_ipsec']:
return
# If bi-directional IKE protocol performed earlier - alert
if not is_bidirectional(five_tuple, 'ike'):
return
protocol_sessions['l2tp_ipsec'].append(five_tuple)
print_alert(five_tuple['timestamp'], protocols_strs['l2tp_ipsec'], five_tuple)
# -------------------------- VPN ports jump tables
tcp_vpn_ports = {1723: pp2e_gre_openvpn_ike_handler,
1194: pp2e_gre_openvpn_ike_handler}
udp_vpn_ports = {500: pp2e_gre_openvpn_ike_handler,
1701: l2tp_ipsec_handler,
1194: pp2e_gre_openvpn_ike_handler}
# -------------------------- Functions
def process_packet(timestamp, packet):
"""
Given a packet process it for detecting a VPN communication
:param packet:
:param timestamp:
:return:
"""
# Parse the input
eth_frame = dpkt.ethernet.Ethernet(packet)
# Check if IP
if eth_frame.type != dpkt.ethernet.ETH_TYPE_IP:
return
# If not IP return
ip_frame = eth_frame.data
# if TCP or UDP
if ip_frame.p not in (dpkt.ip.IP_PROTO_TCP, dpkt.ip.IP_PROTO_UDP):
return
# Extract L3 frame
frame = ip_frame.data
# Extract ports
frame_ports = (frame.sport, frame.dport)
# get VPN ports in session
detected_ports = set(tcp_vpn_ports).intersection(frame_ports)
# If TCP VPN port was not detected return
if not detected_ports:
return
# Get detected port
port = detected_ports.pop()
# Translate port to str
protocol_str = port_protocols[port]
# Choose handler by port
handler = tcp_vpn_ports[port]
# Extract 5-tuple parameters from frames
five_tuple = {'src_ip': socket.inet_ntoa(ip_frame.src),
'dest_ip': socket.inet_ntoa(ip_frame.dst),
'src_port': frame.sport,
'dest_port': frame.dport,
'protocol': protocol_str,
'timestamp': timestamp}
# Invoke the chosen handler
handler(five_tuple)

"Am I too naive to think I can detect those VPNs tunneling protocols only by their ports?:
"The official OpenVPN port number is 1194, but any port number between 1 and 65535 will work. If you don't provide the 'port' option, 1194 will be used.
So if your code is looking for 1194 traffic, as per the dictionary entries, you will capture default Open VPN flows only.
The SSTP message is encrypted with the SSL channel of the HTTPS protocol. So I don't see how you would identify this traffic as it is encrypted. (Source)

Modbus TCP Python Server not working

I am trying to use asynchronous modbus server based to stream multiple values of data but I simply see zeros. Please help. My code is based on
https://pymodbus.readthedocs.io/en/latest/examples/asynchronous-server.html
#!/usr/bin/env python
'''
# Pymodbus Asynchronous Server Example
#----------------------------------------------------------------------#----
#The asynchronous server is a high performance implementation using the
#twisted library as its backend. This allows it to scale to many #thousands
#of nodes which can be helpful for testing monitoring software.
'''
#---------------------------------------------------------------------------#
# import the various server implementations
#---------------------------------------------------------------------------#
from pymodbus.server.async import StartTcpServer
from pymodbus.server.async import StartUdpServer
from pymodbus.server.async import StartSerialServer
from pymodbus.device import ModbusDeviceIdentification
from pymodbus.datastore import ModbusSequentialDataBlock
from pymodbus.datastore import ModbusSlaveContext, ModbusServerContext
from pymodbus.transaction import ModbusRtuFramer, ModbusAsciiFramer
#---------------------------------------------------------------------------#
# configure the service logging
#---------------------------------------------------------------------------#
import logging
logging.basicConfig()
log = logging.getLogger()
log.setLevel(logging.DEBUG)
#---------------------------------------------------------------------------#
# initialize your data store
#---------------------------------------------------------------------------#
# The datastores only respond to the addresses that they are initialized to.
# Therefore, if you initialize a DataBlock to addresses of 0x00 to 0xFF, a
# request to 0x100 will respond with an invalid address exception. This is
# because many devices exhibit this kind of behavior (but not all)::
#
# block = ModbusSequentialDataBlock(0x00, \[0\]*0xff)
#
# Continuing, you can choose to use a sequential or a sparse DataBlock #in
# your data context. The difference is that the sequential has no gaps #in
# the data while the sparse can. Once again, there are devices that #exhibit
# both forms of behavior::
#
# block = ModbusSparseDataBlock({0x00: 0, 0x05: 1})
# block = ModbusSequentialDataBlock(0x00, \[0\]*5)
#
# Alternately, you can use the factory methods to initialize the #DataBlocks
# or simply do not pass them to have them initialized to 0x00 on the #full
# address range::
#
# store = ModbusSlaveContext(di = #ModbusSequentialDataBlock.create())
# store = ModbusSlaveContext()
#
# Finally, you are allowed to use the same DataBlock reference for #every
# table or you you may use a seperate DataBlock for each table. This #depends
# if you would like functions to be able to access and modify the same #data
# or not::
#
# block = ModbusSequentialDataBlock(0x00, \[0\]*0xff)
# store = ModbusSlaveContext(di=block, co=block, hr=block,ir=block)
#
# The server then makes use of a server context that allows the server #to
# respond with different slave contexts for different unit ids. By #default
# it will return the same context for every unit id supplied (broadcast
# mode). However, this can be overloaded by setting the single flag to #False
# and then supplying a dictionary of unit id to context mapping::
#
# slaves = {
# 0x01: ModbusSlaveContext(...),
# 0x02: ModbusSlaveContext(...),
# 0x03: ModbusSlaveContext(...),
# }
# context = ModbusServerContext(slaves=slaves, single=False)
#
# The slave context can also be initialized in zero_mode which means #that a
# request to address(0-7) will map to the address (0-7). The default is
# False which is based on section 4.4 of the specification, so address(0-7)
# will map to (1-8)::
#
# store = ModbusSlaveContext(..., zero_mode=True)
#---------------------------------------------------------------------------#
store = ModbusSlaveContext(
di = ModbusSequentialDataBlock(5, \[15\]*100),
co = ModbusSequentialDataBlock(6, \[16\]*100),
hr = ModbusSequentialDataBlock(7, \[17\]*100),
ir = ModbusSequentialDataBlock(8, \[18\]*100))
#context = ModbusServerContext(slaves=store, single=True)
slaves = {
0x01: ModbusSlaveContext(1, \[15\]*100),
0x02: ModbusSlaveContext(2, \[16\]*100),
0x03: ModbusSlaveContext(3, \[17\]*100),
0x04: ModbusSlaveContext(4, \[15\]*100),
0x05: ModbusSlaveContext(5, \[16\]*100),
0x06: ModbusSlaveContext(6, \[17\]*100),
}
context = ModbusServerContext(slaves=slaves, single=False)
#---------------------------------------------------------------------------#
# initialize the server information
#---------------------------------------------------------------------------#
# If you don't set this or any fields, they are defaulted to empty strings.
#---------------------------------------------------------------------------#
identity = ModbusDeviceIdentification()
identity.VendorName = 'Pymodbus'
identity.ProductCode = 'PM'
identity.VendorUrl = 'http://github.com/bashwork/pymodbus/'
identity.ProductName = 'Pymodbus Server'
identity.ModelName = 'Pymodbus Server'
identity.MajorMinorRevision = '1.0'
#---------------------------------------------------------------------------#
# run the server you want
#---------------------------------------------------------------------------#
StartTcpServer(context, identity=identity, address=("localhost", 502))
#StartTcpServer(context, identity=identity, address=("localhost", 5020))
#StartUdpServer(context, identity=identity, address=("localhost", 502))
#StartSerialServer(context, identity=identity, port='/dev/pts/3', framer=ModbusRtuFramer)
#StartSerialServer(context, identity=identity, port='/dev/pts/3', framer=ModbusAsciiFramer)

Memory limit hit with appengine-mapreduce

I'm working on appengine-mapreduce function and have modified the demo to fit my purpose.
Basically I have a million over lines in the following format: userid, time1, time2. My purpose is to find the difference between time1 and time2 for each userid.
However, as I run this on Google App Engine, I encountered this error message in the logs section:
Exceeded soft private memory limit with 180.56 MB after servicing 130 requests total
While handling this request, the process that handled this request was found to be using too much memory and was terminated. This is likely to cause a new process to be used for the next request to your application. If you see this message frequently, you may have a memory leak in your application.
def time_count_map(data):
"""Time count map function."""
(entry, text_fn) = data
text = text_fn()
try:
q = text.split('\n')
for m in q:
reader = csv.reader([m.replace('\0', '')], skipinitialspace=True)
for s in reader:
"""Calculate time elapsed"""
sdw = s[1]
start_date = time.strptime(sdw,"%m/%d/%y %I:%M:%S%p")
edw = s[2]
end_date = time.strptime(edw,"%m/%d/%y %I:%M:%S%p")
time_difference = time.mktime(end_date) - time.mktime(start_date)
yield (s[0], time_difference)
except IndexError, e:
logging.debug(e)
def time_count_reduce(key, values):
"""Time count reduce function."""
time = 0.0
for subtime in values:
time += float(subtime)
realtime = int(time)
yield "%s: %d\n" % (key, realtime)
Can anyone suggest how else I can optimize my code better? Thanks!!
Edited:
Here's the pipeline handler:
class TimeCountPipeline(base_handler.PipelineBase):
"""A pipeline to run Time count demo.
Args:
blobkey: blobkey to process as string. Should be a zip archive with
text files inside.
"""
def run(self, filekey, blobkey):
logging.debug("filename is %s" % filekey)
output = yield mapreduce_pipeline.MapreducePipeline(
"time_count",
"main.time_count_map",
"main.time_count_reduce",
"mapreduce.input_readers.BlobstoreZipInputReader",
"mapreduce.output_writers.BlobstoreOutputWriter",
mapper_params={
"blob_key": blobkey,
},
reducer_params={
"mime_type": "text/plain",
},
shards=32)
yield StoreOutput("TimeCount", filekey, output)
Mapreduce.yaml:
mapreduce:
- name: Make messages lowercase
params:
- name: done_callback
value: /done
mapper:
handler: main.lower_case_posts
input_reader: mapreduce.input_readers.DatastoreInputReader
params:
- name: entity_kind
default: main.Post
- name: processing_rate
default: 100
- name: shard_count
default: 4
- name: Make messages upper case
params:
- name: done_callback
value: /done
mapper:
handler: main.upper_case_posts
input_reader: mapreduce.input_readers.DatastoreInputReader
params:
- name: entity_kind
default: main.Post
- name: processing_rate
default: 100
- name: shard_count
default: 4
The rest of the files are exactly the same as the demo.
I've uploaded a copy of my codes on dropbox: http://dl.dropbox.com/u/4288806/demo%20compressed%20fail%20memory.zip

Also consider calling gc.collect() at regular points during your code. I've seen several SO questions about exceeding soft memory limits that were alleviated by calling gc.collect(), most having to do with blobstore.

It is likely your input file exceeds the soft memory limit in size. For big files use either BlobstoreLineInputReader or BlobstoreZipLineInputReader.
These input readers pass something different to the map function, they pass the start_position in the file and the line of text.
Your map function might look something like:
def time_count_map(data):
"""Time count map function."""
text = data[1]
try:
reader = csv.reader([text.replace('\0', '')], skipinitialspace=True)
for s in reader:
"""Calculate time elapsed"""
sdw = s[1]
start_date = time.strptime(sdw,"%m/%d/%y %I:%M:%S%p")
edw = s[2]
end_date = time.strptime(edw,"%m/%d/%y %I:%M:%S%p")
time_difference = time.mktime(end_date) - time.mktime(start_date)
yield (s[0], time_difference)
except IndexError, e:
logging.debug(e)
Using BlobstoreLineInputReader will allow the job to run much faster as it can use more than one shard, up to 256, but it means you need to upload your files uncompressed, which can be a pain. I handle it by uploading the compressed files to an EC2 windows server, then decompress and upload from there, since upstream bandwidth is so big.