We Keep Coding

Apache Beam WriteToJdbc : java.lang.IllegalArgumentException: Unknown Coder URN beam:coder:pickled_python:v1

I have apache beam pipeline to read data from a file, process it and ingest it to a RDBMS(MySQL in this case) with help of apache beam library WriteToJdbc.
Data is in ndjson format which is flattened and the lenght of each record may vary.
Pipeline:
import argparse
import json
import logging
from sys import argv
import apache_beam as beam
import flatsplode
from apache_beam.io.jdbc import WriteToJdbc
from apache_beam.options.pipeline_options import PipelineOptions
data_targets = {"rdbms": {
"rdbms properties"
}}
class ProcessData(beam.DoFn):
def process(self, element):
element = json.loads(element)
flat_json = flatsplode.flatsplode(element, "_")
for data in flat_json:
yield data
if __name__ == '__main__':
logging.getLogger().setLevel(logging.INFO)
parser = argparse.ArgumentParser()
known_args, pipeline_args = parser.parse_known_args(argv)
pipeline_options = PipelineOptions(pipeline_args)
p = beam.Pipeline(options=PipelineOptions(pipeline_args))
read_data = p | "Read Text" >> beam.io.ReadFromText("data.json")
process_data = read_data | "Process JSON" >> beam.ParDo(ProcessData())
# process_data | beam.Map(print)
process_data | f"Write to RDBMS" >> WriteToJdbc(
table_name=data_targets["rdbms"]["tablename"],
driver_class_name=data_targets["rdbms"]['driver_class_name'],
jdbc_url=data_targets["rdbms"]['jdbc_url'],
username=data_targets["rdbms"]['username'],
password=data_targets["rdbms"]['password'])
p.run()
I got this issue
Traceback (most recent call last):
File "/home/somnath_c_datametica_com/cloud_repo/albertsons/python_ingestion/test.py", line 53, in <module>
process_data | f"Write to RDBMS" >> WriteToJdbc(
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/pvalue.py", line 137, in __or__
return self.pipeline.apply(ptransform, self)
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/pipeline.py", line 655, in apply
return self.apply(
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/pipeline.py", line 666, in apply
return self.apply(transform, pvalueish)
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/pipeline.py", line 712, in apply
pvalueish_result = self.runner.apply(transform, pvalueish, self._options)
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/runners/dataflow/dataflow_runner.py", line 142, in apply
return super().apply(transform, input, options)
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/runners/runner.py", line 185, in apply
return m(transform, input, options)
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/runners/runner.py", line 215, in apply_PTransform
return transform.expand(input)
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/transforms/external.py", line 605, in expand
raise RuntimeError(response.error)
RuntimeError: org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.UncheckedExecutionException: org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.UncheckedExecutionException: java.lang.IllegalArgumentException: Unknown Coder URN beam:coder:pickled_python:v1. Known URNs: [beam:coder:avro:generic:v1, beam:coder:bytes:v1, beam:coder:bool:v1, beam:coder:string_utf8:v1, beam:coder:kv:v1, beam:coder:varint:v1, beam:coder:interval_window:v1, beam:coder:iterable:v1, beam:coder:timer:v1, beam:coder:length_prefix:v1, beam:coder:global_window:v1, beam:coder:windowed_value:v1, beam:coder:param_windowed_value:v1, beam:coder:double:v1, beam:coder:row:v1, beam:coder:sharded_key:v1, beam:coder:custom_window:v1, beam:coder:nullable:v1]
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2050)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache.get(LocalCache.java:3952)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3974)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4958)
at org.apache.beam.runners.core.construction.RehydratedComponents.getPCollection(RehydratedComponents.java:139)
at org.apache.beam.sdk.expansion.service.ExpansionService.lambda$expand$0(ExpansionService.java:497)
at java.base/java.util.stream.Collectors.lambda$uniqKeysMapAccumulator$1(Collectors.java:178)
at java.base/java.util.stream.ReduceOps$3ReducingSink.accept(ReduceOps.java:169)
at java.base/java.util.Collections$UnmodifiableMap$UnmodifiableEntrySet.lambda$entryConsumer$0(Collections.java:1576)
at java.base/java.util.Iterator.forEachRemaining(Iterator.java:133)
at java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1801)
at java.base/java.util.Collections$UnmodifiableMap$UnmodifiableEntrySet$UnmodifiableEntrySetSpliterator.forEachRemaining(Collections.java:1601)
at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484)
at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913)
at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578)
at org.apache.beam.sdk.expansion.service.ExpansionService.expand(ExpansionService.java:492)
at org.apache.beam.sdk.expansion.service.ExpansionService.expand(ExpansionService.java:606)
at org.apache.beam.model.expansion.v1.ExpansionServiceGrpc$MethodHandlers.invoke(ExpansionServiceGrpc.java:305)
at org.apache.beam.vendor.grpc.v1p48p1.io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:182)
at org.apache.beam.vendor.grpc.v1p48p1.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:354)
at org.apache.beam.vendor.grpc.v1p48p1.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:866)
at org.apache.beam.vendor.grpc.v1p48p1.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
at org.apache.beam.vendor.grpc.v1p48p1.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.UncheckedExecutionException: java.lang.IllegalArgumentException: Unknown Coder URN beam:coder:pickled_python:v1. Known URNs: [beam:coder:avro:generic:v1, beam:coder:bytes:v1, beam:coder:bool:v1, beam:coder:string_utf8:v1, beam:coder:kv:v1, beam:coder:varint:v1, beam:coder:interval_window:v1, beam:coder:iterable:v1, beam:coder:timer:v1, beam:coder:length_prefix:v1, beam:coder:global_window:v1, beam:coder:windowed_value:v1, beam:coder:param_windowed_value:v1, beam:coder:double:v1, beam:coder:row:v1, beam:coder:sharded_key:v1, beam:coder:custom_window:v1, beam:coder:nullable:v1]
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2050)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache.get(LocalCache.java:3952)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3974)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4958)
at org.apache.beam.runners.core.construction.RehydratedComponents.getCoder(RehydratedComponents.java:168)
at org.apache.beam.runners.core.construction.PCollectionTranslation.fromProto(PCollectionTranslation.java:51)
at org.apache.beam.runners.core.construction.RehydratedComponents$3.load(RehydratedComponents.java:108)
at org.apache.beam.runners.core.construction.RehydratedComponents$3.load(RehydratedComponents.java:98)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3528)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2277)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2154)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2044)
... 27 more
Caused by: java.lang.IllegalArgumentException: Unknown Coder URN beam:coder:pickled_python:v1. Known URNs: [beam:coder:avro:generic:v1, beam:coder:bytes:v1, beam:coder:bool:v1, beam:coder:string_utf8:v1, beam:coder:kv:v1, beam:coder:varint:v1, beam:coder:interval_window:v1, beam:coder:iterable:v1, beam:coder:timer:v1, beam:coder:length_prefix:v1, beam:coder:global_window:v1, beam:coder:windowed_value:v1, beam:coder:param_windowed_value:v1, beam:coder:double:v1, beam:coder:row:v1, beam:coder:sharded_key:v1, beam:coder:custom_window:v1, beam:coder:nullable:v1]
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument(Preconditions.java:440)
at org.apache.beam.runners.core.construction.CoderTranslation.fromKnownCoder(CoderTranslation.java:165)
at org.apache.beam.runners.core.construction.CoderTranslation.fromProto(CoderTranslation.java:145)
at org.apache.beam.runners.core.construction.RehydratedComponents$2.load(RehydratedComponents.java:87)
at org.apache.beam.runners.core.construction.RehydratedComponents$2.load(RehydratedComponents.java:82)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3528)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2277)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2154)
at org.apache.beam.vendor.guava.v26_0_jre.com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2044)
... 38 more
I tried using
process_data | beam.Map(lambda x: x).with_output_types(typing.Iterable[str]) | f"Write to RDBMS" >> WriteToJdbc(
table_name=data_targets["rdbms"]["tablename"],
driver_class_name=data_targets["rdbms"]['driver_class_name'],
jdbc_url=data_targets["rdbms"]['jdbc_url'],
username=data_targets["rdbms"]['username'],
password=data_targets["rdbms"]['password'])
but got the following error
Traceback (most recent call last):
File "/home/somnath_c_datametica_com/cloud_repo/albertsons/python_ingestion/test.py", line 46, in <module>
process_data | beam.Map(lambda x: x).with_output_types(typing.Iterable[str]) | f"Write to RDBMS" >> WriteToJdbc(
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/pvalue.py", line 137, in __or__
return self.pipeline.apply(ptransform, self)
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/pipeline.py", line 655, in apply
return self.apply(
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/pipeline.py", line 666, in apply
return self.apply(transform, pvalueish)
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/pipeline.py", line 712, in apply
pvalueish_result = self.runner.apply(transform, pvalueish, self._options)
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/runners/dataflow/dataflow_runner.py", line 142, in apply
return super().apply(transform, input, options)
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/runners/runner.py", line 185, in apply
return m(transform, input, options)
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/runners/runner.py", line 215, in apply_PTransform
return transform.expand(input)
File "/home/somnath_c_datametica_com/.local/lib/python3.9/site-packages/apache_beam/transforms/external.py", line 605, in expand
raise RuntimeError(response.error)
RuntimeError: java.lang.IllegalStateException: Cannot call getSchema when there is no schema
at org.apache.beam.sdk.values.PCollection.getSchema(PCollection.java:331)
at org.apache.beam.sdk.io.jdbc.JdbcSchemaIOProvider$JdbcSchemaIO$2.expand(JdbcSchemaIOProvider.java:146)
at org.apache.beam.sdk.io.jdbc.JdbcSchemaIOProvider$JdbcSchemaIO$2.expand(JdbcSchemaIOProvider.java:140)
at org.apache.beam.sdk.Pipeline.applyInternal(Pipeline.java:548)
at org.apache.beam.sdk.Pipeline.applyTransform(Pipeline.java:499)
at org.apache.beam.sdk.expansion.service.ExpansionService$TransformProvider.apply(ExpansionService.java:400)
at org.apache.beam.sdk.expansion.service.ExpansionService.expand(ExpansionService.java:526)
at org.apache.beam.sdk.expansion.service.ExpansionService.expand(ExpansionService.java:606)
at org.apache.beam.model.expansion.v1.ExpansionServiceGrpc$MethodHandlers.invoke(ExpansionServiceGrpc.java:305)
at org.apache.beam.vendor.grpc.v1p48p1.io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:182)
at org.apache.beam.vendor.grpc.v1p48p1.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:354)
at org.apache.beam.vendor.grpc.v1p48p1.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:866)
at org.apache.beam.vendor.grpc.v1p48p1.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
at org.apache.beam.vendor.grpc.v1p48p1.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
To overcome this I used
ExampleRow = typing.NamedTuple('ExampleRow', pcoll_schema)
coders.registry.register_coder(ExampleRow, coders.RowCoder)
but this didn't work either
Can anyone help me with this?

According to the documentation of WriteToJdbc it requires the input rows in the form of a NamedTuple (see programming guide):
A PTransform which writes Rows to the specified database via JDBC.
This transform receives Rows defined as NamedTuple type and registered
in the coders registry
which need to be registered in the coder registration. If you allow me to steal the example from WriteToJdbc you need to write something like this
# define NamedTuple
ExampleRow = typing.NamedTuple('ExampleRow',
[('id', int), ('name', unicode)])
# Register Coder
coders.registry.register_coder(ExampleRow, coders.RowCoder)
with TestPipeline() as p:
_ = (
p
| beam.Create([ExampleRow(1, 'abc')])
.with_output_types(ExampleRow) # use name of registered coder as output type
| 'Write to jdbc' >> WriteToJdbc(
driver_class_name='org.postgresql.Driver',
jdbc_url='jdbc:postgresql://localhost:5432/example',
username='postgres',
password='postgres',
statement='INSERT INTO example_table VALUES(?, ?)',
))
If your data is more complex, it might be easier if you don't define the NamedTuple inline, e.g.
import typing
class ExampleRow(typing.NamedTuple):
id: int
name: unicode
and register as in the official example above.

How to display pydatatable frame output in streamlit web app?

I have a script to display datatable frame output in streamlit app as:
import datatable as dt
import streamlit as st
import pandas as pd
st.set_page_config(
page_title="pydatatable demo",
layout="wide",
initial_sidebar_state="expanded")
DT = dt.Frame({
'class':['a','b','c','d','e'],
'score':[1,2,3,4,5]
})
st.table(DT)
On executing this script it sends out an error as:
2021-08-28 05:34:40.905 Traceback (most recent call last):
File "/usr/local/lib/python3.7/dist-packages/streamlit/script_runner.py", line 350, in _run_script
exec(code, module.__dict__)
File "/content/pydt_demo.py", line 16, in <module>
st.table(DT)
File "/usr/local/lib/python3.7/dist-packages/streamlit/elements/dataframe_selector.py", line 118, in table
return self.dg._arrow_table(data)
File "/usr/local/lib/python3.7/dist-packages/streamlit/elements/arrow.py", line 119, in _arrow_table
marshall(proto, data, default_uuid)
File "/usr/local/lib/python3.7/dist-packages/streamlit/elements/arrow.py", line 160, in marshall
proto.data = type_util.data_frame_to_bytes(df)
File "/usr/local/lib/python3.7/dist-packages/streamlit/type_util.py", line 371, in data_frame_to_bytes
table = pa.Table.from_pandas(df)
File "pyarrow/table.pxi", line 1479, in pyarrow.lib.Table.from_pandas
File "/usr/local/lib/python3.7/dist-packages/pyarrow/pandas_compat.py", line 591, in dataframe_to_arrays
for c, f in zip(columns_to_convert, convert_fields)]
File "/usr/local/lib/python3.7/dist-packages/pyarrow/pandas_compat.py", line 591, in <listcomp>
for c, f in zip(columns_to_convert, convert_fields)]
File "/usr/local/lib/python3.7/dist-packages/pyarrow/pandas_compat.py", line 577, in convert_column
raise e
File "/usr/local/lib/python3.7/dist-packages/pyarrow/pandas_compat.py", line 571, in convert_column
result = pa.array(col, type=type_, from_pandas=True, safe=safe)
File "pyarrow/array.pxi", line 301, in pyarrow.lib.array
File "pyarrow/array.pxi", line 83, in pyarrow.lib._ndarray_to_array
File "pyarrow/error.pxi", line 84, in pyarrow.lib.check_status
pyarrow.lib.ArrowInvalid: ('Could not convert | class\n | str32\n-- + -----\n 0 | a \n 1 | b \n 2 | c \n 3 | d \n 4 | e \n[5 rows x 1 column]\n with type datatable.Frame: did not recognize Python value type when inferring an Arrow data type', 'Conversion failed for column 0 with type object')
Here as a workaround if i convert this datatable Frame to pandas frame it shows output perfectly.

It doesn't seem like streamlit.table() supports datatable frames: https://docs.streamlit.io/en/stable/api.html#streamlit.table
So for the moment, the only option is to convert your frame into anything that is supported, i.e. pandas.DataFrame, pandas.Styler, pyarrow.Table, numpy.ndarray, Iterable, dict, or None. Luckily, datatable supports conversion into all of these formats: https://datatable.readthedocs.io/en/latest/api/frame.html#convert-into-other-formats

keyerror when trying to insert datetime into Postgresql (pg8000)

Im trying to insert a datetime into pgsql with
f'UPDATE userdata SET claimed = {fclaim} WHERE userid = {user.id}'
heres how the fclaim variable is made
nextclaim = datetime.datetime.now() + datetime.timedelta(hours=24)
fclaim = nextclaim + datetime.timedelta(hours=24)
When i try to insert fclaim into postgresql i get the following error
File "/home/server/.local/lib/python3.6/site-packages/discord/ext/commands/core.py", line 83, in wrapped
ret = await coro(*args, **kwargs)
File "/home/server/Documents/blutonium/extensions/points.py", line 162, in daily
isClaimed = blutapi.getclaimed(ctx.author)
File "/home/server/Documents/blutonium/blutapi.py", line 445, in getclaimed
db.run(sql)
File "/home/server/.local/lib/python3.6/site-packages/pg8000/core.py", line 954, in run
self._run_cursor.execute(sql, params, stream=stream)
File "/home/server/.local/lib/python3.6/site-packages/pg8000/core.py", line 351, in execute
self._c.execute_unnamed(self, operation, args)
File "/home/server/.local/lib/python3.6/site-packages/pg8000/core.py", line 1262, in execute_unnamed
args = make_args(vals)
File "/home/server/.local/lib/python3.6/site-packages/pg8000/core.py", line 213, in make_args
return tuple(vals[p] for p in placeholders)
File "/home/server/.local/lib/python3.6/site-packages/pg8000/core.py", line 213, in <genexpr>
return tuple(vals[p] for p in placeholders)
KeyError: '37'
using python 3.6.11 on ubuntu 20.04

Following the docs, here's an example of doing an UPDATE with pg8000:
conn.run(
"UPDATE userdata SET claimed = :fclaim WHERE userid = :user_id", fclaim=fclaim,
user_id=user.id)

How to create Google Cloud Dataflow Wordcount custom template in Python?

I can't create a custom Google Cloud Dataflow template using the wordcount example following the instructions here: https://cloud.google.com/dataflow/docs/guides/templates/creating-templates
I get an error relating to the RuntimeValueProvider being unaccessible. What am I doing wrong?
My main function wordcount.py:
"""A word-counting workflow."""
from __future__ import absolute_import
import argparse
import logging
import re
from past.builtins import unicode
import apache_beam as beam
from apache_beam.io import ReadFromText
from apache_beam.io import WriteToText
from apache_beam.metrics import Metrics
from apache_beam.metrics.metric import MetricsFilter
from apache_beam.options.pipeline_options import PipelineOptions, GoogleCloudOptions
from apache_beam.options.pipeline_options import SetupOptions
class WordExtractingDoFn(beam.DoFn):
"""Parse each line of input text into words."""
def __init__(self):
self.words_counter = Metrics.counter(self.__class__, 'words')
self.word_lengths_counter = Metrics.counter(self.__class__, 'word_lengths')
self.word_lengths_dist = Metrics.distribution(
self.__class__, 'word_len_dist')
self.empty_line_counter = Metrics.counter(self.__class__, 'empty_lines')
def process(self, element):
"""Returns an iterator over the words of this element.
The element is a line of text. If the line is blank, note that, too.
Args:
element: the element being processed
Returns:
The processed element.
"""
text_line = element.strip()
if not text_line:
self.empty_line_counter.inc(1)
words = re.findall(r'[\w\']+', text_line, re.UNICODE)
for w in words:
self.words_counter.inc()
self.word_lengths_counter.inc(len(w))
self.word_lengths_dist.update(len(w))
return words
def run(argv=None):
"""Main entry point; defines and runs the wordcount pipeline."""
class WordcountOptions(PipelineOptions):
#classmethod
def _add_argparse_args(cls, parser):
# Use add_value_provider_argument for arguments to be templatable
# Use add_argument as usual for non-templatable arguments
parser.add_value_provider_argument(
'--input',
default='gs://wordcount_custom_template/input/example.txt',
help='Path of the file to read from')
parser.add_value_provider_argument(
'--output',
required=True,
default='gs//wordcount_custom_template/output/count',
help='Output file to write results to.')
pipeline_options = PipelineOptions(['--output', 'some/output_path'])
pipeline_options.view_as(SetupOptions).save_main_session = True
p = beam.Pipeline(options=pipeline_options)
wordcount_options = pipeline_options.view_as(WordcountOptions)
# Read the text file[pattern] into a PCollection.
lines = p | 'read' >> ReadFromText(wordcount_options.input)
# Count the occurrences of each word.
def count_ones(word_ones):
(word, ones) = word_ones
return (word, sum(ones))
counts = (lines
| 'split' >> (beam.ParDo(WordExtractingDoFn())
.with_output_types(unicode))
| 'pair_with_one' >> beam.Map(lambda x: (x, 1))
| 'group' >> beam.GroupByKey()
| 'count' >> beam.Map(count_ones))
# Format the counts into a PCollection of strings.
def format_result(word_count):
(word, count) = word_count
return '%s: %d' % (word, count)
output = counts | 'format' >> beam.Map(format_result)
# Write the output using a "Write" transform that has side effects.
# pylint: disable=expression-not-assigned
output | 'write' >> WriteToText(wordcount_options.output)
result = p.run()
result.wait_until_finish()
# Do not query metrics when creating a template which doesn't run
if (not hasattr(result, 'has_job') # direct runner
or result.has_job): # not just a template creation
empty_lines_filter = MetricsFilter().with_name('empty_lines')
query_result = result.metrics().query(empty_lines_filter)
if query_result['counters']:
empty_lines_counter = query_result['counters'][0]
logging.info('number of empty lines: %d', empty_lines_counter.result)
word_lengths_filter = MetricsFilter().with_name('word_len_dist')
query_result = result.metrics().query(word_lengths_filter)
if query_result['distributions']:
word_lengths_dist = query_result['distributions'][0]
logging.info('average word length: %d', word_lengths_dist.result.mean)
if __name__ == '__main__':
logging.getLogger().setLevel(logging.INFO)
run()
My template creation code:
#!/usr/bin/env bash
python wordcount.py \
--runner DataflowRunner \
--project $PROJECT \
--staging_location gs://wordcount_custom_template/staging \
--temp_location gs://wordcount_custom_template/temp \
--template_location gs://wordcount_custom_template/template/wordcount_template
The error I receive:
raise error.RuntimeValueProviderError('%s not accessible' % obj)
apache_beam.error.RuntimeValueProviderError: RuntimeValueProvider(option: input, type: str, default_value: 'gs://wordcount_custom_template/input/example.txt') not accessible
I don't really understand what this error message means as gs://wordcount_custom_template/input/example.txt is accessible
Full stacktrace:
INFO:root:Missing pipeline option (runner). Executing pipeline using the default runner: DirectRunner.
INFO:root:==================== <function annotate_downstream_side_inputs at 0x108e5fa28> ====================
INFO:root:==================== <function lift_combiners at 0x108e5ff50> ====================
INFO:root:==================== <function expand_gbk at 0x108e5fde8> ====================
INFO:root:==================== <function sink_flattens at 0x108e5fe60> ====================
INFO:root:==================== <function greedily_fuse at 0x108e5f848> ====================
INFO:root:==================== <function sort_stages at 0x108e5faa0> ====================
INFO:root:Running (ref_AppliedPTransform_read/Read_3)+((ref_AppliedPTransform_split_4)+((ref_AppliedPTransform_pair_with_one_5)+(group/Write)))
INFO:root:start <DataOutputOperation group/Write >
INFO:root:start <DoOperation pair_with_one output_tags=['out']>
INFO:root:start <DoOperation split output_tags=['out']>
INFO:root:start <ReadOperation read/Read source=SourceBundle(weight=1.0, source=<apache_beam.io.textio._TextSource object at 0x108cfcd50>, start_position=None, stop_position=None)>
Traceback (most recent call last):
File "wordcount.py", line 121, in <module>
run()
File "wordcount.py", line 100, in run
result = p.run()
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/pipeline.py", line 369, in run
self.to_runner_api(), self.runner, self._options).run(False)
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/pipeline.py", line 382, in run
return self.runner.run_pipeline(self)
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/runners/direct/direct_runner.py", line 129, in run_pipeline
return runner.run_pipeline(pipeline)
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 215, in run_pipeline
return self.run_via_runner_api(pipeline.to_runner_api())
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 218, in run_via_runner_api
return self.run_stages(*self.create_stages(pipeline_proto))
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 837, in run_stages
pcoll_buffers, safe_coders).process_bundle.metrics
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 938, in run_stage
self._progress_frequency).process_bundle(data_input, data_output)
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 1110, in process_bundle
result_future = self._controller.control_handler.push(process_bundle)
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/runners/portability/fn_api_runner.py", line 1003, in push
response = self.worker.do_instruction(request)
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 185, in do_instruction
request.instruction_id)
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 202, in process_bundle
processor.process_bundle(instruction_id)
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 286, in process_bundle
op.start()
File "apache_beam/runners/worker/operations.py", line 227, in apache_beam.runners.worker.operations.ReadOperation.start
File "apache_beam/runners/worker/operations.py", line 228, in apache_beam.runners.worker.operations.ReadOperation.start
File "apache_beam/runners/worker/operations.py", line 229, in apache_beam.runners.worker.operations.ReadOperation.start
File "apache_beam/runners/worker/operations.py", line 231, in apache_beam.runners.worker.operations.ReadOperation.start
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/io/filebasedsource.py", line 197, in get_range_tracker
return self._get_concat_source().get_range_tracker(start_position,
File "/Users/chris/.pyenv/versions/cl2/lib/python2.7/site-packages/apache_beam/options/value_provider.py", line 123, in _f
raise error.RuntimeValueProviderError('%s not accessible' % obj)
apache_beam.error.RuntimeValueProviderError: RuntimeValueProvider(option: input, type: str, default_value: 'gs://wordcount_custom_template/input/example.txt') not accessible
Another thing I don't understand is how can it be that I specify the DataflowRunner yet the DirectRunner is called as shown in the stacktrace?

I successfully generated the pipeline after I modified run(argv) to pick up the args from commandline:
parser = argparse.ArgumentParser()
known_args, pipeline_args = parser.parse_known_args(argv)
pipeline_options = PipelineOptions(pipeline_args)
So I think the problem is that argv is not passed to your program correctly.
Also I think if you'd like to make output a template arg, please do not mark it as required.

Pickling Error PySpark

I am writing a spark streaming application to stream data from s3, do some aggregation and raise appropriate error. I am stuck as I keep getting this error:
Traceback (most recent call last):
File "/home/plivo/Downloads/spark-1.4.0-bin-hadoop2.4/python/lib/pyspark.zip/pyspark/streaming/util.py", line 59, in call
return r._jrdd
File "/home/plivo/Downloads/spark-1.4.0-bin-hadoop2.4/python/lib/pyspark.zip/pyspark/rdd.py", line 2351, in _jrdd
pickled_cmd, bvars, env, includes = _prepare_for_python_RDD(self.ctx, command, self)
File "/home/plivo/Downloads/spark-1.4.0-bin-hadoop2.4/python/lib/pyspark.zip/pyspark/rdd.py", line 2271, in _prepare_for_python_RDD
pickled_command = ser.dumps(command)
File "/home/plivo/Downloads/spark-1.4.0-bin-hadoop2.4/python/lib/pyspark.zip/pyspark/serializers.py", line 427, in dumps
return cloudpickle.dumps(obj, 2)
File "/home/plivo/Downloads/spark-1.4.0-bin-hadoop2.4/python/lib/pyspark.zip/pyspark/cloudpickle.py", line 622, in dumps
cp.dump(obj)
File "/home/plivo/Downloads/spark-1.4.0-bin-hadoop2.4/python/lib/pyspark.zip/pyspark/cloudpickle.py", line 111, in dump
raise pickle.PicklingError(msg)
PicklingError: Could not pickle object as excessively deep recursion required.
Here is the code I am trying it on
import time
from pyspark import SparkContext
from pyspark.streaming import StreamingContext
if __name__ == '__main__':
limit = {'111111':200,'222222':100,'333333':100,'444444':100,'555555':100, '666666':100,}
current_value = { str(x)*6 : [ int(time.time())/60, 0 ] for x in range(1, 7) }
def check(x):
response = client.put_object(Key = 'key', Body = 'body', Bucket = 'bucket')
return True
sc = SparkContext('local[2]', 's3_streaming')
sc._jsc.hadoopConfiguration().set("fs.s3n.awsAccessKeyId","<key>")
sc._jsc.hadoopConfiguration().set("fs.s3n.awsSecretAccessKey","<key>")
ssc = StreamingContext(sc, 10)
rdd = ssc.textFileStream('s3n://sparktest01')
pairs = rdd.map(lambda x: (x.split(',')[0], int(x.split(',')[3])))
aggr = pairs.reduceByKey(lambda x,y: int(x) + int(y))
final = aggr.map(lambda x: (x, check(x) ) )
final.pprint()
ssc.start()
ssc.awaitTermination()