There is something which is funny, I have the following external function.
def get_A_records(domain, default="A"):
"""
Get the A Records of a domain
"""
try:
resolver = Resolver()
a_record = resolver.query(domain, default)
a_records = list()
for r in a_record:
a_dict = dict()
a_dict["domain"]=domain
a_dict["a_ip"]=r.address
a_dict["rtype"]=default
a_records.append(a_dict)
return a_records
except Exception as e:
print(e)
return {}
When I import this function and i run it in the interactive shell, it was in less than a second
The function is meant to be used in celery. But when I invoke the function in celery
#app.task
def update_domain(domain):
try:
ips_v4 = get_A_records(domain)
except Exception as e:
print("Failure... recorded!")
I get the following error
Why is it like that? And What is an alternative? Thanks!
I finally found a solution a working one. In case someone faces the same problem.
Use asynchronous dns module which is supported in dnspython.
from dns import asyncresolver
import dns
async def async_get_A_records(domain, default):
"""
Asyncrhonous
"""
a_record = await asyncresolver.resolve(domain, default)
a_records = list()
for r in a_record:
a_dict = dict()
a_dict["domain"]=domain
a_dict["a_ip"]=r.address
a_dict["rtype"]=default
a_records.append(a_dict)
return a_records
Now in celery app.task
async def query_domain_a(domain):
result = await async_get_A_records(domain)
print(result)
#app.task
def update_domain(domain):
try:
asyncio.run(query_domain_a(domain, "A"))
asyncio.run(query_domain_a(domain, "AAAA"))
except Domain.DoesNotExist as E:
print(E)
except Exception as e:
print(e)
And you won't see timeouts.
HEADS-UP. It is really discouraged by the celery docs to use asyncio or run none asynchronous task.
I am new to Kafka. I was trying to run the example code stated in https://github.com/confluentinc/confluent-kafka-python/blob/master/examples/avro-cli.py.
As per the code, this is how producer and consumer work:
def produce(topic, conf):
"""
Produce User records
"""
from confluent_kafka.avro import AvroProducer
producer = AvroProducer(conf, default_value_schema=record_schema)
print("Producing user records to topic {}. ^c to exit.".format(topic))
while True:
# Instantiate new User, populate fields, produce record, execute callbacks.
record = User()
try:
record.name = input("Enter name: ")
record.favorite_number = int(input("Enter favorite number: "))
record.favorite_color = input("Enter favorite color: ")
# The message passed to the delivery callback will already be serialized.
# To aid in debugging we provide the original object to the delivery callback.
producer.produce(topic=topic, value=record.to_dict(),
callback=lambda err, msg, obj=record: on_delivery(err, msg, obj))
# Serve on_delivery callbacks from previous asynchronous produce()
producer.poll(0)
except KeyboardInterrupt:
break
except ValueError:
print("Invalid input, discarding record...")
continue
print("\nFlushing records...")
producer.flush()
def consume(topic, conf):
"""
Consume User records
"""
from confluent_kafka.avro import AvroConsumer
from confluent_kafka.avro.serializer import SerializerError
print("Consuming user records from topic {} with group {}. ^c to exit.".format(topic, conf["group.id"]))
c = AvroConsumer(conf, reader_value_schema=record_schema)
c.subscribe([topic])
while True:
try:
msg = c.poll(1)
# There were no messages on the queue, continue polling
if msg is None:
print("There are no messages in the queue.")
continue
if msg.error():
print("Consumer error: {}".format(msg.error()))
continue
record = User(msg.value())
print("name: {}\n\tfavorite_number: {}\n\tfavorite_color: {}\n".format(
record.name, record.favorite_number, record.favorite_color))
except SerializerError as e:
# Report malformed record, discard results, continue polling
print("Message deserialization failed {}".format(e))
continue
except KeyboardInterrupt:
break
print("Shutting down consumer..")
c.close()
When I try to consume the message, the queue appears empty.
I get the following output:
Consuming user records from topic example_avro with group example_avro. ^c to exit.
There are no messages in the queue.
This has led me to suspect that I have not produced message in the first place. I use the same bootstrap servers, schema registry and topic. Can anyone help me understand what am I not understanding correctly?
Currently I am aware that you can pass arguments directly to a glue job if you trigger it directly with start_job_run(). However, what if I want to trigger a workflow that subsequently triggers the job but does some initial stuff first? I believe I can set workflow properties, but in order to retrieve them with get_workflow_run_properties(), I need to have name (easily done) and runId (not easily done) as arguments, which is kind of a catch 22.
My trigger code:
def lambda_handler(event, context):
glue_client = boto3.client("glue")
workflow_name = os.environ["workflow_name"]
try:
response = glue_client.start_workflow_run(Name=workflow_name)
run_id = response["RunId"]
except Exception as e:
logging.error(e)
try:
response = glue_client.put_workflow_run_properties(
Name=workflow_name,
RunId=run_id
RunProperties={
'runId': run_id,
'someOtherParameter": "12345"
}
)
except Exception as e:
logging.error(e)
I need to call a celery task for each GRPC request, and return the result.
In default GRPC implementation, each request is processed in a separate thread from a threadpool.
In my case, the server is supposed to process ~400 requests in batch mode per second. So one request may have to wait 1 second for the result due to the batch processing, which means the size of the threadpool must be larger than 400 to avoid blocking.
Can this be done asynchronously?
Thanks a lot.
class EventReporting(ss_pb2.BetaEventReportingServicer, ss_pb2.BetaDeviceMgtServicer):
def ReportEvent(self, request, context):
res = tasks.add.delay(1,2)
result = res.get() ->here i have to block
return ss_pb2.GeneralReply(message='Hello, %s!' % result.message)
As noted by #Michael in a comment, as of version 1.32, gRPC now supports asyncio in its Python API. If you're using an earlier version, you can still use the asyncio API via the experimental API: from grpc.experimental import aio. An asyncio hello world example has also been added to the gRPC repo. The following code is a copy of the example server:
import logging
import asyncio
from grpc import aio
import helloworld_pb2
import helloworld_pb2_grpc
class Greeter(helloworld_pb2_grpc.GreeterServicer):
async def SayHello(self, request, context):
return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name)
async def serve():
server = aio.server()
helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server)
listen_addr = '[::]:50051'
server.add_insecure_port(listen_addr)
logging.info("Starting server on %s", listen_addr)
await server.start()
await server.wait_for_termination()
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
asyncio.run(serve())
See my other answer for how to implement the client.
It can be done asynchronously if your call to res.get can be done asynchronously (if it is defined with the async keyword).
While grpc.server says it requires a futures.ThreadPoolExecutor, it will actually work with any futures.Executor that calls the behaviors submitted to it on some thread other than the one on which they were passed. Were you to pass to grpc.server a futures.Executor implemented by you that only used one thread to carry out four hundred (or more) concurrent calls to EventReporting.ReportEvent, your server should avoid the kind of blocking that you describe.
In my opinion is good simple implementation async grpc server, same like http based on aiohttp.
import asyncio
from concurrent import futures
import functools
import inspect
import threading
from grpc import _server
def _loop_mgr(loop: asyncio.AbstractEventLoop):
asyncio.set_event_loop(loop)
loop.run_forever()
# If we reach here, the loop was stopped.
# We should gather any remaining tasks and finish them.
pending = asyncio.Task.all_tasks(loop=loop)
if pending:
loop.run_until_complete(asyncio.gather(*pending))
class AsyncioExecutor(futures.Executor):
def __init__(self, *, loop=None):
super().__init__()
self._shutdown = False
self._loop = loop or asyncio.get_event_loop()
self._thread = threading.Thread(target=_loop_mgr, args=(self._loop,),
daemon=True)
self._thread.start()
def submit(self, fn, *args, **kwargs):
if self._shutdown:
raise RuntimeError('Cannot schedule new futures after shutdown')
if not self._loop.is_running():
raise RuntimeError("Loop must be started before any function can "
"be submitted")
if inspect.iscoroutinefunction(fn):
coro = fn(*args, **kwargs)
return asyncio.run_coroutine_threadsafe(coro, self._loop)
else:
func = functools.partial(fn, *args, **kwargs)
return self._loop.run_in_executor(None, func)
def shutdown(self, wait=True):
self._loop.stop()
self._shutdown = True
if wait:
self._thread.join()
# --------------------------------------------------------------------------- #
async def _call_behavior(rpc_event, state, behavior, argument, request_deserializer):
context = _server._Context(rpc_event, state, request_deserializer)
try:
return await behavior(argument, context), True
except Exception as e: # pylint: disable=broad-except
with state.condition:
if e not in state.rpc_errors:
details = 'Exception calling application: {}'.format(e)
_server.logging.exception(details)
_server._abort(state, rpc_event.operation_call,
_server.cygrpc.StatusCode.unknown, _server._common.encode(details))
return None, False
async def _take_response_from_response_iterator(rpc_event, state, response_iterator):
try:
return await response_iterator.__anext__(), True
except StopAsyncIteration:
return None, True
except Exception as e: # pylint: disable=broad-except
with state.condition:
if e not in state.rpc_errors:
details = 'Exception iterating responses: {}'.format(e)
_server.logging.exception(details)
_server._abort(state, rpc_event.operation_call,
_server.cygrpc.StatusCode.unknown, _server._common.encode(details))
return None, False
async def _unary_response_in_pool(rpc_event, state, behavior, argument_thunk,
request_deserializer, response_serializer):
argument = argument_thunk()
if argument is not None:
response, proceed = await _call_behavior(rpc_event, state, behavior,
argument, request_deserializer)
if proceed:
serialized_response = _server._serialize_response(
rpc_event, state, response, response_serializer)
if serialized_response is not None:
_server._status(rpc_event, state, serialized_response)
async def _stream_response_in_pool(rpc_event, state, behavior, argument_thunk,
request_deserializer, response_serializer):
argument = argument_thunk()
if argument is not None:
# Notice this calls the normal `_call_behavior` not the awaitable version.
response_iterator, proceed = _server._call_behavior(
rpc_event, state, behavior, argument, request_deserializer)
if proceed:
while True:
response, proceed = await _take_response_from_response_iterator(
rpc_event, state, response_iterator)
if proceed:
if response is None:
_server._status(rpc_event, state, None)
break
else:
serialized_response = _server._serialize_response(
rpc_event, state, response, response_serializer)
print(response)
if serialized_response is not None:
print("Serialized Correctly")
proceed = _server._send_response(rpc_event, state,
serialized_response)
if not proceed:
break
else:
break
else:
break
_server._unary_response_in_pool = _unary_response_in_pool
_server._stream_response_in_pool = _stream_response_in_pool
if __name__ == '__main__':
server = grpc.server(AsyncioExecutor())
# Add Servicer and Start Server Here
link to original:
https://gist.github.com/seglberg/0b4487b57b4fd425c56ad72aba9971be
I have some code in a python-eve app that retrieves some data from a device and populates a resource when that resource is requested for the first time. Sometimes the code can't successfully connect to the device. In this case, I would like to return an error message that explains this better, rather than just a plain 500 error. Here is the on_fetch_item hook:
def before_returning_item(resource, _id, document):
if resource == "switches":
if "interfaces" not in document.keys():
# retrieve and store switch config if it hasn't been stored yet
r = app.data.driver.db[resource]
try:
switch = prepare_switch_from_document(document)
except socket.timeout:
# raise some more meaningful error with message
pass
interface_list = switch.get_formatted_interfaces()
r.update({'_id': _id}, {'interfaces': interface_list})
document['interfaces'] = interface_list
app.on_fetch_item += before_returning_item
Thanks in advance.
All you have to do is take advantage of Flask's abort method:
from flask import abort
def before_returning_item(resource, _id, document):
if resource == "switches":
if "interfaces" not in document.keys():
# retrieve and store switch config if it hasn't been stored yet
r = app.data.driver.db[resource]
try:
switch = prepare_switch_from_document(document)
except socket.timeout:
# raise some more meaningful error with message
abort(500)
interface_list = switch.get_formatted_interfaces()
r.update({'_id': _id}, {'interfaces': interface_list})
document['interfaces'] = interface_list
app.on_fetch_item += before_returning_item
If you want to add a custom description:
abort(500, description='My custom abort description')
I like to create custom exceptions, and raise those with meaning comments, Eg:
class MyExcept(Exception): pass
def before_returning_item():
...
if error_condition:
raise MyException('detailed explanation')
...
try:
before_returning_item()
except MyException, exc:
if 'device not ready' in str(exc):
print("Device was not rdy...Try agin?")
else:
raise