We Keep Coding

python grpc.channel._Rendevezous exception status Unknwon

I have a grpc connection established and i try to make a request in the channel but when i call the service from the client i get the following exception. Does anyone know something about it? Does the usage of threads is the reason for it? I can't figure out what is wrong with it.
This is the protobuf schema with the service:
service P4Runtime {
// Update one or more P4 entities on the target.
rpc Write(WriteRequest) returns (WriteResponse) {
}
// Read one or more P4 entities from the target.
rpc Read(ReadRequest) returns (stream ReadResponse) {
}
// Sets the P4 forwarding-pipeline config.
rpc SetForwardingPipelineConfig(SetForwardingPipelineConfigRequest)
returns (SetForwardingPipelineConfigResponse) {
}
// Gets the current P4 forwarding-pipeline config.
rpc GetForwardingPipelineConfig(GetForwardingPipelineConfigRequest)
returns (GetForwardingPipelineConfigResponse) {
}
// Represents the bidirectional stream between the controller and the
// switch (initiated by the controller), and is managed for the following
// purposes:
// - connection initiation through client arbitration
// - indicating switch session liveness: the session is live when switch
// sends a positive client arbitration update to the controller, and is
// considered dead when either the stream breaks or the switch sends a
// negative update for client arbitration
// - the controller sending/receiving packets to/from the switch
// - streaming of notifications from the switch
rpc StreamChannel(stream StreamMessageRequest)
returns (stream StreamMessageResponse) {
}
rpc Capabilities(CapabilitiesRequest) returns (CapabilitiesResponse) {
}
}
Below is the function that i call and exception happens:
def write_IPv4_Rules(p4info_helper,ingress_sw,ipv4_dst,lpm,dst_mac,out_port):
table_entry = p4info_helper.buildTableEntry(
table_name="MyIngress.ipv4_lpm",
match_fields={
"hdr.ipv4.dstAddr": (ipv4_dst, lpm)
},
action_name="MyIngress.ipv4_forward",
action_params={
"dstAddr": dst_mac,
"port": out_port
})
ingress_sw.WriteTableEntry(table_entry)
print("Installed ipv4 rule on %s" % ingress_sw.name)
This is the invocation of the above function which is inside a thead:
write_IPv4_Rules(p4info_helper,ingress_sw,ip_dest,32,dst_mac,2)
Below i have the code of a controller that uses grpc service:
class SwitchConnection(object):
def __init__(self, name=None, address='127.0.0.1:50051', device_id=0,
proto_dump_file=None):
self.name = name
self.address = address
self.device_id = device_id
self.p4info = None
self.channel = grpc.insecure_channel(self.address)
if proto_dump_file is not None:
interceptor = GrpcRequestLogger(proto_dump_file)
self.channel = grpc.intercept_channel(self.channel, interceptor)
self.client_stub = p4runtime_pb2_grpc.P4RuntimeStub(self.channel)
self.requests_stream = IterableQueue()
self.stream_msg_resp = self.client_stub.StreamChannel(iter(self.requests_stream))
self.proto_dump_file = proto_dump_file
connections.append(self)
def WriteTableEntry(self, table_entry, dry_run=False):
request = p4runtime_pb2.WriteRequest()
request.device_id = self.device_id
request.election_id.low = 1
update = request.updates.add()
if table_entry.is_default_action:
update.type = p4runtime_pb2.Update.MODIFY
else:
update.type = p4runtime_pb2.Update.INSERT
update.entity.table_entry.CopyFrom(table_entry)
if dry_run:
print("P4Runtime Write:", request)
else:
self.client_stub.Write(request)
class GrpcRequestLogger(grpc.UnaryUnaryClientInterceptor,
grpc.UnaryStreamClientInterceptor):
"""Implementation of a gRPC interceptor that logs request to a file"""
def __init__(self, log_file):
self.log_file = log_file
with open(self.log_file, 'w') as f:
# Clear content if it exists.
f.write("")
def log_message(self, method_name, body):
with open(self.log_file, 'a') as f:
ts = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
msg = str(body)
f.write("\n[%s] %s\n---\n" % (ts, method_name))
if len(msg) < MSG_LOG_MAX_LEN:
f.write(str(body))
else:
f.write("Message too long (%d bytes)! Skipping log...\n" % len(msg))
f.write('---\n')
def intercept_unary_unary(self, continuation, client_call_details, request):
self.log_message(client_call_details.method, request)
return continuation(client_call_details, request)
def intercept_unary_stream(self, continuation, client_call_details, request):
self.log_message(client_call_details.method, request)
return continuation(client_call_details, request)
class IterableQueue(Queue):
_sentinel = object()
def __iter__(self):
return iter(self.get, self._sentinel)
def close(self):
self.put(self._sentinel)
The exception that i receive when i run the program:
Exception in thread Thread-11:
Traceback (most recent call last):
File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "./mycontroller.py", line 348, in packet_router_processing
ipv4_forwarding(p4info_helper,extract_header,ingress_sw,packetIn.packet.metadata)
File "./mycontroller.py", line 256, in ipv4_forwarding
write_IPv4_Rules(p4info_helper,ingress_sw,ip_dest,24,dst_mac,1)
File "./mycontroller.py", line 38, in write_IPv4_Rules
ingress_sw.WriteTableEntry(table_entry)
File "/home/p4/tutorials/exercises/test/../../utils/p4runtime_lib/switch.py", line 102, in WriteTableEntry
self.client_stub.Write(request)
File "/usr/local/lib/python3.8/dist-packages/grpc/_interceptor.py", line 207, in __call__
response, ignored_call = self._with_call(
File "/usr/local/lib/python3.8/dist-packages/grpc/_interceptor.py", line 240, in _with_call
call = self._interceptor.intercept_unary_unary(
File "/home/p4/tutorials/exercises/test/../../utils/p4runtime_lib/switch.py", line 220, in intercept_unary_unary
return continuation(client_call_details, request)
File "/usr/local/lib/python3.8/dist-packages/grpc/_interceptor.py", line 228, in continuation
response, call = self._thunk(new_method).with_call(
File "/usr/local/lib/python3.8/dist-packages/grpc/_channel.py", line 557, in with_call
return _end_unary_response_blocking(state, call, True, None)
File "/usr/local/lib/python3.8/dist-packages/grpc/_channel.py", line 466, in _end_unary_response_blocking
raise _Rendezvous(state, None, None, deadline)
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
status = StatusCode.UNKNOWN
details = ""
debug_error_string = "{"created":"#1646087190.862135612","description":"Error received from peer","file":"src/core/lib/surface/call.cc","file_line":1036,"grpc_message":"","grpc_status":2}"

Prometheus counter not yielding in custom collector

I am writing this custome collector where I want to add a counter.
#!/usr/bin/env python3
import sys
import time
from prometheus_client import start_http_server
from prometheus_client.core import CollectorRegistry, Counter
class MyCollector():
def __init__(self):
self.mymetrics_counter = Counter('observability_total', 'Status of My Services', ['app', 'test'])
def describe(self):
print("Started: Metrics Collector!")
return list()
def collect(self):
self.mymetrics_counter.labels('observability', 'test').inc()
yield self.mymetrics_counter
if __name__ == '__main__':
try:
myregistry = CollectorRegistry()
myregistry.register(MyCollector())
start_http_server(port=9100, registry=myregistry)
while True:
time.sleep(10)
except KeyboardInterrupt:
print("Ended: Metrics Collector!")
sys.exit(0)
But I am getting below error upon yeild
(venv) test_collector % python mycollector.py
Started: Metrics Collector!
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/wsgiref/handlers.py", line 137, in run
self.result = application(self.environ, self.start_response)
File "/Users/myid/Documents/myproj/workspace/test_collector/venv/lib/python3.9/site-packages/prometheus_client/exposition.py", line 123, in prometheus_app
status, header, output = _bake_output(registry, accept_header, params)
File "/Users/myid/Documents/myproj/workspace/test_collector/venv/lib/python3.9/site-packages/prometheus_client/exposition.py", line 105, in _bake_output
output = encoder(registry)
File "/Users/myid/Documents/myproj/workspace/test_collector/venv/lib/python3.9/site-packages/prometheus_client/exposition.py", line 179, in generate_latest
mname = metric.name
AttributeError: ("'Counter' object has no attribute 'name'", prometheus_client.metrics.Counter(observability))

collect returns metric families, not metrics. If you yield each of the results of mymetrics_counter.collect() it'd work.
Also, when you create the Counter its getting registered to the default registry which you don't want in this soft of usage as it'll end up returned twice which is invalid.

Unable to make high number of posts on remote machine vs local using asynio and aiohttp

I wrote a program that would post events using asyncio and aiohttp. This program works when I run it locally. I can post 10k events no problem. However, I SCPed the whole codebase to a remote machine and within that machine I can't post more than 15 events without getting this error:
RuntimeError: Event loop is closed
Exception ignored in: <coroutine object Poster.async_post_event at 0x7f4a53989410>
Traceback (most recent call last):
File "/home/bli1/qe-trinity/tracer/utils/poster.py", line 63, in async_post_event
File "/home/bli1/py/python3.5/lib/python3.5/site-packages/aiohttp/client.py", line 565, in __aenter__
File "/home/bli1/py/python3.5/lib/python3.5/site-packages/aiohttp/client.py", line 198, in _request
File "/home/bli1/py/python3.5/lib/python3.5/site-packages/aiohttp/connector.py", line 316, in connect
File "/home/bli1/py/python3.5/lib/python3.5/site-packages/aiohttp/connector.py", line 349, in _release_waiter
File "/home/bli1/py/python3.5/lib/python3.5/asyncio/futures.py", line 332, in set_result
File "/home/bli1/py/python3.5/lib/python3.5/asyncio/futures.py", line 242, in _schedule_callbacks
File "/home/bli1/py/python3.5/lib/python3.5/asyncio/base_events.py", line 447, in call_soon
File "/home/bli1/py/python3.5/lib/python3.5/asyncio/base_events.py", line 456, in _call_soon
File "/home/bli1/py/python3.5/lib/python3.5/asyncio/base_events.py", line 284, in _check_closed
RuntimeError: Event loop is closed
Exception ignored in: <coroutine object Poster.async_post_event at 0x7f4a5397ffc0>
Traceback (most recent call last):
File "/home/bli1/qe-trinity/tracer/utils/poster.py", line 63, in async_post_event
File "/home/bli1/py/python3.5/lib/python3.5/site-packages/aiohttp/client.py", line 565, in __aenter__
File "/home/bli1/py/python3.5/lib/python3.5/site-packages/aiohttp/client.py", line 198, in _request
File "/home/bli1/py/python3.5/lib/python3.5/site-packages/aiohttp/connector.py", line 316, in connect
File "/home/bli1/py/python3.5/lib/python3.5/site-packages/aiohttp/connector.py", line 349, in _release_waiter
File "/home/bli1/py/python3.5/lib/python3.5/asyncio/futures.py", line 332, in set_result
File "/home/bli1/py/python3.5/lib/python3.5/asyncio/futures.py", line 242, in _schedule_callbacks
File "/home/bli1/py/python3.5/lib/python3.5/asyncio/base_events.py", line 447, in call_soon
File "/home/bli1/py/python3.5/lib/python3.5/asyncio/base_events.py", line 456, in _call_soon
File "/home/bli1/py/python3.5/lib/python3.5/asyncio/base_events.py", line 284, in _check_closed
RuntimeError: Event loop is closed
How can I debug this or find out the source of this problem?
Here is the class that I created and I use the method post() to run:
import uuid
import os
import asyncio
import time
import random
import json
import aiohttp
from tracer.utils.phase import Phase
class Poster(Phase):
def __init__(self, log, endpoint, num_post, topic, datafile, timeout, oracles, secure=False, thru_proxy=True):
Phase.__init__(self, log, "post", oracles, secure, thru_proxy)
self.log = log
self.num_post = int(num_post)
self.datafile = datafile.readlines()
self.topic = topic
self.endpoint = self.set_endpoint(endpoint, self.topic)
self.response = None
self.timeout = timeout
def random_line(self):
""" Returns random line from file and converts it to JSON """
return json.loads(random.choice(self.datafile))
#staticmethod
def change_uuid(event):
""" Creates new UUID for event_id """
new_uuid = str(uuid.uuid4())
event["event_header"]["event_id"] = new_uuid
return event
#staticmethod
def wrapevent(event):
""" Wrap event with metadata for analysis later on """
return {
"tracer": {
"post": {
"statusCode": None,
"timestamp": None,
},
"awsKafkaTimestamp": None,
"qdcKakfaTimestamp": None,
"hdfsTimestamp": None
},
"event": event
}
def gen_random_event(self):
random_event = self.random_line()
event = self.change_uuid(random_event)
dataspec = self.wrapevent(event)
return dataspec
async def async_post_event(self, event, session):
async with session.post(self.endpoint, data=event, proxy=self.proxy) as resp:
event["tracer"]["post"]["timestamp"] = time.time() * 1000.0
event["tracer"]["post"]["statusCode"] = resp.status
unique_id = event["event"]["event_header"]["event_id"]
oracle_endpoint = os.path.join(self.oracle, unique_id)
async with session.put(oracle_endpoint, data=json.dumps(event), proxy=self.proxy) as resp:
if resp.status != 200:
self.log.debug("Post to ElasticSearch not 200")
self.log.debug(event["event"]["event_header"]["event_id"])
self.log.debug("Status code: " + str(resp.status))
return event["event"]["event_header"]["event_id"], resp.status
async def async_post_events(self, events):
coros = []
conn = aiohttp.TCPConnector(verify_ssl=self.secure)
async with aiohttp.ClientSession(connector=conn) as session:
for event in events:
coros.append(self.async_post_event(event, session))
return await asyncio.gather(*coros)
def post(self):
event_loop = asyncio.get_event_loop()
try:
events = [self.gen_random_event() for i in range(self.num_post)]
start_time = time.time()
results = event_loop.run_until_complete(self.async_post_events(events))
print("Time taken: " + str(time.time() - start_time))
finally:
event_loop.close()

You cannot re-use a loop once it's closed. From AbstractEventLoop.close documentation:
This is idempotent and irreversible. No other methods should be called after this one.
Either remove the loop.close call or create a new loop for each post.
My advice would be to avoid those problems by running everything inside the loop and awaiting async_post_events when needed.

AttributeError: object has no attribute 'runTest'

I'm am new to unittest and I am not sure why I am getting this error:
runTest (__main__.TestTimeInterval)
No test ... Traceback (most recent call last):
File "/Users/bli1/Development/Trinity/qa-trinity/python_lib/qe/tests/test_timestamp_interval.py", line 122, in <module>
sys.exit(main(sys.argv))
File "/Users/bli1/Development/Trinity/qa-trinity/python_lib/qe/tests/test_timestamp_interval.py", line 110, in main
result_set = runner.run(suite)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/unittest/runner.py", line 168, in run
test(result)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/unittest/suite.py", line 87, in __call__
return self.run(*args, **kwds)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/unittest/suite.py", line 125, in run
test(result)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/unittest/case.py", line 625, in __call__
return self.run(*args, **kwds)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/unittest/case.py", line 555, in run
testMethod = getattr(self, self._testMethodName)
AttributeError: 'TestTimeInterval' object has no attribute 'runTest'
I wanted to create a simple test to see if everything is working properly but I got the error above. I added the test to the suite and then ran it with .run()
class TestTimeInterval(unittest.TestCase):
def __init__(self, log, runtag, interval, path_file):
super(TestTimeInterval, self).__init__()
self.interval = interval
self.path_file = path_file
self.log = log
self.runtag = runtag
def test_record(self):
self.assertTrue(1 > 0)
##############################################################################
def main(argv):
exit_code = 0
global me; me = os.path.basename(argv[0]) # name of this program
global mydir; mydir = os.path.dirname(os.path.abspath(__file__))
parser = argparse.ArgumentParser(description=main.__doc__)
parser.add_argument("file", metavar="FILE",
help="File filled with hdfs paths separated by newlines")
parser.add_argument("runtag", metavar="RUNTAG", help="tag for the test run")
parser.add_argument("-t", "--time", default="10", dest="interval",
help="Time interval (minutes) between server_timestamp and interval given by HDFS folder name")
args = parser.parse_args(args=argv[1:])
log = logging.getLogger(me)
logfile = args.runtag + ".log"
if os.path.exists(logfile):
os.remove(logfile)
log.addHandler(logging.FileHandler(logfile))
console = logging.StreamHandler(sys.stderr); console.setLevel(logging.WARNING); log.addHandler(console)
if exit_code == 0:
runner = unittest.TextTestRunner(stream=sys.stdout, descriptions=True, verbosity=2)
suite = unittest.TestSuite()
print(args)
suite.addTest(TestTimeInterval(log, args.runtag, args.interval, args.file))
try:
log.info("{0}: START: {1}".format(me, datetime.datetime.now().ctime()))
result_set = runner.run(suite)
except KeyboardInterrupt as e:
log.info("{0}: exit on keyboard interrupt".format(me))
exit_code = 1
else:
exit_code = len(result_set.errors) + len(result_set.failures)
finally:
log.info("{0}: FINISH: {1}".format(me, datetime.datetime.now().ctime()))
return exit_code
##############################################################################
# The following code calls main only if this program is invoked standalone
if __name__ == "__main__":
sys.exit(main(sys.argv))

You almost never need to create TestSuites and TestRunners yourself. Normally, you'd do something like:
# my_test.py
import unittest
class Something(object):
def __init__(self):
self.foo = 1
class TestSomething(unittest.TestCase):
def setUp(self):
super(TestSomething, self).setUp()
self.something = Something()
def test_record(self):
self.assertTrue(1 > 0)
def test_something_foo_equals_1(self):
self.assertEqual(self.something.foo, 1)
if __name__ == '__main__':
unittest.main()
now, to run your test, you just execute your script.
python my_test.py

Flask: 'Response' object is not iterable with response-producing exceptions

I can't seem to generate responses from exceptions anymore in Flask 0.10.1 (the same happened with 0.9). This code:
from flask import Flask, jsonify
from werkzeug.exceptions import HTTPException
import flask, werkzeug
print 'Flask version: %s' % flask.__version__
print 'Werkzeug version: %s' % werkzeug.__version__
app = Flask(__name__)
app.config['PROPAGATE_EXCEPTIONS'] = True
class JSONException(HTTPException):
response = None
def get_body(self, environ):
return jsonify(a=1)
def get_headers(self, environ):
return [('Content-Type', 'application/json')]
#app.route('/x')
def x():
return jsonify(a=1)
#app.route('/y')
def y():
raise JSONException()
c = app.test_client()
r = c.get('x')
print r.data
r = c.get('y')
print r.data
prints
Flask version: 0.10.1
Werkzeug version: 0.9.4
{
"a": 1
}
Traceback (most recent call last):
File "flask_error.py", line 33, in <module>
print r.data
File "/home/path/lib/python2.7/site-packages/werkzeug/wrappers.py", line 881, in get_data
self._ensure_sequence()
File "/home/path/lib/python2.7/site-packages/werkzeug/wrappers.py", line 938, in _ensure_sequence
self.make_sequence()
File "/home/path/lib/python2.7/site-packages/werkzeug/wrappers.py", line 953, in make_sequence
self.response = list(self.iter_encoded())
File "/home/path/lib/python2.7/site-packages/werkzeug/wrappers.py", line 81, in _iter_encoded
for item in iterable:
File "/home/path/lib/python2.7/site-packages/werkzeug/wsgi.py", line 682, in __next__
return self._next()
File "/home/path/lib/python2.7/site-packages/werkzeug/wrappers.py", line 81, in _iter_encoded
for item in iterable:
File "/home/path/lib/python2.7/site-packages/werkzeug/wsgi.py", line 682, in __next__
return self._next()
File "/home/path/lib/python2.7/site-packages/werkzeug/wrappers.py", line 81, in _iter_encoded
for item in iterable:
TypeError: 'Response' object is not iterable
The traceback is unexpected.

jsonify() produces a full response object, not a response body, so use HTTPException.get_response(), not .get_body():
class JSONException(HTTPException):
def get_response(self, environ):
return jsonify(a=1)
The alternative is to just use json.dumps() to produce a body here:
class JSONException(HTTPException):
def get_body(self, environ):
return json.dumps({a: 1})
def get_headers(self, environ):
return [('Content-Type', 'application/json')]