We Keep Coding

How can I resolve this error ? pyicloud.exceptions.PyiCloudAPIResponseException: Authentication required for Account. (421)

I've been trying to do a quick Python program to clean event before a certain date.
(I have 2FA)
Here is my try code:
from pyicloud import PyiCloudService
from pyicloud.exceptions import PyiCloudAPIResponseException
import datetime
# Get login credentials for iCloud account
apple_id = "xxxxxxxxxxx#gmail.com" # My iCloud login
password = "xxxx-xxxx-xxxx-xxxx" # The app-specific password
#apple_id = input("Enter your iCloud login: ")
#password = input("Enter your password: ")
# Connect to the iCloud service
api = PyiCloudService(apple_id, password)
api.authenticate()
# Get the calendar data
calendar_service = api.calendar
# The date before which events will be deleted
delete_before = datetime.datetime(2023, 2, 10)
events = calendar_service.events(
from_dt=datetime.datetime(2023, 1, 1),
to_dt=datetime.datetime(2023, 12, 31),
)
# Loop through all events
for event in events:
event_start = datetime.datetime.fromtimestamp(event['start'])
if event_start < delete_before:
# Delete the event
try:
calendar_service.delete(event)
except PyiCloudAPIResponseException as e:
print(f"Error deleting event: {e}")
And here is the error I've been getting:
Traceback (most recent call last):
File "XXXXXX\delete_event_icloud.py", line 22, in <module>
events = calendar_service.events(
File "XXXXXX\AppData\Local\Programs\Python\Python310\lib\site-packages\pyicloud\services\calendar.py", line 64, in events
self.refresh_client(from_dt, to_dt)
File "XXXXXX\AppData\Local\Programs\Python\Python310\lib\site-packages\pyicloud\services\calendar.py", line 57, in refresh_client
req = self.session.get(self._calendar_refresh_url, params=params)
File "XXXXXX\AppData\Local\Programs\Python\Python310\lib\site-packages\requests\sessions.py", line 600, in get
return self.request("GET", url, **kwargs)
File "XXXXXX\AppData\Local\Programs\Python\Python310\lib\site-packages\pyicloud\base.py", line 131, in request
return self.request(method, url, **kwargs)
File "XXXXXX\AppData\Local\Programs\Python\Python310\lib\site-packages\pyicloud\base.py", line 133, in request
self._raise_error(response.status_code, response.reason)
File "XXXXXX\AppData\Local\Programs\Python\Python310\lib\site-packages\pyicloud\base.py", line 189, in _raise_error
raise api_error
pyicloud.exceptions.PyiCloudAPIResponseException: Authentication required for Account. (421)
I've tried to change the app-specific password by creating new one,
I've also tried to add:
"dsid": self.session.service.data["dsInfo"]["dsid"],
to the calendar.py of the pyicloud library, but maybe I did it wrong.
I've tried to use my real password instead of the app-specific.

getting this error ,google.api_core.exceptions.InvalidArgument: 400 Request contains an invalid argument. From google BigQuery Storage Write API

I am trying to pull a huge amount of data (in millions) and I am getting the following error when running my code. If I run the same code with a small range (to be exact a range of 2) it runs successfully. Please assist in helping me know if this is my issue or is coming from the API side
Thanks
The Error I am getting
DEBUG:google.api_core.bidi:Started helper thread Thread-ConsumeBidirectionalStream
DEBUG:google.api_core.bidi:Thread-ConsumeBidirectionalStream caught error 400 Request contains an invalid argument. and will exit. Generally this is due to the RPC itself being cancelled and the error will be surfaced to the calling code.
Traceback (most recent call last):
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/google/api_core/grpc_helpers.py", line 147, in error_remapped_callable
return _StreamingResponseIterator(
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/google/api_core/grpc_helpers.py", line 73, in __init__
self._stored_first_result = next(self._wrapped)
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/grpc/_channel.py", line 426, in __next__
return self._next()
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/grpc/_channel.py", line 826, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.INVALID_ARGUMENT
details = "Request contains an invalid argument."
debug_error_string = "{"created":"#1652904360.179503883","description":"Error received from peer ipv4:173.194.76.95:443","file":"src/core/lib/surface/call.cc","file_line":952,"grpc_message":"Request contains an invalid argument.","grpc_status":3}"
>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/google/api_core/bidi.py", line 636, in _thread_main
self._bidi_rpc.open()
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/google/api_core/bidi.py", line 279, in open
call = self._start_rpc(iter(request_generator), metadata=self._rpc_metadata)
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/google/cloud/bigquery_storage_v1/services/big_query_write/client.py", line 678, in append_rows
response = rpc(
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py", line 154, in __call__
return wrapped_func(*args, **kwargs)
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/google/api_core/retry.py", line 283, in retry_wrapped_func
return retry_target(
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/google/api_core/retry.py", line 190, in retry_target
return target()
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/google/api_core/grpc_helpers.py", line 151, in error_remapped_callable
raise exceptions.from_grpc_error(exc) from exc
google.api_core.exceptions.InvalidArgument: 400 Request contains an invalid argument.
INFO:google.api_core.bidi:Thread-ConsumeBidirectionalStream exiting
DEBUG:google.cloud.bigquery_storage_v1.writer:Finished stopping manager.
Traceback (most recent call last):
File "write_data_to_db2.py", line 207, in <module>
p.append_rows_pending(project_id='dwingestion', dataset_id='ke',
File "write_data_to_db2.py", line 188, in append_rows_pending
response_future_1 = append_rows_stream.send(request)
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/google/cloud/bigquery_storage_v1/writer.py", line 234, in send
return self._open(request)
File "/home/coyugi/teltel_env/lib/python3.8/site-packages/google/cloud/bigquery_storage_v1/writer.py", line 207, in _open
raise request_exception
google.api_core.exceptions.Unknown: None There was a problem opening the stream. Try turning on DEBUG level logs to see the error.
Summary Of My Code
# PULLING DATA FROM THE API
def whole_teltel_raw_data():
# Creating a session to introduce network consistency
session = requests.Session()
retry = Retry(connect=3, backoff_factor=1.0)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
url = "https://my_api_url"
the_headers = {"X-API-KEY": 'my key'}
offset_limit = 1249500
teltel_data = []
# Loop through the results and if present extend the teltel_data list
#======================================================================================================================
# WRITE THE DATA TO THE DATA WAREHOUSE
# ======================================================================================================================
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'dwingestion-b033d9535e9d.json'
def create_row_data(tuple_data):
call_id, starttime, stoptime, direction, type, status, duration_sec, rate, cost, transfer, extra_prefix, audio_url, \
hangup_element, caller_number, caller_type, caller_cid, caller_dnid, caller_user_id, caller_user_short, \
callee_number, calle_type, callee, hangup_element_name, hangup_element_element, callee_user_id, callee_user_short, \
caller = tuple_data
row = teltel_call_data_pb2.TeltelCall()
row.call_id = call_id
row.starttime = starttime
row.stoptime = stoptime
row.direction = direction
row.type = type
row.status = status
row.duration_sec = duration_sec
row.rate = rate
row.cost = cost
row.transfer = transfer
row.extra_prefix = extra_prefix
row.audio_url = audio_url
row.hangup_element = hangup_element
row.caller_number = caller_number
row.caller_type = caller_type
row.caller_cid = caller_cid
row.caller_dnid = caller_dnid
row.caller_user_id = caller_user_id
row.caller_user_short = caller_user_short
row.callee_number = callee_number
row.calle_type = calle_type
row.callee = callee
row.hangup_element_name = hangup_element_name
row.hangup_element_title = hangup_element_element
row.callee_user_id = callee_user_id
row.callee_user_short = callee_user_short
row.caller = caller
return row.SerializeToString()
# Creating connection to the data warehouse
def create_bigquery_storage_client(google_credentials):
return bigquery_storage_v1.client.BigQueryWriteClient(
credentials=google_credentials
)
class GcpBigqueryStorageService(object):
def __init__(self, google_credentials=None, gcp_config=None):
self.client = create_bigquery_storage_client(google_credentials)
self.config = gcp_config
def append_rows_pending(self, project_id: str, dataset_id: str, table_id: str):
"""Create a write stream, write some sample data, and commit the stream."""
# write_client = self.client
parent = self.client.table_path(project_id, dataset_id, table_id)
write_stream = types.WriteStream()
# When creating the stream, choose the type. Use the PENDING type to wait
write_stream.type_ = types.WriteStream.Type.PENDING
write_stream = self.client.create_write_stream(
parent=parent, write_stream=write_stream
)
stream_name = write_stream.name
# Create a template with fields needed for the first request.
request_template = types.AppendRowsRequest()
# The initial request must contain the stream name.
request_template.write_stream = stream_name
# So that BigQuery knows how to parse the serialized_rows, generate a
# protocol buffer representation of your message descriptor.
proto_schema = types.ProtoSchema()
proto_descriptor = descriptor_pb2.DescriptorProto()
teltel_call_data_pb2.TeltelCall.DESCRIPTOR.CopyToProto(proto_descriptor)
proto_schema.proto_descriptor = proto_descriptor
proto_data = types.AppendRowsRequest.ProtoData()
proto_data.writer_schema = proto_schema
request_template.proto_rows = proto_data
# Some stream types support an unbounded number of requests. Construct an
# AppendRowsStream to send an arbitrary number of requests to a stream.
append_rows_stream = writer.AppendRowsStream(self.client, request_template)
# Create a batch of row data by appending proto2 serialized bytes to the
# serialized_rows repeated field.
proto_rows = types.ProtoRows()
row_number = 0
for row in whole_teltel_raw_data():
proto_rows.serialized_rows.append(create_row_data(row))
# checking the writing progress
row_number = row_number + 1
print("Writing to the database row number", row_number)
# The first request must always have an offset of 0.
request = types.AppendRowsRequest()
proto_data = types.AppendRowsRequest.ProtoData()
proto_data.rows = proto_rows
request.proto_rows = proto_data
append_rows_stream.close()
# A PENDING type stream must be "finalized" before being committed. No new
# records can be written to the stream after this method has been called.
self.client.finalize_write_stream(name=write_stream.name)
# Commit the stream you created earlier.
batch_commit_write_streams_request = types.BatchCommitWriteStreamsRequest()
batch_commit_write_streams_request.parent = parent
batch_commit_write_streams_request.write_streams = [write_stream.name]
self.client.batch_commit_write_streams(batch_commit_write_streams_request)
print(f"Writes to stream: '{write_stream.name}' have been committed.")
p = GcpBigqueryStorageService()
p.append_rows_pending(project_id='my_project', dataset_id='my_id', table_id='teltel_call_2')

Python Exception Not Caught?

I set up a try catch in my code, but it appears that my exception was not correct because it did not seem to catch it.
I am using an exception from a module, and perhaps I didn't import it correctly? Here is my code:
import logging
import fhirclient.models.bundle as b
from fhirclient.server import FHIRUnauthorizedException
logging.disable(logging.WARNING)
def get_all_resources(resource, struct, smart):
'''Perform a search on a resource type and get all resources entries from all retunred bundles.\n
This function takes all paginated bundles into consideration.'''
if smart.ready == False:
smart.reauthorize
search = resource.where(struct)
bundle = search.perform(smart.server)
resources = [entry.resource for entry in bundle.entry or []]
next_url = _get_next_url(bundle.link)
while next_url != None:
try:
json_dict = smart.server.request_json(next_url)
except FHIRUnauthorizedException:
smart.reauthorize
continue
bundle = b.Bundle(json_dict)
resources += [entry.resource for entry in bundle.entry or []]
next_url = _get_next_url(bundle.link)
return resources
Now when i ran the code I got the following error:
Traceback (most recent call last):
File "code.py", line 79, in <module>
main()
File "code.py", line 42, in main
reports = get_all_resources(dr.DiagnosticReport, search, smart)
File "somepath/fhir_tools/resource.py", line 23, in get_all_resources
json_dict = smart.server.request_json(next_url)
File "/usr/local/lib/python3.6/dist-packages/fhirclient/server.py", line 153, in request_json
res = self._get(path, headers, nosign)
File "/usr/local/lib/python3.6/dist-packages/fhirclient/server.py", line 181, in _get
self.raise_for_status(res)
File "/usr/local/lib/python3.6/dist-packages/fhirclient/server.py", line 256, in raise_for_status
raise FHIRUnauthorizedException(response)
server.FHIRUnauthorizedException: <Response [401]>
Shouldn't my exception catch this?

Softlayer getAllBillingItems stopped working?

The following python script worked like a charm last month:
Script:
import SoftLayer
client = SoftLayer.Client(username='someUser', api_key='someKey')
LastInvoice = client['Account'].getAllBillingItems()
print LastInvoice
Today's result:
C:\Python27\python.exe C:/Users/username/Documents/Python/Softlayer/Softlayer5.py
Traceback (most recent call last):
File "C:/Users/username/Documents/Python/Softlayer/Softlayer5.py", line 8, in <module>
LastInvoice = client['Account'].getAllBillingItems()
File "C:\Python27\lib\site-packages\SoftLayer\API.py", line 392, in call_handler
return self(name, *args, **kwargs)
File "C:\Python27\lib\site-packages\SoftLayer\API.py", line 360, in call
return self.client.call(self.name, name, *args, **kwargs)
File "C:\Python27\lib\site-packages\SoftLayer\API.py", line 263, in call
return self.transport(request)
File "C:\Python27\lib\site-packages\SoftLayer\transports.py", line 197, in __call__
raise exceptions.TransportError(ex.response.status_code, str(ex))
SoftLayer.exceptions.TransportError: TransportError(500): 500 Server Error: Internal Server Error for url: https://api.softlayer.com/xmlrpc/v3.1/SoftLayer_Account
Other api actions work fine... any thoughts?

well the charm has a defect and it is when the response has a big amount of data, that causes timeouts in the response and the conection is closed.
but this issue can be easily solved by using result limits take a look to this example:
import SoftLayer
# Your SoftLayer API username and key.
USERNAME = 'set me'
API_KEY = 'set me'
client = SoftLayer.Client(username=USERNAME, api_key=API_KEY)
offset = 0
limit = 50
accountService = client['SoftLayer_Account']
while True:
try:
result = accountService.getAllBillingItems(limit=limit, offset=offset)
offset = offset + limit
limit = limit + limit
print(result)
if not result:
break
except SoftLayer.SoftLayerAPIError as e:
print("Unable to retrieve the servers . " % (e.faultCode, e.faultString))
exit(1)
Regards

ValueError: No closing quotation

I have the following code which returns the public IP's
def gather_public_ip():
ACCESS_KEY = config.get('aws','access_key')
SECRET_KEY = config.get('aws','secret_key')
regions = regions = ['us-west-2','eu-central-1','ap-southeast-1']
# regions = config.get('aws','region').split(',')
all_EIP = []
for region in regions:
client = boto3.client('ec2',aws_access_key_id=ACCESS_KEY,aws_secret_access_key=SECRET_KEY,region_name=region,)
addresses_dict = client.describe_addresses()
for eip_dict in addresses_dict['Addresses']:
if 'PrivateIpAddress' in eip_dict:
print eip_dict['PublicIp']
# return str(eip_dict['PublicIp'])
all_EIP.append(eip_dict['PublicIp'])
print all_EIP
# print str(all_EIP)
return str(all_EIP)
This is called and returned as :
net_range = gather_public_ip()
for ip in net_range:
r = s.run(ip)
run looks like :
def run(self, targets="" ,options="-Pn"):
#start a new nmap scan on localhost with some specific options
syslog.syslog("Scan started")
parsed = None
nmproc = NmapProcess(targets,options)
rc = nmproc.run()
if rc != 0:
syslog.syslog("nmap scan failed: {0}".format(nmproc.stderr))
try:
parsed = NmapParser.parse(nmproc.stdout)
self.report = parsed
except NmapParserException as e:
syslog.syslog("Exception raised while parsing scan: {0}".format(e.msg))
syslog.syslog("Scan complete")
syslog.syslog("Scan duration: "+ str(parsed.elapsed))
self.report = parsed
return parsed
after printing the list , this throws me :
Traceback (most recent call last):
File "portwatch.py", line 300, in <module>
r = s.run(ip)
File "portwatch.py", line 239, in run
rc = nmproc.run()
File "/usr/local/lib/python2.7/dist-packages/libnmap/process.py", line 257, in run
else shlex.split(self.__nmap_command_line)
File "/usr/lib/python2.7/shlex.py", line 279, in split
return list(lex)
File "/usr/lib/python2.7/shlex.py", line 269, in next
token = self.get_token()
File "/usr/lib/python2.7/shlex.py", line 96, in get_token
raw = self.read_token()
File "/usr/lib/python2.7/shlex.py", line 172, in read_token
raise ValueError, "No closing quotation"
ValueError: No closing quotation

Make sure your ip is not "" or shlex will fail, cf Which exception to raise if a given string does not match some format?