I have the following development which I'm working with the ElementTree and Pandas module in Python:
import xml.etree.ElementTree as ET
import pandas as pd
file_xml = ET.parse('example1.xml')
rootXML = file_xml.getroot()
def transfor_data_atri(rootXML):
file_xml = ET.parse(rootXML)
data_XML = [
{"Name": signal.attrib["Name"],
# "Value": signal.attrib["Value"]
"Value": int(signal.attrib["Value"].split(' ')[0])
} for signal in file_xml.findall(".//Signal")
]
signals_df = pd.DataFrame(data_XML)
extract_name_value(signals_df)
def extract_name_value(signals_df):
#print(signals_df)
signal_ig_st = signals_df[signals_df.Name.isin(["Status"])]
row_values_ig_st = signal_ig_st.T
vector_ig_st = row_values_ig_st.iloc[[1]]
signal_nav_DSP_rq = signals_df[signals_df.Name.isin(["SetDSP"])]
row_values_nav_DSP_rq = signal_nav_DSP_rq.T
vector_nav_DSP_rq = row_values_nav_DSP_rq.iloc[[1]]
signal_HMI_st = signals_df[signals_df.Name.isin(["HMI"])]
row_values_HMI_st = signal_HMI_st.T
vector_HMI_st = row_values_HMI_st.iloc[[1]]
signal_delay_ac = signals_df[signals_df.Name.isin(["Delay"])]
row_values_delay_ac = signal_delay_ac.T
vector_delay_ac = row_values_delay_ac.iloc[[1]]
signal_AutoConfigO_Rear = signals_df[signals_df.Name.isin(["AutoConfigO_Rear"])]
row_values_AutoConfigO_Rear = signal_AutoConfigO_Rear.T
vector_AutoConfigO_Rear = row_values_AutoConfigO_Rear.iloc[[1]]
signal_ACO_Front = signals_df[signals_df.Name.isin(["AutoConfigO_Front"])]
row_values_ACO_Front = signal_ACO_Front.T
vertor_ACO_Front = row_values_ACO_Front.iloc[[1]]
signal_ACO_Drvr = signals_df[signals_df.Name.isin(["AutoConfigO_Drvr"])]
row_values_ACO_Drvr = signal_ACO_Drvr.T
vector_ACO_Drvr = row_values_ACO_Drvr.iloc[[1]]
signal_ACO_Allst = signals_df[signals_df.Name.isin(["AutoConfigO_Allst"])]
row_values_ACO_Allst = signal_ACO_Allst.T
vector_ACO_Allst = row_values_ACO_Allst.iloc[[1]]
signal_RURRq_st = signals_df[signals_df.Name.isin(["RUResReqstStat"])]
row_values_RURRq_st = signal_RURRq_st.T
vector_RURRq_st = row_values_RURRq_st.iloc[[1]]
signal_RURqSy_st = signals_df[signals_df.Name.isin(["RUReqstrSystem"])]
row_values_RURqSy_st = signal_RURqSy_st.T
vector_RURqSy_st = row_values_RURqSy_st.iloc[[1]]
signal_RUAudS_st = signals_df[signals_df.Name.isin(["RUSource"])]
row_values_RUAudS_st = signal_RUAudS_st.T
vector_RUAudS_st = row_values_RUAudS_st.iloc[[1]]
signal_DSP_st = signals_df[signals_df.Name.isin(["DSP"])]
row_values_DSP = signal_DSP.T
vector_DSP = row_values_DSP.iloc[[1]]
print('1: ', vector_ig_st)
print('2: ', vector_nav_DSP_rq)
print('3: ', vector_HMI_st)
print('4: ', vector_delay_ac)
The output of the above is the following, they are the first 4 prints and it is fine, because it is what they want, but I have to simplify the code, so that any type of xml file of the type example.xml, can be read not only example1.xml:
The simplified code is required to bring the data as it is in the names_list variable, but not to use this variable, which is actually hard-coded:
names_list = [
'Status', 'SetDSP', 'HMI', 'Delay', 'AutoConfigO_Rear',
'AutoConfigO_Front', 'AutoConfigO_Drvr','AutoConfigO_Allst',
'RUResReqstStat', 'RUReqstrSystem', 'RUSource', 'DSP'
]
So when the client wants to put another XML file with the same structure, but with other names that are not in the code, it can read them without problem. Beforehand thank you very much.
I hope I'm understanding the questions correctly. my understanding is that
you want to dynamically produce the extract_name_value() function, and make it not as bulky in your code.
Im sorry, but I failed to comprehend the for i in signal_name: print(i) part of the question. perhaps you can rephrase the question, and help me understand?
my solution to the extract_name_value() part would be using the exec() function.
it is a built-in solution for dynamic execution.
name_list = ['Status', 'SetDSP', 'HMI', 'Delay', 'AutoConfigO_Rear',
'AutoConfigO_Front', 'AutoConfigO_Drvr', 'AutoConfigO_Allst',
'RUResReqstStat', 'RUReqstrSystem', 'RUSource', 'DSP']
def _build_extract_name_value_func(name_list):
extract_name_value_func = ""
for name in name_list:
holder_func = f"""
signal_{name} = signals_df[signals_df.Name.isin([{name}])]
row_values_{name} = signal_{name}.T
vector_{name} = row_values_{name}.iloc[[1]]
vector_list.append(vector_{name})
"""
extract_name_value_func += holder_func
return extract_name_value_func
def extract_name_value(name_list):
extract_name_value_func = build_extract_name_value_func(name_list)
exec(extract_name_value_func)
the code was not tested with actual data, because I am not familiar with handling xml structures. But I hope the python part can be some help to you.
I was able to solve it, I used a for loop and iterated the dataframe itself:
for i in signals_df.Name:
signal = signals_df [signals_df.Name.isin ([i])]
row_values = signal.T
vector = row_values.iloc [[1]]
print (vector)
Folks,
I am trying to parse log file into json format.
I have a lot of logs, there is one of them
How can I parse this?
03:02:03.113 [info] ext_ref = BANK24AOS_cl_reqmarketcreditorderstate_6M8I1NT8JKYD_1591844522410384_4SGA08M8KIXQ reqid = 1253166 type = INREQ channel = BANK24AOS sid = msid_1591844511335516_KRRNBSLH2FS duration = 703.991 req_uri = marketcredit/order/state login = 77012221122 req_type = cl_req req_headers = {"accept-encoding":"gzip","connection":"close","host":"test-mobileapp-api.bank.kz","user-agent":"okhttp/4.4.1","x-forwarded-for":"212.154.169.134","x-real-ip":"212.154.169.134"} req_body = {"$sid":"msid_1591844511335516_KRRNBSLH2FS","$sid":"msid_1591844511335516_KRRNBSLH2FS","app":"bank","app_version":"2.3.2","channel":"aos","colvir_token":"GExPR0lOX1BBU1NXT1JEX0NMRUFSVEVYVFNzrzh4Thk1+MjDKWl/dDu1fQPsJ6gGLSanBp41yLRv","colvir_commercial_id":"-1","colvir_id":"000120.335980","openway_commercial_id":"6247520","openway_id":"6196360","$lang":"ru","ekb_id":"923243","inn":"990830221722","login":"77012221122","bank24_id":"262"} resp_body = {"task_id":"","status":"success","data":{"state":"init","applications":[{"status":"init","id":"123db561-34a3-4a8d-9fa7-03ed6377b44f","name":"Sulpak","amount":101000,"items":[{"name":"Switch CISCO x24","price":100000,"count":1,"amount":100000}]}],"segment":{"range":{"min":6,"max":36,"step":1},"payment_day":{"max":28,"min":1}}}}
Into this type of json, or any other format (but I guess json is best one)
{
"time":"03:02:03.113",
"class_req":"info",
"ext_ref":"BANK24AOS_cl_reqmarketcreditorderstate_6M8I1NT8JKYD_1591844522410384_4SGA08M8KIXQ",
"reqid":"1253166",
"type":"INREQ",
"channel":"BANK24AOS",
"sid":"msid_1591844511335516_KRRNBSLH2FS",
"duration":"703.991",
"req_uri":"marketcredit/order/state",
"login":"77012221122",
"req_type":"cl_req",
"req_headers":{
"accept-encoding":"gzip",
"connection":"close",
"host":"test-mobileapp-api.bank.kz",
"user-agent":"okhttp/4.4.1",
"x-forwarded-for":"212.154.169.134",
"x-real-ip":"212.154.169.134"
},
"req_body":{
"$sid":"msid_1591844511335516_KRRNBSLH2FS",
"$sid":"msid_1591844511335516_KRRNBSLH2FS",
"app":"bank",
"app_version":"2.3.2",
"channel":"aos",
"colvir_token":"GExPR0lOX1BBU1NXT1JEX0NMRUFSVEVYVFNzrzh4Thk1+MjDKWl/dDu1fQPsJ6gGLSanBp41yLRv",
"colvir_commercial_id":"-1",
"colvir_id":"000120.335980",
"openway_commercial_id":"6247520",
"openway_id":"6196360",
"$lang":"ru",
"ekb_id":"923243",
"inn":"990830221722",
"login":"77012221122",
"bank24_id":"262"
},
"resp_body":{
"task_id":"",
"status":"success",
"data":{
"state":"init",
"applications":[
{
"status":"init",
"id":"123db561-34a3-4a8d-9fa7-03ed6377b44f",
"name":"Sulpak",
"amount":101000,
"items":[
{
"name":"Switch CISCO x24",
"price":100000,
"count":1,
"amount":100000
}
]
}
],
"segment":{
"range":{
"min":6,
"max":36,
"step":1
},
"payment_day":{
"max":28,
"min":1
}
}
}
}
}
I am trying to split first whole text, but there I met another problem is to match keys to values depending on '=' sign. Also there might be some keys with empty values. For ex.:
type = INREQ channel = sid = duration = 1.333 (to get to know that there is an empty value, you need to pay attention on number of spaces. Usually there is 1 space between prev.value and next key). So this example should look like this:
{
"type":"INREQ",
"channel":"",
"sid":"",
"duration":"1.333"
}
Thanks ahead!
Here, one thing pass for duplicate key about "$sid":"msid_1591844511335516_KRRNBSLH2FS"
import re
text = """03:02:03.113 [info] ext_ref = reqid = 1253166 type = INREQ channel = BANK24AOS sid = msid_1591844511335516_KRRNBSLH2FS duration = 703.991 req_uri = marketcredit/order/state login = 77012221122 req_type = cl_req req_headers = {"accept-encoding":"gzip","connection":"close","host":"test-mobileapp-api.bank.kz","user-agent":"okhttp/4.4.1","x-forwarded-for":"212.154.169.134","x-real-ip":"212.154.169.134"} req_body = {"$sid":"msid_1591844511335516_KRRNBSLH2FS","$sid":"msid_1591844511335516_KRRNBSLH2FS","app":"bank","app_version":"2.3.2","channel":"aos","colvir_token":"GExPR0lOX1BBU1NXT1JEX0NMRUFSVEVYVFNzrzh4Thk1+MjDKWl/dDu1fQPsJ6gGLSanBp41yLRv","colvir_commercial_id":"-1","colvir_id":"000120.335980","openway_commercial_id":"6247520","openway_id":"6196360","$lang":"ru","ekb_id":"923243","inn":"990830221722","login":"77012221122","bank24_id":"262"} resp_body = {"task_id":"","status":"success","data":{"state":"init","applications":[{"status":"init","id":"123db561-34a3-4a8d-9fa7-03ed6377b44f","name":"Sulpak","amount":101000,"items":[{"name":"Switch CISCO x24","price":100000,"count":1,"amount":100000}]}],"segment":{"range":{"min":6,"max":36,"step":1},"payment_day":{"max":28,"min":1}}}}"""
index1 = text.index('[')
index2 = text.index(']')
new_text = 'time = '+ text[:index1-1] + ' class_req = ' + text[index1+1:index2] + text[index2+2:]
lst = re.findall(r'\S+? = |\S+? = \{.*?\} |\S+? = \{.*?\}$|\S+? = \S+? ', new_text)
res = {}
for item in lst:
key, equal, value = item.partition('=')
key, value = key.strip(), value.strip()
if value.startswith('{'):
try:
value = json.loads(value)
except:
print(value)
res[key] = value
you can try regulation in python.
here is what i write, it works for your problem.
for convenience i deleted string before "ext_ref...",you can directly truncate the raw string.
import re
import json
string = 'ext_ref = BANK24AOS_cl_reqmarketcreditorderstate_6M8I1NT8JKYD_1591844522410384_4SGA08M8KIXQ reqid = 1253166 type = INREQ channel = BANK24AOS sid = msid_1591844511335516_KRRNBSLH2FS duration = 703.991 req_uri = marketcredit/order/state login = 77012221122 req_type = cl_req req_headers = {"accept-encoding":"gzip","connection":"close","host":"test-mobileapp-api.bank.kz","user-agent":"okhttp/4.4.1","x-forwarded-for":"212.154.169.134","x-real-ip":"212.154.169.134"} req_body = {"$sid":"msid_1591844511335516_KRRNBSLH2FS","$sid":"msid_1591844511335516_KRRNBSLH2FS","app":"bank","app_version":"2.3.2","channel":"aos","colvir_token":"GExPR0lOX1BBU1NXT1JEX0NMRUFSVEVYVFNzrzh4Thk1+MjDKWl/dDu1fQPsJ6gGLSanBp41yLRv","colvir_commercial_id":"-1","colvir_id":"000120.335980","openway_commercial_id":"6247520","openway_id":"6196360","$lang":"ru","ekb_id":"923243","inn":"990830221722","login":"77012221122","bank24_id":"262"} resp_body = {"task_id":"","status":"success","data":{"state":"init","applications":[{"status":"init","id":"123db561-34a3-4a8d-9fa7-03ed6377b44f","name":"Sulpak","amount":101000,"items":[{"name":"Switch CISCO x24","price":100000,"count":1,"amount":100000}]}],"segment":{"range":{"min":6,"max":36,"step":1},"payment_day":{"max":28,"min":1}}}}'
position = re.search("req_headers",string) # position of req_headers
resp_body_pos = re.search("resp_body",string)
resp_body = string[resp_body_pos.span()[0]:]
res1 = {}
res1.setdefault(resp_body.split("=")[0],resp_body.split("=")[1])
print(res1)
before = string[:position.span()[0]]
after = string[position.span()[0]:resp_body_pos.span()[0]] # handle req_body seperately
res2 = re.findall("(\S+) = (\S+)",before)
print(res2)
res3 = re.findall("(\S+) = ({.*?})",after)
print(res3)
#res1 type: dict{'resp_body':'...'} content in resp_body
#res2 type: list[(),()..] content before req_head
#res3 type: list[(),()..] the rest content
and now you can do what you want to do with the data(.e.g. transform it into json respectively)
Hope this is helpful
I am not getting the expected results from the following iteration. The issue is the iteration value is not updating in the browser.find_element_by_xpath function (4th line of the code below). It always yields the first result. Note that I get the expected results when I manually run the iteration by assigning sequentially values 1,2,3,4 to the variable review_num instead of using the argument + str(review_num)
review_num = 0
for review_num in range(216): # for every review
review_num = review_num + 1
fixedelement = browser.find_element_by_xpath('//*[#id="collapseReviews"]/div/div[2]/div[2]/div[1]/div["+ str(review_num)"]')
#fixedelement = browser.find_element_by_xpath('//*[#id="collapseReviews"]/div/div[2]/div[2]/div[1]/div[1]')
#fixedelement = browser.find_element_by_xpath('//*[#id="collapseReviews"]/div/div[2]/div[2]/div[1]/div[2]')
#fixedelement = browser.find_element_by_xpath('//*[#id="collapseReviews"]/div/div[2]/div[2]/div[1]/div[3]')
#fixedelement = browser.find_element_by_xpath('//*[#id="collapseReviews"]/div/div[2]/div[2]/div[1]/div[4]')
R_title = fixedelement.find_element_by_xpath('./h4')
R_author = fixedelement.find_element_by_xpath('./div[2]/p[1]/span')
R_stars = fixedelement.find_element_by_xpath('./div[1]/div[1]/div[1]/span')
R_date = fixedelement.find_element_by_xpath('./div[1]/div[1]/div[2]/small')
R_comment = fixedelement.find_element_by_xpath('./div[1]/div[3]')
R_Yesvotes = fixedelement.find_element_by_xpath('./div[2]/div/div[1]/a[1]/span')
R_Novotes = fixedelement.find_element_by_xpath('./div[2]/div/div[1]/a[2]/span')
R_title_text = R_title.text
R_author_text = R_author.text
R_stars_text = R_stars.text
R_date_text = R_date.text
R_comment_text = R_comment.text
R_Yesvotes_text = R_Yesvotes.text
R_Novotes_text = R_Novotes.text
print(R_author_text)
with open(csvfile, "a", newline='', encoding='utf-8') as output:
writer = csv.writer(output, dialect='excel')
# writer.writerow(["namerow_id", "Name", "Position_Location"])
writer.writerow([review_num, R_title_text, R_author_text, R_stars_text, R_date_text, R_comment_text, R_Yesvotes_text, R_Novotes_text])
In the line
fixedelement = browser.find_element_by_xpath('//*[#id="collapseReviews"]/div/div[2]/div[2]/div[1]/div["+ str(review_num)"]')
you're searching for the element with xpath equal to
'//*[#id="collapseReviews"]/div/div[2]/div[2]/div[1]/div["+ str(review_num)"]'
with the "str(review_num)" hardcoded into the xpath.
You want to do something like
xpath = '//*[#id="collapseReviews"]/div/div[2]/div[2]/div[1]/div[{}]'.format(review_num)
fixedelement = browser.find_element_by_xpath(xpath)
this my code, i'm looking, is other way to code this in most efficient way?
i have multiple variables and inserted to the dictionary.
please feel to suggest and other options like array and etc will do.
def momentEndSpan(span_type,max_combo,length):
if "simply supported" == span_type:
q = max_combo
force = {}
RA = {"PA" : q*length/2}
RB = {"PB" : q*length/2}
RA_moment = {"MA" : 0}
R_mid_moment = {"Mmid": (q*math.pow(length,2))/8 }
RB_moment = { "MB" : 0}
force.update(RA)
force.update(RB)
force.update(RA_moment)
force.update(R_mid_moment)
force.update(RB_moment)
return force
elif "one end continuous" == span_type:
q = max_combo
x = (3/8)*length
force = {}
RA = {"Phinge" : 3*q*length/8}
RB = {"Pfixed" : 5*q*length/8}
RA_moment = {"Mhinge" : 0}
R_mid_moment = {"Mmid": (q*math.pow(length,2))*(9/128) }
RB_moment = { "MB" : -1*(q*math.pow(length,2))/8 }
force.update(RA)
force.update(RB)
force.update(RA_moment)
force.update(R_mid_moment)
force.update(RB_moment)
return force
Thank you very much
The "More Pythonic" way is to create one dictionary and update once.
q = max_combo
force = {}
if "simply supported" == span_type:
new = {"PA" : q*length/2,
"PB" : q*length/2,
"MA" : 0, "Mmid": (q*math.pow(length,2))/8,
"MB" : 0}
elif "one end continuous" == span_type:
x = (3/8)*length
new = {"Phinge" : 3*q*length/8,
"Pfixed" : 5*q*length/8,
"Mhinge" : 0,
"Mmid": (q*math.pow(length,2))*(9/128),
"MB" : -1*(q*math.pow(length,2))/8 }
force.update(new)
Also, note that if the force dictionary doesn't contain any previously defined items you can simply return the new and/or just continue to update the new in your next operations if there are any. Or just use name force instead of new.
q = max_combo
if "simply supported" == span_type:
force = {...}
elif "one end continuous" == span_type:
x = (3/8)*length
force = {...}
I am trying to create JSON object in Python, and it works just fine despite the fact that I can't get multiple keys with the same name - but I need to do it.
Here's a function:
findings = AutoTree()
findings['report']['numberOfConditions'] = num_cond
if r == 'Mammography':
f_temp = df['Relevant findings'].values.tolist()[0:8]
f_list = [x for i, x in enumerate(f_temp) if i == f_temp.index(x)]
f_num_total = len(f_list)
f_rand = random.randrange(1, f_num_total + 1)
iter_params_mass = ['shape', 'margin', 'density']
for i in range(num_cond):
br = get_birad(row, 2, 7)
cond = camelCase(get_cond_name())
findings[cond]['biRad'] = br
for k in range(f_rand + 1):
f = camelCase(random.choice(f_list))
#f = 'mass'
if f == 'mass':
rep_temp = create_rep(iter_params_mass, row, f, r)
findings[cond][f] = rep_temp
"""I also have a lot elif conditions, and it just grabs parameters."""
report = json.dumps(findings)
print(report)
Output:
{
"report":{
"id":85,
"name":"Lydia",
"age":39,
"relevantModality":"Mammography",
"numberOfConditions":2
},
"ductEctasia":{
"biRad":"birad[1]",
"calcifications":[
{
"typicallyBenign":"Vascular",
"suspiciousMorphology":"Coarse heterogeneous",
"distribution":"Diffuse"
}
],
"lymphNodes":[
{
"lymphNodes":"Lymph nodes \u2013 axillary"
}
]
}
}
And I want to have multiple "lymphNodes" and "calcifications" objects. Is it possible? Maybe, you can suggest another way to create JSON object, not nested dictionaries? The problem is that I need to create object respectively to random parameter chosen from the database.