I'm trying to convert Excel to nested XML and could not succeed as expected.
Here is my code.
import openpyxl
import xml.etree.ElementTree as etree
# reading data from the source, xls
wb1 = openpyxl.load_workbook(filename='C:\GSH\parent_child.xlsx')
ws1 = wb1.get_sheet_by_name('Sheet1')
row_max = ws1.max_row
# creating xml tree structure
root = etree.Element('Hierarchy')
# iterating through the xls and creating children based on the condition
for row_values in range(2, row_max+1):
parent = etree.SubElement(root, 'parent')
parent.text = ws1.cell(column=1, row=row_values).value
root.append(parent)
if (ws1.cell(column=1, row = row_values).value == ws1.cell(column=2, row = row_values-1).value):
print("------Inside if condition")
print(ws1.cell(column=2, row=row_values).value)
child = etree.SubElement(parent, 'child')
child.text = ws1.cell(column=2, row=row_values).value
parent.append(child)
print("-------Inside if condition")
tree = etree.ElementTree(root)
tree.write('C:\GSH\gsh.xml')
I am getting XML like this..
However, my XML should look like this.
Any suggestions, please.
The above is the source XLS from which I am working on.
You can set variable name instead of parent and child. This code is only part of your list and seems tricky but works fine. d[child[i]].text = " " is only to show both sides of tags. For making var in loop with dictionary, please refer this.
import xml.etree.ElementTree as ET
India = ET.Element('India') # set root
parent = ['India', 'Telangana', 'Telangana', 'Telangana','Nalgonda'] # parent list
child = ['Telangana', 'Cyberabad', 'Warangal','Nalgonda','BusStation'] # child list
d = {} # use dictionary to define var in loop
d['India'] = India
for i in range(len(child)):
for k, v in d.items():
if k == parent[i]:
pa = v
break
d[child[i]] = ET.SubElement(pa, child[i])
d[child[i]].text = " " # to get both side of tags
tree = ET.ElementTree(India)
tree.write('gsh.xml')
# <India>
# <Telangana>
# <Cyberabad> </Cyberabad>
# <Warangal> </Warangal>
# <Nalgonda>
# <BusStation> </BusStation>
# </Nalgonda>
# </Telangana>
# </India>
Related
Here is a xml file :
<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/">
<SOAP-ENV:Header />
<SOAP-ENV:Body>
<ADD_LandIndex_001>
<CNTROLAREA>
<BSR>
<status>ADD</status>
<NOUN>LandIndex</NOUN>
<REVISION>001</REVISION>
</BSR>
</CNTROLAREA>
<DATAAREA>
<LandIndex>
<reportId>AMI100031</reportId>
<requestKey>R3278458</requestKey>
<SubmittedBy>EN4871</SubmittedBy>
<submittedOn>2015/01/06 4:20:11 PM</submittedOn>
<LandIndex>
<agreementdetail>
<agreementid>001 4860</agreementid>
<agreementtype>NATURAL GAS</agreementtype>
<currentstatus>
<status>ACTIVE</status>
<statuseffectivedate>1965/02/18</statuseffectivedate>
<termdate>1965/02/18</termdate>
</currentstatus>
<designatedrepresentative></designatedrepresentative>
</agreementdetail>
</LandIndex>
</LandIndex>
</DATAAREA>
</ADD_LandIndex_001>
</SOAP-ENV:Body>
</SOAP-ENV:Envelope>
I want to save in a dataframe : 1) the path and 2) the text of the elements corresponding to the path. To do this dataframe, I am thinking to do a dictionary to store both. So first I would like to get a dictionary like that (where I have the values associated to the corresonding path).
{'/Envelope/Body/ADD_LandIndex_001/CNTROLAREA/BSR/status': 'ADD', /Envelope/Body/ADD_LandIndex_001/CNTROLAREA/BSR/NOUN: 'LandIndex',...}
Like that I just have to use the function df=pd.DataFrame() to create a dataframe that I can export in a excel sheet. I have already a part for the listing of the path, however I can not get text from those paths. I do not get how the lxml library works. I tried the function .text() and text_content() but I have an error.
Here is my code :
from lxml import etree
import xml.etree.ElementTree as et
from bs4 import BeautifulSoup
import pandas as pd
filename = 'file_try.xml'
with open(filename, 'r') as f:
soap = f.read()
root = etree.XML(soap.encode())
tree = etree.ElementTree(root)
mylist_path = []
mylist_data = []
mydico = {}
mylist = []
for target in root.xpath('//text()'):
if len(target.strip())>0:
path = tree.getpath(target.getparent()).replace('SOAP-ENV:','')
mydico[path] = target.text()
mylist_path.append(path)
mylist_data.append(target.text())
mylist.append(mydico)
df=pd.DataFrame(mylist)
df.to_excel("data_xml.xlsx")
print(mylist_path)
print(mylist_data)
Thank you for the help !
Here is an example of traversing XML tree. For this purpose recursive function will be needed. Fortunately lxml provides all functionality for this.
from lxml import etree as et
from collections import defaultdict
import pandas as pd
d = defaultdict(list)
root = et.fromstring(xml)
tree = et.ElementTree(root)
def traverse(el, d):
if len(list(el)) > 0:
for child in el:
traverse(child, d)
else:
if el.text is not None:
d[tree.getelementpath(el)].append(el.text)
traverse(root, d)
df = pd.DataFrame(d)
df.head()
Output:
{
'{http://schemas.xmlsoap.org/soap/envelope/}Body/ADD_LandIndex_001/CNTROLAREA/BSR/status': ['ADD'],
'{http://schemas.xmlsoap.org/soap/envelope/}Body/ADD_LandIndex_001/CNTROLAREA/BSR/NOUN': ['LandIndex'],
'{http://schemas.xmlsoap.org/soap/envelope/}Body/ADD_LandIndex_001/CNTROLAREA/BSR/REVISION': ['001'],
'{http://schemas.xmlsoap.org/soap/envelope/}Body/ADD_LandIndex_001/DATAAREA/LandIndex/reportId': ['AMI100031'],
'{http://schemas.xmlsoap.org/soap/envelope/}Body/ADD_LandIndex_001/DATAAREA/LandIndex/requestKey': ['R3278458'],
'{http://schemas.xmlsoap.org/soap/envelope/}Body/ADD_LandIndex_001/DATAAREA/LandIndex/SubmittedBy': ['EN4871'],
'{http://schemas.xmlsoap.org/soap/envelope/}Body/ADD_LandIndex_001/DATAAREA/LandIndex/submittedOn': ['2015/01/06 4:20:11 PM'],
'{http://schemas.xmlsoap.org/soap/envelope/}Body/ADD_LandIndex_001/DATAAREA/LandIndex/LandIndex/agreementdetail/agreementid': ['001 4860'],
'{http://schemas.xmlsoap.org/soap/envelope/}Body/ADD_LandIndex_001/DATAAREA/LandIndex/LandIndex/agreementdetail/agreementtype': ['NATURAL GAS'],
'{http://schemas.xmlsoap.org/soap/envelope/}Body/ADD_LandIndex_001/DATAAREA/LandIndex/LandIndex/agreementdetail/currentstatus/status': ['ACTIVE'],
'{http://schemas.xmlsoap.org/soap/envelope/}Body/ADD_LandIndex_001/DATAAREA/LandIndex/LandIndex/agreementdetail/currentstatus/statuseffectivedate': ['1965/02/18'],
'{http://schemas.xmlsoap.org/soap/envelope/}Body/ADD_LandIndex_001/DATAAREA/LandIndex/LandIndex/agreementdetail/currentstatus/termdate': ['1965/02/18']
}
Please note, the dictionary d contains lists as values. That's because elements can be repeated in XML and otherwise last value will override previous one. If that's not the case for your particular XML, use regular dict instead of defaultdict d = {} and use assignment instead of appending d[tree.getelementpath(el)] = el.text.
The same when reading from file:
d = defaultdict(list)
with open('output.xml', 'rb') as file:
root = et.parse(file).getroot()
tree = et.ElementTree(root)
def traverse(el, d):
if len(list(el)) > 0:
for child in el:
traverse(child, d)
else:
if el.text is not None:
d[tree.getelementpath(el)].append(el.text)
traverse(root, d)
df = pd.DataFrame(d)
print(d)
Lets assume we have an arbitrary XML document like below
<?xml version="1.0" encoding="UTF-8"?>
<programs xmlns="http://something.org/schema/s/program">
<program xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://something.org/schema/s/program http://something.org/schema/s/program.xsd">
<orgUnitId>Organization 1</orgUnitId>
<requiredLevel>academic bachelor</requiredLevel>
<requiredLevel>academic master</requiredLevel>
<programDescriptionText xml:lang="nl">Here is some text; blablabla</programDescriptionText>
<searchword xml:lang="nl">Scrum master</searchword>
</program>
<program xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://something.org/schema/s/program http://something.org/schema/s/program.xsd">
<requiredLevel>bachelor</requiredLevel>
<requiredLevel>academic master</requiredLevel>
<requiredLevel>academic bachelor</requiredLevel>
<orgUnitId>Organization 2</orgUnitId>
<programDescriptionText xml:lang="nl">Text from another organization about some stuff.</programDescriptionText>
<searchword xml:lang="nl">Excutives</searchword>
</program>
<program xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<orgUnitId>Organization 3</orgUnitId>
<programDescriptionText xml:lang="nl">Also another huge text description from another organization.</programDescriptionText>
<searchword xml:lang="nl">Negotiating</searchword>
<searchword xml:lang="nl">Effective leadership</searchword>
<searchword xml:lang="nl">negotiating techniques</searchword>
<searchword xml:lang="nl">leadership</searchword>
<searchword xml:lang="nl">strategic planning</searchword>
</program>
</programs>
Currently I'm looping over the elements I need by using their absolute paths, since I'm not able to use any of the get or find methods in ElementTree. As such, my code looks like below:
import pandas as pd
import xml.etree.ElementTree as ET
import numpy as np
import itertools
tree = ET.parse('data.xml')
root = tree.getroot()
root.tag
dfcols=['organization','description','level','keyword']
organization=[]
description=[]
level=[]
keyword=[]
for node in root:
for child in
node.findall('.//{http://something.org/schema/s/program}orgUnitId'):
organization.append(child.text)
for child in node.findall('.//{http://something.org/schema/s/program}programDescriptionText'):
description.append(child.text)
for child in node.findall('.//{http://something.org/schema/s/program}requiredLevel'):
level.append(child.text)
for child in node.findall('.//{http://something.org/schema/s/program}searchword'):
keyword.append(child.text)
The goal, of course, is to create one dataframe. However, since each node in the XML file contains one or multiple elements, such as requiredLevel or searchword I'm currently losing data when I'm casting it to a dataframe by either:
df=pd.DataFrame(list(itertools.zip_longest(organization,
description,level,searchword,
fillvalue=np.nan)),columns=dfcols)
or using pd.Series as given here or another solution which I don't seem to get it fit from here
My best bet is not to use Lists at all, since they don't seem to index the data correctly. That is, I lose data from the 2nd to Xth child node. But right now I'm stuck, and don't see any other options.
What my end result should look like is this:
organization description level keyword
Organization 1 .... academic bachelor, Scrum master
academic master
Organization 2 .... bachelor, Executives
academic master,
academic bachelor
Organization 3 .... Negotiating,
Effective leadership,
negotiating techniques,
....
Consider building a list of dictionaries with comma-collapsed text values. Then pass list into the pandas.DataFrame constructor:
dicts = []
for node in root:
orgs = ", ".join([org.text for org in node.findall('.//{http://something.org/schema/s/program}orgUnitId')])
desc = ", ".join([desc.text for desc in node.findall('.//{http://something.org/schema/s/program}programDescriptionText')])
lvls = ", ".join([lvl.text for lvl in node.findall('.//{http://something.org/schema/s/program}requiredLevel')])
wrds = ", ".join([wrd.text for wrd in node.findall('.//{http://something.org/schema/s/program}searchword')])
dicts.append({'organization': orgs, 'description': desc, 'level': lvls, 'keyword': wrds})
final_df = pd.DataFrame(dicts, columns=['organization','description','level','keyword'])
Output
print(final_df)
# organization description level keyword
# 0 Organization 1 Here is some text; blablabla academic bachelor, academic master Scrum master
# 1 Organization 2 Text from another organization about some stuff. bachelor, academic master, academic bachelor Excutives
# 2 Organization 3 Also another huge text description from anothe... Negotiating, Effective leadership, negotiating...
A lightweight xml_to_dict converter can be found here. It can be improved by this to handle namespaces.
def xml_to_dict(xml='', remove_namespace=True):
"""Converts an XML string into a dict
Args:
xml: The XML as string
remove_namespace: True (default) if namespaces are to be removed
Returns:
The XML string as dict
Examples:
>>> xml_to_dict('<text><para>hello world</para></text>')
{'text': {'para': 'hello world'}}
"""
def _xml_remove_namespace(buf):
# Reference: https://stackoverflow.com/a/25920989/1498199
it = ElementTree.iterparse(buf)
for _, el in it:
if '}' in el.tag:
el.tag = el.tag.split('}', 1)[1]
return it.root
def _xml_to_dict(t):
# Reference: https://stackoverflow.com/a/10077069/1498199
from collections import defaultdict
d = {t.tag: {} if t.attrib else None}
children = list(t)
if children:
dd = defaultdict(list)
for dc in map(_xml_to_dict, children):
for k, v in dc.items():
dd[k].append(v)
d = {t.tag: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}}
if t.attrib:
d[t.tag].update(('#' + k, v) for k, v in t.attrib.items())
if t.text:
text = t.text.strip()
if children or t.attrib:
if text:
d[t.tag]['#text'] = text
else:
d[t.tag] = text
return d
buffer = io.StringIO(xml.strip())
if remove_namespace:
root = _xml_remove_namespace(buffer)
else:
root = ElementTree.parse(buffer).getroot()
return _xml_to_dict(root)
So let s be the string which holds your xml. We can convert it to a dict via
d = xml_to_dict(s, remove_namespace=True)
Now the solution is straight forward:
rows = []
for program in d['programs']['program']:
cols = []
cols.append(program['orgUnitId'])
cols.append(program['programDescriptionText']['#text'])
try:
cols.append(','.join(program['requiredLevel']))
except KeyError:
cols.append('')
try:
searchwords = program['searchword']['#text']
except TypeError:
searchwords = []
for searchword in program['searchword']:
searchwords.append(searchword['#text'])
searchwords = ','.join(searchwords)
cols.append(searchwords)
rows.append(cols)
df = pd.DataFrame(rows, columns=['organization', 'description', 'level', 'keyword'])
I have checked this link but it doesnt solved my problem.
I have 2 lists:
a = [['txt','stxt','pi','min','max'],['txt1','stxt1','pi1','min1','max1']]
b = [[0.45,1.23],[[0.75,1.53]]
for l1 in a:
for l2 in b:
root = ET.Element("Class ",name = l1[0])
doc = ET.SubElement(root, "subclass" , name = l1[1])
ET.SubElement(doc, l1[4], min = str(l2 [0]),max = str(l2 [1]))
tree = ET.ElementTree(root)
tree.write(FilePath)
The last record is overwriting all the previous records. So if i want all the records to be written to the xml file? how can i do that using python programming. I also want each record to be saved to the xml file in new line but not pretty printing.
Output i need to be added to the xml:
<Class name="txt"><subclass name="stxt"><pi max="1.23" min="0.45" /></subclass></Class >
<Class name="txt1"><subclass name="stxt1"><pi1 max1="1.53" min1="0.75" /></subclass></Class >
But i am getting is only one record in xml:
<Class name="txt1"><subclass name="stxt1"><pi1 max1="0.1077" min1="-0.0785" /></subclass></Class >
You are writing to same file every time. You need to create new file for every input and the two for loops will make 4 files with undesired combinations. Instead zip is what you need
a = [['txt','stxt','pi','min','max'],['txt1','stxt1','pi1','min1','max1']]
b = [[0.45,1.23],[0.75,1.53]]
from xml.etree import ElementTree as ET
root = ET.Element("xml")
for l1 in zip(a,b):
sroot_root = ET.Element("Class ",name = l1[0][0])
doc = ET.SubElement(sroot_root, "subclass" , name = l1[0][1])
ET.SubElement(doc, l1[0][4], min = str(l1[1][0]),max = str(l1[1][1]))
root.append(sroot_root)
tree = ET.ElementTree(root)
tree.write("test.xml")
Output :
Filename: test.xml
<xml><Class name="txt"><subclass name="stxt"><max max="1.23" min="0.45" /></subclass></Class ><Class name="txt1"><subclass name="stxt1"><max1 max="1.53" min="0.75" /></subclass></Class ></xml>
I am trying to import urls like this one (http://musicbrainz.org/ws/2/artist/72c536dc-7137-4477-a521-567eeb840fa8) into python and extract the value of the "gender".
import urllib2
import codecs
import sys
import os
from xml.dom import minidom
import xml.etree.cElementTree as ET
#urlbob = urllib2.urlopen('http://musicbrainz.org/ws/2/artist/72c536dc-7137-4477-a521-567eeb840fa8')
url = 'dylan.xml'
#attempt 1 - using minidom
xmldoc = minidom.parse(url)
itemlist = xmldoc.getElementsByTagName('artist')
#attempt 2 - using ET
tree = ET.parse('dylan.xml')
root = tree.getroot()
for child in root:
print child.tag, child.attrib
I can't seem to get at gender either via the mini-dom stuff or the etree stuff. In its current form, the script returns
{http://musicbrainz.org/ns/mmd-2.0#}artist {'type': 'Person', 'id': '72c536dc-7137-4477-a521-567eeb840fa8'}
That is because you're looping root which is only the root of the tree, does that make sense? When you loop the root, it will only return the next child and stop there.
You need to loop the iterable so it will get returning next node and get the result, see this:
tree = ET.parse('dylan.xml')
root = tree.getroot()
# loop the root iterable which will keep returning next node
for node in root.iter(): # or root.getiterator() if < Python 2.7
print node.tag, node.attrib, node.text
Results:
{http://musicbrainz.org/ns/mmd-2.0#}metadata {} None
{http://musicbrainz.org/ns/mmd-2.0#}artist {'type': 'Person', 'id': '72c536dc-7137-4477-a521-567eeb840fa8'} None
{http://musicbrainz.org/ns/mmd-2.0#}name {} Bob Dylan
{http://musicbrainz.org/ns/mmd-2.0#}sort-name {} Dylan, Bob
{http://musicbrainz.org/ns/mmd-2.0#}ipi {} 00008955074
{http://musicbrainz.org/ns/mmd-2.0#}ipi-list {} None
{http://musicbrainz.org/ns/mmd-2.0#}ipi {} 00008955074
{http://musicbrainz.org/ns/mmd-2.0#}ipi {} 00008955172
{http://musicbrainz.org/ns/mmd-2.0#}isni-list {} None
{http://musicbrainz.org/ns/mmd-2.0#}isni {} 0000000121479733
{http://musicbrainz.org/ns/mmd-2.0#}gender {} Male
{http://musicbrainz.org/ns/mmd-2.0#}country {} US
{http://musicbrainz.org/ns/mmd-2.0#}area {'id': '489ce91b-6658-3307-9877-795b68554c98'} None
{http://musicbrainz.org/ns/mmd-2.0#}name {} United States
{http://musicbrainz.org/ns/mmd-2.0#}sort-name {} United States
{http://musicbrainz.org/ns/mmd-2.0#}iso-3166-1-code-list {} None
{http://musicbrainz.org/ns/mmd-2.0#}iso-3166-1-code {} US
{http://musicbrainz.org/ns/mmd-2.0#}begin-area {'id': '04e60741-b1ae-4078-80bb-ffe8ae643ea7'} None
{http://musicbrainz.org/ns/mmd-2.0#}name {} Duluth
{http://musicbrainz.org/ns/mmd-2.0#}sort-name {} Duluth
{http://musicbrainz.org/ns/mmd-2.0#}life-span {} None
{http://musicbrainz.org/ns/mmd-2.0#}begin {} 1941-05-24
## This prints out the tree as the xml lib sees it
## (I found it made debugging a little easier)
#def print_xml(node, depth = 0):
# for child in node:
# print "\t"*depth + str(child)
# print_xml(child, depth = depth + 1)
#print_xml(root)
# attempt 1
xmldoc = minidom.parse(url)
genders = xmldoc.getElementsByTagName('gender') # <== you want gender not artist
for gender in genders:
print gender.firstChild.nodeValue
# attempt 2
ns = "{http://musicbrainz.org/ns/mmd-2.0#}"
xlpath = "./" + ns + "artist/" + ns + "gender"
genders = root.findall(xlpath) # <== xpath was made for this..
for gender in genders:
print gender.text
So.. the problem with your first attempt is that you're looking at a list of all the artist elements not the gender elements (the child of the only artist element in the list).
The problem with your second attempt is that you are looking at a list of the children of the root element (which is a list containing a single metadata element).
The underlying structure is:
<artist>
<name>
<sort-name>
<ipi>
<ipi-list>
<ipi>
<ipi>
<isni-list>
<isni>
<gender>
<country>
<area>
<name>
<sort-name>
<iso-3166-1-code-list>
<iso-3166-1-code>
<begin-area>
<name>
<sort-name>
<life-span>
<begin>
so you need to get root -> artist -> gender, or just search for the node you actually want (gender in this case).
My xml :-
<users>
</users>
i need to just append a child element :-
<users>
<user name="blabla" age="blabla" ><group>blabla</group>
</users>
My code giving some error :(
import xml.etree.ElementTree as ET
doc = ET.parse("users.xml")
root_node = doc.find("users")
child = ET.SubElement(root_node, "user")
child.set("username","srquery")
group = ET.SubElement(child,"group")
group.text = "fresher"
tree = ET.ElementTree(root_node)
tree.write("users.xml")
I missed out "append" , but i don't have idea where to add that. Thanks in advance.
Change this line
root_node = doc.find("users")
...to this line
root_node = doc.getroot()
The key takeaway here is that doc is already a reference to the root node, and is accessed with getroot(). doc.find('users') will not return anything, since users is not a child of the root, it's the root itself.
A slightlly modified version to explain what happens:
root = ET.fromstring('<users></users>') # same as your doc=ET.parse(...).find(...), btw. doc=root
el = ET.Element('group') # creating a new element/xml-node
root.append(el) # and adding it to the root
ET.tostring(root)
>>> '<users><group /></users>'
el.text = "fresher" # adding your text
ET.tostring(root)
>>> '<users><group>fresher</group></users>'