We Keep Coding

Convert many complex txt files into excel.xls and save their names in python script

I have many txt files with the standard format in the same folder:
Name: 321;
Score:100; Used Time: 1:09:308;
GTime: 6/28/2024 10:04:18 PM;
Core Version : 21.0.0.0;
Software Version : 21.0.0.0;
AppID: 0S0; MapDispName: Future City; MapName:MapName123;
Key:A0000-abcde-Q0000-F0000-00H00; REG Date : 2/27/2021 1:16:34 PM; Expiry : 7/7/2024 12:00:00 AM
I would like to convert those text files into an excle.xls (table) using a python script. At the same time, I would like to save text filenames as well.
Team ID, Result, Used Time,Software Ver, Core Ver, AppID, Key, REG Date, Expiry,MapName,TXTName
321,100,69.308s,21.0.0.0,21.0.0.0,0S0,A0000-abcde-Q0000-F0000-00H00,2/27/2021 1:16:34 PM,7/7/2024 12:00:00 AM,MapName123,TXTName1
Part of my code as below, but it's not working. TypeError: unsupported operand type(s) for +: 'dict' and 'str' in the penultimate line.
list_drr=[]
xls_name=None
for path, file_dir, files in os.walk(file_name_path):
for file_name in files:
list_drr.append(os.path.join(path, file_name))
for dir in file_dir:
list_drr.append(os.path.join(path, dir))#
excel_data= ExcelData(xls_name,"")
excel_datas= excel_data.readExcel()
print(excel_datas)
excel_header=['Team ID', 'Result', 'Used Time', 'Software Ver', 'Core Version', 'AppID', 'Key', 'REG Date', 'Expiry','MapName','TXTName']
file= WFile(excel_header)
for drr in list_drr:
file_datas= getFileData(drr)
file_datas=file_datas[:7]
data_list=[]
for data in file_datas:
lis= data.split(";")
for li in lis:
data_list.append(li)
data_dic={}
for data in data_list:
pos= data.find(":")
ddq=data[:pos].strip()
data_dic[ddq]=data[pos+1:].strip()
file.write((data_dic) + (os.path.basename(file_name)))
file.save("excel.xls")
Please advise what should I do, thanks.
Updated wri_file.py as below.
import xlwt
class WFile():
def __init__(self,head_name):
super().__init__()
self.head_name=head_name
self.index=0
self. workbook =xlwt. Workbook(encoding='utf-8')
self. worksheet = self.workbook.add_sheet('Sheet1')
self.line =len( self.head_name)
for i in range( self.line):
self.worksheet.write( self.index,i ,self.head_name[i])
self.index+=1
def write(self,d):
for i in range( self.line):
name=d.get(self.head_name[i])
self.worksheet.write( self.index,i ,(name))
self.index += 1
def writes(self, d):
for i in range(self.line):
self.worksheet.write(self.index, i,d[i])
self.index += 1
def save(self,name):
self.workbook.save(name)

You are trying to add file_name as another entry to be written. As you have constructed a dictionary, a better approach would be to add a new key value pair before writing:
data_dic['TXTName'] = os.path.basename(drr)
file.write(data_dic)
As your output header is not consistent with the keys in your text files, you need to create a dictionary to map between the two, e.g. field_mapping
You were also storing directory names into your list of file? A simpler approach is to just use glob.glob() which also can work recursively and also just on .txt files if needed.
import glob
import xlwt
# {Your header : Key in text file}
field_mapping = {
'Team ID' : 'Name',
'Result' : 'Score',
'Used Time' : 'Used Time',
'Core Version' : 'Core Version',
'Software Ver' : 'Software Version',
'MapName' : 'MapName',
'AppID' : 'AppID',
'Key' : 'Key',
'REG Date' : 'REG Date',
'Expiry' : 'Expiry',
'TXTName' : 'TXTName',
}
def getFileData(drr):
with open(drr) as f_input :
return f_input.readlines()
class WFile():
def __init__(self,head_name):
super().__init__()
self.head_name = head_name
self.index = 0
self.workbook = xlwt.Workbook(encoding='utf-8')
self.worksheet = self.workbook.add_sheet('Sheet1')
self.line = len(self.head_name)
for i in range(self.line):
self.worksheet.write(self.index, i, self.head_name[i])
self.index += 1
def write(self, d):
for i, v in enumerate(self.head_name):
name = d.get(field_mapping[v])
self.worksheet.write(self.index, i, (name))
self.index += 1
def save(self,name):
self.workbook.save(name)
file_name_path = r"\my file\location\*.txt" # or just * for all files
excel_header = ['Team ID', 'Result', 'Used Time', 'Software Ver', 'Core Version', 'AppID', 'Key', 'REG Date', 'Expiry','MapName','TXTName']
file = WFile(excel_header)
for file_name in glob.glob(file_name_path, recursive=True):
file_datas = getFileData(file_name)
file_datas = file_datas[:7]
data_list = []
for data in file_datas:
for li in data.split(";"):
data_list.append(li)
data_dic = {}
for data in data_list:
pos = data.find(":")
ddq = data[:pos].strip()
data_dic[ddq] = data[pos+1:].strip()
data_dic['TXTName'] = os.path.basename(file_name)
file.write(data_dic)
file.save("excel.xls")
Giving you an old style output .xls file of:
You should consider using the newer .xlsx format, with a library such as openpyxl.
import openpyxl
from itertools import islice
import glob
import os
field_mapping = {
'Name' : 'Team ID',
'Score' : 'Result',
'Used Time' : 'Used Time',
'Core Version' : 'Core Ver',
'Software Version' : 'Software Ver',
'AppID' : 'AppID',
'Key' : 'Key',
'REG Date' : 'REG Date',
'Expiry' : 'Expiry',
'TXTName' : 'TXTName'
}
wb = openpyxl.Workbook()
ws = wb.active
ws.append(list(field_mapping.values())) # write a header
for filename in glob.glob('/my folder/*.txt', recursive=True):
key_values = {}
with open(filename) as f_input:
for field in ''.join(islice(f_input, 0, 7)).replace('\n', '').split(';'):
key, value = map(str.strip, field.split(':', 1)) # only split on the first colon
key_values[field_mapping.get(key, None)] = value
key_values['TXTName'] = os.path.basename(filename)
output_row = [value for key, value in key_values.items() if key] # Ignore key=None
ws.append(output_row)
wb.save('excel.xlsx')

How to add dictionaries inside of a list

I want to connect separated messages of a chat, so I created a list for all the dictionaries
messages = ["Hello", "How you doing","fine","how can I help you", "how to do this?", "like this", "thanks","man","no problem"]
Person1= [True,True,False,False,True,False,True,True,False]
data =[]
chat_messages = messages
people = Person1
k = 0
for i in range(len(messages)):
if people[i] == people[i+1]:
chat_messages[i+1] = chat_messages[i] +" " +chat_messages[i+1]
chatData = {'text': chat_messages[i+1], 'person1': people[i]}
data[k] = chatData
else:
k +=1
chatData = {'text': chat_messages[i+1], 'person1': people[i+1]}
print(chatData )
data[k] = chatData
print(data)
I'm getting errors in here
File "main.py", line 20, in <module>
data[k] = chatData
IndexError: list assignment index out of range
How can I fix it please?
I want the output of data to look like this:
data = [{'text': 'Hello How you doing', 'person1': True} , {'text': 'fine how can I help you', 'person1': False}, {'text': 'how to do this?', 'person1': True}]

You cant add elements to a list in python this way. you have to use python method append().
data.append(chatData)
This method will add elements at the end of the list.
You can learn more python list methods using this link
https://www.geeksforgeeks.org/list-methods-python/

The problem is that when you add the index i + 1, it gives an error when you get to the nr 9, because your list index stops at 8. Here is my solution:
messages = ["Hello", "How you doing","fine","how can I help you", "how to do this?", "like this", "thanks","man","no problem"]
Person1= [True,True,False,False,True,False,True,True,False]
data =[]
chat_messages = messages
people = Person1
k = 0
data = []
for i, msg in enumerate(messages):
try:
if people[i] == people[i+1]:
chat_messages[i+1] = chat_messages[i] +" " +chat_messages[i+1]
data.append({'text': chat_messages[i+1], 'person1': people[i]})
except:
pass
print(data)

messages = ["Hello", "How you doing","fine","how can I help you", "how to do this?", "like this", "thanks","man","no problem"]
Person1= [True,True,False,False,True,False,True,True,False]
data =[]
chat_messages = messages
people = Person1
k = 0
for i in range(len(messages)):
if len(messages)-1 is i:
None
else:
if people[i] == people[i+1]:
chat_messages[i+1] = chat_messages[i] +" " +chat_messages[i+1]
chatData = {'text': chat_messages[i+1], 'person1': people[i]}
data.append(chatData)
else:
chatData = {'text': chat_messages[i+1], 'person1': people[i+1]}
print(chatData )
data.append(chatData)
print(data)

Web Scraping to CSV - ValueError Could not broadcast input array from shape (2) into shape (1)

I've been playing around with python and beautifulsoup last few days. I've taken to trying to scrape a local gun advertising platform.
I've been on and got help and its been great. with those help stages ive pushed to the next "right so how do i go about doing this...."
This is where i am at the moment:
I'm exporting (failing to) my data to csv.
I'm getting the error - ValueError - Could not broadcast input array from shape (2) into shape (1).
Some reading suggests im pulling in a 2d array into a 1d array? - looking at my code i cant see where i could be possibly doing this?
Would anyone mind having a scan and seeing where im going wrong?
Thanks!
full script:
from bs4 import BeautifulSoup
import requests
import urllib.request
import csv
import pandas
from pandas import DataFrame
import re
#csv creation
with open('Guntrader_Dealer.csv', mode='w') as csv_file:
fieldnames = ['Title', 'Make', 'Model', 'Licence', 'Orientation', 'Barrel Length', 'Stock Length', 'Chokes', 'Origin', 'Trigger', 'Ejection', 'Scope', 'Serial No', 'Stock No', 'Condition', 'Description', 'Price']
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writeheader()
#all links list
all_links=[]
#grab all links which contain the href specifed
url="https://www.guntrader.uk/dealers/minsterley/minsterley-ranges/guns?page={}"
for page in range(1,3):
res=requests.get(url).text
soup=BeautifulSoup(res,'html.parser')
for link in soup.select('a[href*="dealers/minsterley/minsterley-ranges/guns/"]'):
all_links.append("https://www.guntrader.uk" + link['href'])
for a_link in all_links:
#Defining the span text in GunDetails lookups
def make_span(make):
return make.name=='span' and 'Make:' in make.parent.contents[0]
def model_span(model):
return model.name=='span' and 'Model:' in model.parent.contents[0]
def licence_span(licence):
return licence.name=='span' and 'Licence:' in licence.parent.contents[0]
def orient_span(orient):
return orient.name=='span' and 'Orient.:' in orient.parent.contents[0]
def barrel_span(barrel):
return barrel.name=='span' and 'Barrel:' in barrel.parent.contents[0]
def stock_span(stock):
return stock.name=='span' and 'Stock:' in stock.parent.contents[0]
def choke_span(choke):
return choke.name=='span' and 'Chokes:' in choke.parent.contents[0]
def origin_span(origin):
return origin.name=='span' and 'Origin:' in origin.parent.contents[0]
def trigger_span(trigger):
return trigger.name=='span' and 'Trigger:' in trigger.parent.contents[0]
def ejection_span(ejection):
return ejection.name=='span' and 'Ejection:' in ejection.parent.contents[0]
def serial_span(serial):
return serial.name=='span' and 'Serial #:' in serial.parent.contents[0]
def stockno_span(stockno):
return stockno.name=='span' and 'Stock #:' in stockno.parent.contents[0]
def condition_span(condition):
return condition.name=='span' and 'Condition:' in condition.parent.contents[0]
def scope_span(scope):
return scope.name=='span' and 'Scope:' in scope.parent.contents[0]
res = urllib.request.urlopen(a_link)
soup = BeautifulSoup(res, 'html.parser')
#soup searches using the define criteria
makes = soup.find(make_span)
gun_makes = makes.content if makes else 'none'
models = soup.find(model_span)
gun_models = models.contents if models else 'none'
licences = soup.find(licence_span)
gun_licences = licences.contents if licences else 'none'
orients = soup.find(orient_span)
gun_orients = orients.contents if orients else 'none'
barrels = soup.find(barrel_span)
gun_barrels = barrels.contents if barrels else 'none'
stocks = soup.find(stock_span)
gun_stocks = stocks.contents if stocks else 'none'
chokes = soup.find(choke_span)
gun_chokes = chokes.contents if chokes else 'none'
origins = soup.find(origin_span)
gun_origins = origins.contents if origins else 'none'
triggers = soup.find(trigger_span)
gun_triggers = triggers.contents if triggers else 'none'
ejections = soup.find(ejection_span)
gun_ejections = ejections.contents if ejections else 'none'
scopes = soup.find(scope_span)
gun_scopes = scopes.contents if scopes else 'none'
serials = soup.find(serial_span)
gun_serials = serials.contents if serials else 'none'
stocknos = soup.find(stockno_span)
gun_stocknos = stocknos.contents if stocknos else 'none'
conditions = soup.find(condition_span)
gun_conditions = conditions.contents if conditions else 'none'
#title price and description
title = soup.select_one('h1[itemprop="name"]')
gun_title = title.text if title else 'none'
price = soup.select_one('p.price')
gun_price = price.text if price else 'none'
description = soup.select_one('p[itemprop="description"]')
gun_description = description.text if description else 'none'
data = { 'Title': gun_title, 'Make': gun_makes, 'Model': gun_models, 'Licence': gun_licences, 'Orientation': gun_orients, 'Barrel Length': gun_barrels, 'Stock Length': gun_stocks, 'Chokes': gun_chokes, 'Origin': gun_origins, 'Trigger': gun_triggers, 'Ejection': gun_ejections, 'Scope': gun_scopes, 'Serial No': gun_serials, 'Stock No': gun_stocknos, 'Condition': gun_conditions, 'Description': gun_description, 'Price': gun_price}
df = DataFrame(data, columns = ['Title', 'Make', 'Model', 'Licence', 'Orientation', 'Barrel Length', 'Stock Length', 'Chokes', 'Origin', 'Trigger', 'Ejection', 'Scope', 'Serial No', 'Stock No', 'Condition', 'Description', 'Price'], index=[0])
df.to_csv(r'Guntrader_Dealer.csv')

I've wrote down the script for you. Instead of overwriting different df's to same file, I've created main df which appends all df's in for loop.
Here is the final code:
from bs4 import BeautifulSoup
import requests
import csv
import pandas
from pandas import DataFrame
import re
import os
import locale
os.environ["PYTHONIOENCODING"] = "utf-8"
#csv creation
with open('Guntrader_Dealer.csv', mode='w') as csv_file:
fieldnames = ['Title', 'Make', 'Model', 'Licence', 'Orientation', 'Barrel Length', 'Stock Length', 'Chokes', 'Origin', 'Trigger', 'Ejection', 'Scope', 'Serial No', 'Stock No', 'Condition', 'Description', 'Price']
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writeheader()
all_links=[]
#grab all links which contain the href specifed
url="https://www.guntrader.uk/dealers/minsterley/minsterley-ranges/guns?page={}"
for page in range(1,3):
res=requests.get(url).text
soup=BeautifulSoup(res,'html.parser')
for link in soup.select('a[href*="dealers/minsterley/minsterley-ranges/guns/"]'):
all_links.append("https://www.guntrader.uk" + link['href'])
df_main = DataFrame(columns = ['Title', 'Make', 'Model', 'Licence', 'Orientation', 'Barrel Length', 'Stock Length', 'Chokes', 'Origin', 'Trigger', 'Ejection', 'Scope', 'Serial No', 'Stock No', 'Condition', 'Description', 'Price'])
for a_link in all_links:
def make_span(make):
return make.name=='span' and 'Make:' in make.parent.contents[0]
def model_span(model):
return model.name=='span' and 'Model:' in model.parent.contents[0]
def licence_span(licence):
return licence.name=='span' and 'Licence:' in licence.parent.contents[0]
def orient_span(orient):
return orient.name=='span' and 'Orient.:' in orient.parent.contents[0]
def barrel_span(barrel):
return barrel.name=='span' and 'Barrel:' in barrel.parent.contents[0]
def stock_span(stock):
return stock.name=='span' and 'Stock:' in stock.parent.contents[0]
def choke_span(choke):
return choke.name=='span' and 'Chokes:' in choke.parent.contents[0]
def origin_span(origin):
return origin.name=='span' and 'Origin:' in origin.parent.contents[0]
def trigger_span(trigger):
return trigger.name=='span' and 'Trigger:' in trigger.parent.contents[0]
def ejection_span(ejection):
return ejection.name=='span' and 'Ejection:' in ejection.parent.contents[0]
def serial_span(serial):
return serial.name=='span' and 'Serial #:' in serial.parent.contents[0]
def stockno_span(stockno):
return stockno.name=='span' and 'Stock #:' in stockno.parent.contents[0]
def condition_span(condition):
return condition.name=='span' and 'Condition:' in condition.parent.contents[0]
def scope_span(scope):
return scope.name=='span' and 'Scope:' in scope.parent.contents[0]
res = requests.get(a_link)
soup = BeautifulSoup(res.text, 'html.parser')
makes = soup.find(make_span)
gun_makes = makes.content if makes else 'none'
models = soup.find(model_span)
gun_models = models.contents if models else 'none'
licences = soup.find(licence_span)
gun_licences = licences.contents if licences else 'none'
orients = soup.find(orient_span)
gun_orients = orients.contents if orients else 'none'
barrels = soup.find(barrel_span)
gun_barrels = barrels.contents if barrels else 'none'
stocks = soup.find(stock_span)
gun_stocks = stocks.contents if stocks else 'none'
chokes = soup.find(choke_span)
gun_chokes = chokes.contents if chokes else 'none'
origins = soup.find(origin_span)
gun_origins = origins.contents if origins else 'none'
triggers = soup.find(trigger_span)
gun_triggers = triggers.contents if triggers else 'none'
ejections = soup.find(ejection_span)
gun_ejections = ejections.contents if ejections else 'none'
scopes = soup.find(scope_span)
gun_scopes = scopes.contents if scopes else 'none'
serials = soup.find(serial_span)
gun_serials = serials.contents if serials else 'none'
stocknos = soup.find(stockno_span)
gun_stocknos = stocknos.contents if stocknos else 'none'
conditions = soup.find(condition_span)
gun_conditions = conditions.contents if conditions else 'none'
title = soup.select_one('h1[itemprop="name"]')
gun_title = title.text if title else 'none'
price = soup.select_one('p.price')
gun_price = price.text if price else 'none'
description = soup.select_one('p[itemprop="description"]')
gun_description = description.text if description else 'none'
data = { 'Title': gun_title, 'Make': gun_makes, 'Model': gun_models, 'Licence': gun_licences, 'Orientation': gun_orients, 'Barrel Length': gun_barrels, 'Stock Length': gun_stocks, 'Chokes': gun_chokes, 'Origin': gun_origins, 'Trigger': gun_triggers, 'Ejection': gun_ejections, 'Scope': gun_scopes, 'Serial No': gun_serials, 'Stock No': gun_stocknos, 'Condition': gun_conditions, 'Description': gun_description, 'Price': gun_price}
df = DataFrame(data, columns = ['Title', 'Make', 'Model', 'Licence', 'Orientation', 'Barrel Length', 'Stock Length', 'Chokes', 'Origin', 'Trigger', 'Ejection', 'Scope', 'Serial No', 'Stock No', 'Condition', 'Description', 'Price'], index=[0])
df_main = df_main.append(df, ignore_index = True)
df_main.to_csv('Guntrader_Dealer.csv', encoding='UTF-8')

Search for key word and convert the output into new line after it matches in python

I have below List output from the a code which i'm working in python where i'm specifically looking for memberUid string and want every names after that to be printed into new line:
like:
anshulm
jiafan
and while prnting these names as soon it will get 'cn' just stop the print.
[[('cn=oracle,ou=Group,ou=corp,ou=services,o=kk.com', {'description': ['oracle group'], 'businessCategory': ['Private'], 'objectClass': ['top', 'groupOfUniqueNames', 'posixGroup'], 'memberUid': ['anshulm', 'jiafan', 'manasij', 'asbisht', 'karnika', 'junle', 'amitsh', 'fuwei', 'dewansh', 'gouravr', 'harshitb', 'tandel', 'matte', 'izamir', 'elie', 'emiliano', 'mateuszw', 'theo', 'mahdi', 'hassan', 'gshruti', 'makhiles', 'prabhaka', 'shgarg', 'ritolia', 'wadhwani', 'steev', 'rtlsbld', 'nikhilb', 'fwang', 'ankitb', 'rtls', 'amitb', 'agautam', 'pratyush', 'hywang', 'dsouder', 'foutz', 'parimi', 'pradeepn', 'patrickg', 'pkunwar', 'tejinder', 'ramteke', 'jangra', 'kush', 'kundan', 'mohang', 'xiang', 'xinjia', 'anantv', 'christos', 'achugh', 'kbhatt', 'jroy', 'kusantos', 'kamleshm', 'iraa', 'indrajit'], 'gidNumber': ['9393'], 'owner': ['varshney'], 'cn': ['oracle']})]]
Below is my code which is yielding the above output:
import ldap
## first you must open a connection to the server
try:
l = ldap.initialize("ldap://ldapserver:389")
l.protocol_version = ldap.VERSION3
except ldap.LDAPError, e:
print e
baseDN = "ou=group,ou=corp,ou=services,o=kk.com"
searchScope = ldap.SCOPE_SUBTREE
retrieveAttributes = None
searchFilter = raw_input("Enter the Group Name: ")
try:
ldap_result_id = l.search(baseDN, searchScope, searchFilter, retrieveAttributes)
result_set = []
while 1:
result_type, result_data = l.result(ldap_result_id, 0)
if (result_data == []):
break
else:
if result_type == ldap.RES_SEARCH_ENTRY:
result_set.append(result_data)
print result_set
except ldap.LDAPError, e:
print e

You should extract desired part from result_set, for example:
result_set[0][0][1]['memberUid']
and print it with any manner you like:
from pprint import pprint
pprint(result_set[0][0][1]['memberUid'])
or
print('\n'.join(name for name in result_set[0][0][1]['memberUid']))

Convert a csv into category-subcategory using array

Above is the input table i have in csv
I am trying to use array and while loops in python. I am new to this language. Loops should occur twice to give Category\sub-category\sub-category_1 order...I am trying to use split().Ouput should be like below
import csv
with open('D:\\test.csv', 'rb') as f:
reader = csv.reader(f, delimiter='',quotechar='|')
data = []
for name in reader:
data[name] = []

And if you read the lines of your csv and access the data then you can manipulate the way you want later.
cats = {}
with open('my.csv', "r") as ins:
# check each line of the fine
for line in ins:
# remove double quotes: replace('"', '')
# remove break line : rstrip()
a = str(line).replace('"', '').rstrip().split('|')
if a[0] != 'CatNo':
cats[int(a[0])] = a[1:];
for p in cats:
print 'cat_id: %d, value: %s' % (p, cats[p])
# you can access the value by the int ID
print cats[1001]
the output:
cat_id: 100, value: ['Best Sellers', 'Best Sellers']
cat_id: 1001, value: ['New this Month', 'New Products\\New this Month']
cat_id: 10, value: ['New Products', 'New Products']
cat_id: 1003, value: ['Previous Months', 'New Products\\Previous Months']
cat_id: 110, value: ['Promotional Material', 'Promotional Material']
cat_id: 120, value: ['Discounted Products & Special Offers', 'Discounted Products & Special Offers']
cat_id: 1002, value: ['Last Month', 'New Products\\Last Month']
['New this Month', 'New Products\\New this Month']
Updated script for your question:
categories = {}
def get_parent_category(cat_id):
if len(cat_id) <= 2:
return '';
else:
return cat_id[:-1]
with open('my.csv', "r") as ins:
for line in ins:
# remove double quotes: replace('"', '')
# remove break line : rstrip()
a = str(line).replace('"', '').rstrip().split('|')
cat_id = a[0]
if cat_id != 'CatNo':
categories[cat_id] = {
'parent': get_parent_category(cat_id),
'desc': a[1],
'long_desc': a[2]
};
print 'Categories relations:'
for p in categories:
parent = categories[p]['parent']
output = categories[p]['desc']
while parent != '':
output = categories[parent]['desc'] + ' \\ ' + output
parent = categories[parent]['parent']
print '\t', output
output:
Categories relations:
New Products
New Products \ Best Sellers
New Products \ Discounted Products & Special Offers
New Products \ Best Sellers \ Previous Months
New Products \ Best Sellers \ Last Month
New Products \ Best Sellers \ New this Month