We Keep Coding

Getting OS error with Windows (OSError: [WinError 193] %1 is not a valid Win32 application) and Linux with example code

I received a spectrometer sensor and I was trying to run its example Python API code to start working with it. I have tested this on Linux and I got an issue with OSError: libgsl.so.0 cannot open shared files error. So now I am trying to run the same code on Windows 10 but I seem to be getting another OS error. I am running their following code which produces the error at the beginning during #Initialization:
import sys
import csv
sys.path.append("..\..\..\..\wrappers/python")
if(sys.version_info[0] < 3):
from wrapper_python2 import *
from wrapper_python2.core import *
from wrapper_python2.device import *
from wrapper_python2.color import *
print ("**********************************************************************")
print ("[Python-2] Python Version : ", sys.version_info.major, "." ,sys.version_info.minor , " Detected")
print ("**********************************************************************")
else:
print ("**********************************************************************")
print ("[Python-3] Python Version : ", sys.version_info.major, "." ,sys.version_info.minor , " Detected")
print ("**********************************************************************")
from wrapper_python3 import *
from wrapper_python3.core import *
from wrapper_python3.device import *
from wrapper_python3.color import *
#Initialization
if sys.platform == 'win32':
initialize("..\Libs\CrystalBase.dll")
pSpecCore = initialize_core_api("..\Libs\CrystalCore.dll")
pSpecDevice = initialize_device_api("..\Libs\CrystalPort.dll")
else:
initialize("../Libs/libCrystalBase.so")
pSpecCore = initialize_core_api("../Libs/libCrystalCore.so")
pSpecDevice = initialize_device_api("../Libs/libCrystalPort.so")
initialize_color_api(pSpecCore)
connectReturn = connect_device(pSpecDevice) # return total num of devices connected with system
if connectReturn > 0:
(ret, sensorID) = get_sensor_id_device(pSpecDevice)
create_core_object(pSpecCore)
if sys.platform == 'win32':
csInit_Return = load_sensor_file(pSpecCore, b"..\config\sensor_" + sensorID + b".dat")
else:
csInit_Return = load_sensor_file(pSpecCore, b"../config/sensor_" + sensorID + b".dat")
if(csInit_Return > 0):
(ret, sensorID) = get_sensor_id_file(pSpecCore)
get_sensor_parameters_from_device(pSpecDevice)
(adcGain,adcRange) = get_sensor_parameters_from_calibration_file(pSpecCore)
settingReturn = set_sensor_parameters_to_device(pSpecDevice,adcGain,adcRange)
total_num_of_sensors = total_sensors_connected(pSpecDevice)
get_capacity_sensor_data_list(pSpecCore)
for index in range(total_num_of_sensors):
#activate a specific device(sensor)
activatingReturn = index_activation(pSpecDevice,index)
#get sensor id of currently activated device(sensor)
(ret, sensorID) = get_sensor_id_device(pSpecDevice)
#get and set shutter speed of device(sensor)
get_shutter_speed(pSpecDevice)
set_shutter_speed(pSpecDevice,1)
#get one filter output (sensor data)
filterData = get_filter_data(pSpecDevice,20)
#set background data
set_background_data(pSpecCore,filterData)
#get and set shutter speed of device(sensor)
#get_shutter_speed(pSpecDevice)
valid_filters_num = get_num_of_valid_filters(pSpecCore)
valid_filters = get_valid_filters(pSpecCore)
newSS = 50
frame_avg = 20
do_AE = True
#Get shutter speed with AE
if do_AE:
newSS = get_optimal_shutter_speed(pSpecDevice,valid_filters_num,valid_filters)
set_shutter_speed(pSpecDevice,newSS)
#convert shutter speed to exposure time (ms) for your reference
ss_to_exposure_time(pSpecDevice,5,newSS)
filterData = get_filter_data(pSpecDevice,frame_avg)
specSize = get_spectrum_length(pSpecCore)
(ret, specData,wavelengthdata) = calculate_spectrum(pSpecCore,filterData,newSS)
(Start_Wavelength, End_Wavelength, Interval_Wavelength) = get_wavelength_information(pSpecCore)
if sys.version_info[0] < 3:
fileName = (r"SpecrtumData2_" + sensorID + ".csv");
data = []
for i in range(get_spectrum_length(pSpecCore)):
data.append(str(specData[i]).split(","))
with open(fileName, "wb") as csv_file:
writer = csv.writer(csv_file, delimiter=',')
for line in data:
writer.writerow(line)
csv_file.close()
else:
fileName = (b"SpecrtumData3_" + sensorID + b".csv");
data = []
for i in range(get_spectrum_length(pSpecCore)):
data.append(str(specData[i]).split(","))
with open(fileName, 'w', newline='') as csvfile:
filewriter = csv.writer(csvfile, delimiter=',',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
for line in data:
filewriter.writerow(line)
csvfile.close()
else:
print ("**********************************************************************")
print ("[PrismError] Sensor Calibration File Not Present in Config Foler. Please copy Sensor Calibration File in Config file and execute again.")
print ("**********************************************************************")
else:
print ("**********************************************************************")
print ("[PrismError]Device Not Connected. Please connect Device and try again.")
print ("**********************************************************************")
close_color_api(pSpecCore)
close_core_object(pSpecCore)
disconnect_device(pSpecDevice)
However, when I run it, I get the following error and I am perplexed about how I go about resolving it for my specific case.
**********************************************************************
[Python-3] Python Version : 3 . 8 Detected
**********************************************************************
Traceback (most recent call last):
File "c:\Users\Jeff\Downloads\New folder\NSP32_SDK_1_7_windows.tar\nanoLambda\NSP32_SDK\examples\python\win32\example_wrapper_python\example_wrapper_python_color.py", line 24, in <module>
initialize("..\Libs\CrystalBase.dll")
File "../../../../wrappers/python\wrapper_python3\initialize.py", line 10, in initialize
ctypes.CDLL("..\Libs\pthreadVC2.dll")
File "C:\Users\Jeff\miniconda3\lib\ctypes\__init__.py", line 373, in __init__
self._handle = _dlopen(self._name, mode)
OSError: [WinError 193] %1 is not a valid Win32 application
This is the following directory structure for this application

python memoryerror - large loop xml to mongodb

I downloaded a zip file from https://clinicaltrials.gov/AllPublicXML.zip, which contains over 200k xml files (most are < 10 kb in size), to a directory (see 'dirpath_zip' in the CODE) I created in ubuntu 16.04 (using DigitalOcean). What I'm trying to accomplish is loading all of these into MongoDB (also installed in the same location as the zip file).
I ran the CODE below twice and consistently failed when processing the 15988th file.
I've googled around and tried reading other posts regarding this particular error, but couldn't find a way to solve this particular issue. Actually, I'm not really sure what problem really is... any help is much appreciated!!
CODE:
import re
import json
import zipfile
import pymongo
import datetime
import xmltodict
from bs4 import BeautifulSoup
from pprint import pprint as ppt
def timestamper(stamp_type="regular"):
if stamp_type == "regular":
timestamp = str(datetime.datetime.now())
elif stamp_type == "filename":
timestamp = str(datetime.datetime.now()).replace("-", "").replace(":", "").replace(" ", "_")[:15]
else:
sys.exit("ERROR [timestamper()]: unexpected 'stamp_type' (parameter) encountered")
return timestamp
client = pymongo.MongoClient()
db = client['ctgov']
coll_name = "ts_"+timestamper(stamp_type="filename")
coll = db[coll_name]
dirpath_zip = '/glbdat/ctgov/all/alltrials_20180402.zip'
z = zipfile.ZipFile(dirpath_zip, 'r')
i = 0
for xmlfile in z.namelist():
print(i, 'parsing:', xmlfile)
if xmlfile == 'Contents.txt':
print(xmlfile, '==> entering "continue"')
continue
else:
soup = BeautifulSoup(z.read(xmlfile), 'lxml')
json_study = json.loads(re.sub('\s', ' ', json.dumps(xmltodict.parse(str(soup.find('clinical_study'))))).strip())
coll.insert_one(json_study)
i+=1
ERROR MESSAGE:
Traceback (most recent call last):
File "zip_to_mongo_alltrials.py", line 38, in <module>
soup = BeautifulSoup(z.read(xmlfile), 'lxml')
File "/usr/local/lib/python3.5/dist-packages/bs4/__init__.py", line 225, in __init__
markup, from_encoding, exclude_encodings=exclude_encodings)):
File "/usr/local/lib/python3.5/dist-packages/bs4/builder/_lxml.py", line 118, in prepare_markup
for encoding in detector.encodings:
File "/usr/local/lib/python3.5/dist-packages/bs4/dammit.py", line 264, in encodings
self.chardet_encoding = chardet_dammit(self.markup)
File "/usr/local/lib/python3.5/dist-packages/bs4/dammit.py", line 34, in chardet_dammit
return chardet.detect(s)['encoding']
File "/usr/lib/python3/dist-packages/chardet/__init__.py", line 30, in detect
u.feed(aBuf)
File "/usr/lib/python3/dist-packages/chardet/universaldetector.py", line 128, in feed
if prober.feed(aBuf) == constants.eFoundIt:
File "/usr/lib/python3/dist-packages/chardet/charsetgroupprober.py", line 64, in feed
st = prober.feed(aBuf)
File "/usr/lib/python3/dist-packages/chardet/hebrewprober.py", line 224, in feed
aBuf = self.filter_high_bit_only(aBuf)
File "/usr/lib/python3/dist-packages/chardet/charsetprober.py", line 53, in filter_high_bit_only
aBuf = re.sub(b'([\x00-\x7F])+', b' ', aBuf)
File "/usr/lib/python3.5/re.py", line 182, in sub
return _compile(pattern, flags).sub(repl, string, count)
MemoryError

Try to push reading from file and inserting into db in another method.
Also add gc.collect() for garbage collection.
import gc;
def read_xml_insert(xmlfile):
soup = BeautifulSoup(z.read(xmlfile), 'lxml')
json_study = json.loads(re.sub('\s', ' ', json.dumps(xmltodict.parse(str(soup.find('clinical_study'))))).strip())
coll.insert_one(json_study)
for xmlfile in z.namelist():
print(i, 'parsing:', xmlfile)
if xmlfile == 'Contents.txt':
print(xmlfile, '==> entering "continue"')
continue;
else:
read_xml_insert(xmlfile);
i+=1
gc.collect()
`
Please see.

Printing Arabic characters in python/django

I have a script that modifies data in a django app.I have data in an excel file that i process then update my models with it, some of the data is in Arabic and when i execute the script i get the following error:
Traceback (most recent call last):
File "script.py", line 77, in <module>
update_locations(path)
File "script.py", line 36, in update_locations
household.location = new_location
File "/data/envs/ve.maidea/lib/python2.7/site-packages/django/db/models/fields/related_descriptors.py", line 207, in __set__
self.field.remote_field.model._meta.object_name,
ValueError: Cannot assign "'\xd8\xa7\xd9\x84\xd8\xa8\xd8\xad\xd9\x8a\xd8\xb1\xd9\x87'": "Household.location" must be a "Location" instance.
I think the error is been raised by these Arabic characters.
here is my script:
import django
django.setup()
import sys
reload(sys) # to re-enable sys.setdefaultencoding()
sys.setdefaultencoding('utf-8')
import xlrd
from django.db import transaction
from foodnet.apps.registration.models import Household
from geo.models import Location
log_file = "/opt/cv_instances/cv1/autodeploy/branches/nboreports/maidea/egypt/data_import_files/egypt_beheira_locations.txt"
logfile_to_write = open(log_file, "w")
def process_file(path):
book = xlrd.open_workbook(path)
print("Got {0} number of sheets.".format(book.nsheets))
hh_counter = 0
for sheet_num in range(book.nsheets-1, -1, -1):
sheet = book.sheet_by_index(sheet_num)
print("Processing sheet number {0} ({1})".format(sheet_num, sheet.name))
for row_idx in range(1, sheet.nrows):
with transaction.atomic():
try:
household_name = str(sheet.row_values(row_idx)[0]).strip().replace(".0","")
# old_location = str(sheet.row_values(row_idx)[1]).strip().replace(".0","")
new_location = str(sheet.row_values(row_idx)[2]).strip().replace(".0","")
if household_name:
household = Household.objects.get(office__slug='eg-co',name=household_name)
# print(household.name, household.location)
#update new locations
household.location = new_location
household.save()
hh_counter += 1
logfile_to_write.write("Household {0} updated to location {1}".format(household, household.location))
except Household.DoesNotExist:
continue
print("Done looping and updating locations")
print("================================================================================================================================")
def delete_old_locations(path):
"""
Delete old locations no longer needed by the country office
"""
book = xlrd.open_workbook(path)
print("Got {0} number of sheets.".format(book.nsheets))
location_counter = 0
for sheet_num in range(book.nsheets-1, -1, -1):
sheet = book.sheet_by_index(sheet_num)
print("Processing sheet number {0} ({1})".format(sheet_num, sheet.name))
for row_idx in range(1, sheet.nrows):
with transaction.atomic():
try:
old_location = str(sheet.row_values(row_idx)[1]).strip().replace(".0","")
if old_location:
location = Location.objects.get(country__name="Egypt", name=old_location)
# print(location.name, location.country)
location.delete()
location_counter += 1
logfile_to_write.write("Location {0} deleted ".format(location))
except Location.DoesNotExist:
continue
print("Done looping and deleting locations")
print("================================================================================================================================")
#call the our process file method
if __name__=="__main__":
path = "/opt/cv_instances/cv1/autodeploy/branches/nboreports/maidea/egypt/data_import_files/egypt-sf-beheira-enrolments.xlsx"
process_file(path)
delete_old_locations(path)
print("Done processing file")
I kindly need advice on the best way of printing this arabic characters.Thanks in advance.

This has nothing to do with Arabic characters. As the error says, you need to assign an instance of Location there, not a string.

reading and writing a tempfile

Hopefully someone can help me out. This seems like a very simple question that I just can't find the answer to. I am trying to create a tempfile, and then using this same tempfile, I'd like to write into it using the dd command. And then open that same file and time the time it takes to read the file.
So basically:
create tempfile
then, dd if=/dev/zero of=tempfile
and finally, time how long it takes to open and read tempfile
I'm not sure why, but this is the error I'm getting. TypeError: coercing to Unicode: need string or buffer, instance found. I think it's because I've got the same file open at the same time, but not sure. Any ideas?
Here's the code:
import time
import tempfile
import subprocess
import argparse
def readfile(size, block_size, path):
with tempfile.NamedTemporaryFile(prefix='iospeeds-', dir=path, delete=True) as tf:
cmd = ['dd', 'if=/dev/zero', 'of={}'.format(tf), 'bs={}'.format(block_size), 'count={}'.format(size/block_size)]
subprocess.call(cmd, stderr=subprocess.STDOUT)
start_time = time.time()
with open(tf, 'rb') as read_file:
end_time = time.time()
total_time = start_time - end_time
print total_time
return total_time
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--size', type=int, default=1048576)
parser.add_argument('--block-size', type=int, default=4096)
parser.add_argument('--path', default='./')
return parser.parse_args()
def main():
args=parse_args()
size = args.size
block_size = args.block_size
path = args.path
readfile(size, block_size, path)
if __name__ == "__main__":
main()
Here's the Traceback:
Traceback (most recent call last):
File "./rd.py", line 38, in <module>
main()
File "./rd.py", line 35, in main
readfile(size, block_size, path)
File "./rd.py", line 14, in readfile
with open(tf, 'rb') as read_file:
Thanks!

You're trying to open a file with a file type in the name spot, basically you're trying to do open(file, 'rb') instead of open(filename, 'rb'). Try:
with open(tf.name, 'rb') as read_file:

Gzip problem,traceback and IOError: [Errno 2] No such file or directory

I'm new to python and bioinformatics field. I'm using python-2.6. Now I'm trying to select all fastq.gz files, then gzip.open(just a few lines because it's too huge and time-wasting), then count 'J' , then pick out those files with 'J' count NOT equal to 0.
The following is my code:
#!/usr/bin/python
import os,sys,re,gzip
path = "/home/XXX/nearline"
for file in os.listdir(path):
if re.match('.*\.recal.fastq.gz', file):
text = gzip.open(file,'r').readlines()[:10]
word_list = text.split()
number = word_list.count('J') + 1
if number !== 0:
print file
But I got some errors:
Traceback (most recent call last):
File "fastqfilter.py", line 9, in <module>
text = gzip.open(file,'r').readlines()[:10]
File "/share/lib/python2.6/gzip.py", line 33, in open
return GzipFile(filename, mode, compresslevel)
File "/share/lib/python2.6/gzip.py", line 79, in __init__
fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb')
IOError: [Errno 2] No such file or directory: 'ERR001268_1.recal.fastq.gz'
What's this traceback: File......
Is there anything wrong with gzip here?
And why can't it find ERR001268_1.recal.fastq.gz? It's the first fastq file in the list, and DOES exist there.
Hope give me some clues, and any point out any other errors in the script.
THanks a lot.
Edit: thx everyone. I followed Dan's suggestion. And I tried on ONE fastq file first. My script goes like:
#!/usr/bin/python
import os,sys
import gzip
import itertools
file = gzip.open('/home/xug/nearline/ERR001274_1.recal.fastq.gz','r')
list(itertools.islice(file.xreadlines(),10))
word_list = list.split()
number = word_list.count('J') + 1
if number != 0:
print 'ERR001274_1.recal.fastq.gz'
Then errors are:
Traceback (most recent call last):
File "try2.py", line 8, in <module>
list(itertools.islice(text.xreadlines(),10))
AttributeError: GzipFiles instance has no attribute 'xreadlines'
Edit again: Thx Dan, I've solved the problem yesterday. Seems GzipFiles don't support xreadlines. So I tried the similar way as you suggested later. And it works. See below:
#!/usr/bin/python
import os,sys,re
import gzip
from itertools import islice
path = "/home/XXXX/nearline"
for file in os.listdir(path):
if re.match('.*\.recal.fastq.gz', file):
fullpath = os.path.join(path, file)
myfile = gzip.open(fullpath,'r')
head = list(islice(myfile,1000))
word_str = ";".join(str(x) for x in head)
number = word_str.count('J')
if number != 0:
print file

on this line:
text = gzip.open(file,'r').read()
file is a filename not a full path so
fullpath = os.path.join(path, file)
text = gzip.open(fullpath,'r').read()
about F.readlines()[:10] will read the whole file in to a list of lines and then take the first 10
import itertools
list(itertools.islice(F.xreadlines(),10))
this will not read the whole file into memory and will only read the first 10 lines into a list
but as gzip.open returns an object that doesn't have .xreadlines() and but as files are iterable on their lines just:
list(itertools.islice(F,10))
would work as this test shows:
>>> import gzip,itertools
>>> list(itertools.islice(gzip.open("/home/dan/Desktop/rp718.ps.gz"),10))
['%!PS-Adobe-2.0\n', '%%Creator: dvips 5.528 Copyright 1986, 1994 Radical Eye Software\n', '%%Title: WLP-94-int.dvi\n', '%%CreationDate: Mon Jan 16 16:24:41 1995\n', '%%Pages: 6\n', '%%PageOrder: Ascend\n', '%%BoundingBox: 0 0 596 842\n', '%%EndComments\n', '%DVIPSCommandLine: dvips -f WLP-94-int.dvi\n', '%DVIPSParameters: dpi=300, comments removed\n']

Change your code to:
#!/usr/bin/python
import os,sys,re,gzip
path = "/home/XXX/nearline"
for file in os.listdir(path):
if re.match('.*\.recal.fastq.gz', file):
text = gzip.open(os.path.join(path,file),'r').readlines()[:10]
word_list = text.split()
number = word_list.count('J') + 1
if number !== 0:
print file

It's trying to open ERR001268_1.recal.fastq.gz from the working directory, not from /home/XXX/nearline.