I am trying to read Blob-Data which is stored in a csv-file. I can import it in string format but if I want to use numpy.frombuffer with dtype='<f4' (fixed to get correct output), I get the error:
line 52, in <module>
data = np.frombuffer(blob_data, dtype='<f4') #<f4
ValueError: buffer size must be a multiple of element size
The code is the following:
import numpy as np
import datetime
import math
import csv
import pandas
from binascii import unhexlify
#import mysql.connector
# from pylab import figure, plot, show, legend, xlabel, ylabel
from matplotlib import pyplot as plt
def read_CSV(dataid):
with open('spectrometer2.csv') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
line_count = 0
result = bytes('1',encoding='UTF-8')
for row in csv_reader:
if line_count > 0:
#print(row[0])
if str(dataid) == str(row[0]):
result = row[3][2:-1]
print('FOUND####################################')
print(str(row[3])[2:-1])
break
line_count += 1
print(str(type(result)))
return result
####### MAIN PROGRAM #######
#spectrumRange = np.arange(10, 11011, 200)
spectrumRange = np.arange(8000, 9000, 200) # Auszug zur Leistungsoptimierung im Test
query_init = "SELECT * FROM `spectrometer2` WHERE data_id="
plt.figure()
for spectrum_id in spectrumRange:
spectrometerquery = query_init + str(spectrum_id) + ";"
print("Current data ID: " + str(spectrum_id))
#y_stream = interact_with_MySQL("database_name", spectrometerquery)
blob_data = read_CSV(spectrum_id)
#print(binary2int(blob_data))
if(blob_data != 0):
blob_data = bytes(blob_data,encoding = 'UTF-8')
print(blob_data)
data = np.frombuffer(blob_data, dtype='<f4') #<f4
print(data)
plt.plot(data)
plt.title('Spectrometer 2 data')
legend = []
for x in spectrumRange:
legend.append(('data id:', x))
plt.legend(legend)
plt.show()
I don't know much about the csv-file but it should contain the output of a optical sensor. The sql-statements are commented out because I have to replace them with the csv-file.
Extract of the file (shown in excel):
Related
im try to put my loop numpy data Modus_citra into csv file, but i've tried using numpy and using normal write csv didn't work
import glob
import cv2
from os import listdir
from os.path import isfile, join
import os
import numpy as np
from sklearn.utils.validation import check_array
import csv
import pandas as pd
def find_mode(np_array) :
vals,counts = np.unique(np_array, return_counts=True)
index = np.argmax(counts)
return(vals[index])
folder = ("C:/Users/ROG FLOW/Desktop/Untuk SIDANG TA/Sudah Aman/testbikincsv/folderdatacitra/*.jpg")
for file in glob.glob(folder):
image = cv2.imread(file)
rows = image.shape[0]
cols = image.shape[1]
middlex = cols/2
middley = rows/2
middle = [middlex,middley]
titikawalx = middlex - 10
titikawaly = middley - 10
titikakhirx = middlex + 10
titikakhiry = middley + 10
crop = image[int(titikawaly):int(titikakhiry), int(titikawalx):int(titikakhirx)]
c = cv2.cvtColor(crop, cv2.COLOR_BGR2HSV)
H,S,V = cv2.split(c)
Modus_citra = (find_mode(H))
data = (Modus_citra)
with open("foo.csv", 'w') as file:
writer = csv.writer(file)
writer.writerows(data)
error = 'numpy.uint8' object is not iterable
foo.csv = from pictures
60
40
19
11
please can someone help me ? i Appreciate every help
According to the edit, you can try:
folder = "C:/Users/ROG FLOW/Desktop/Untuk SIDANG TA/Sudah Aman/testbikincsv/folderdatacitra/*.jpg"
all_data = []
for file in glob.glob(folder):
# ...
Modus_citra = find_mode(H)
all_data.append(Modus_citra) # <-- add the numpy.uint8 to the all_data list
# after the loop write the data to the CSV file:
with open("foo.csv", "w") as file:
writer = csv.writer(file)
for data in all_data:
writer.writerow([data])
import pyscreenshot as ImageGrab
import time
images_folder = "captured_images/0/"
for i in range(0, 50):
time.sleep(5)
im = ImageGrab.grab(bbox=(150,350,900,950)) #x1, y1, x2, y2
print("saved......",i)
im.save(images_folder+str(i)+'.png')
print("claer screen now and redraw now......")
import cv2
import csv
import glob
header = ["label"]
for i in range(0,784):
header.append("pixel"+str(i))
with open('dataset.csv', 'a') as f:
writer = csv.writer(f)
writer.writerow(header)
for label in range(10):
dirList = glob.glob("captured_images/"+str(label)+"/*.png")
for img_path in dirList:
im = cv2.imread(img_path)
im_gray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)
im_gray = cv2.GaussianBlur(im_gray,(15,15), 0)
roi = cv2.resize(im_gray,(28,28), interpolation = cv2.INTER_AREA)
data=[]
data.append(label)
rows, cols = roi.shape
## Add pixel one by one into data array
for i in range(rows):
for j in range(cols):
k = roi[i,j]
if k>100:
k = 1
else:
k = 0
data.append(k)
with open('dataset.csv', 'a') as f:
writer = csv.writer(f)
writer.writerow(header)
import pandas as pd
from sklearn.utils import shuffle
data = pd.read_csv('dataset.csv')
data = shuffle(data)
data
I want to make a 'handwrite digit recognition', but when I checked the datasheet, the datasheet was created like image 1. I need to assign a value only to the place where pixels are, like image 2, so how should I fix the code?
You are writing the header into the csv instead of writing the data. It should be changed to
with open('dataset.csv', 'a') as f:
writer = csv.writer(f)
writer.writerow(data)
I got this code that has data stored in pickle format, I am trying to unpickle it. After it successfully ran, it was returning this output: <__main__.Container at 0x232ac6bde80>, but I want the data loaded into the DataFrame for further manipulations. I have searched StackOverflow for possible solutions to no avail. Kindly help. Here is the code:
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import rcParams
impor numpy as np
import tpickle
import skill_metrics as sm
from sys import version_info
def load_obj(name):
# Load object from file in pickle format
if version_info[0] == 2:
suffix = 'pkl'
else:
suffix = 'pkl3'
with open(name + '.' + suffix, 'rb') as f:
return pickle.load(f) # Python2 succeeds
class Container(object):
def __init__(self, pred1, pred2, pred3, ref):
self.pred1 = pred1
self.pred2 = pred2
self.pred3 = pred3
self.ref = ref
if __name__ == '__main__':
# Set the figure properties (optional)
rcParams["figure.figsize"] = [8.0, 6.4] #works
# rcParams['lines.linewidth'] = 2 # line width for plots
rcParams.update({'font.size': 12}) # font size of axes text
# Close any previously open graphics windows
plt.close("all")
# Read data from pickle file
load_obj('target_data')
df = pd.read_pickle('target_data.pkl')
df
Thanks.
I have the following code to read three components of two waves and extract each of them to a csv file.
I am still a beginner and so the code is very long, at least I don't want to type the wave name(EHMH011604150003.EW1 EHMH011604150003.NS1 ...) 6 times in my read part. how can I put the Variable that I defined as "name" there?
Any tips on making the code look smart is also appreciated.
Thank you
from pathlib import Path
import os
import numpy as np
import csv
#p =Path('D:/Jobs_2020/RJA/')
#p2 = p/'20160415波形'
#p3 = p2/'kik'
name = 'EHMH011604150003'
# Function 'getKiK-net'
def Convert2Acc(data):
tokens = data.split()
# Scale factor
(Scale, Factor) = tokens[tokens.index('Factor')+1].split('(gal)/')
# Strong motion
items = tokens[tokens.index('Memo.')+1:]
rdata = np.array(items, dtype=np.float64) # rdata: raw data
acc_gal = (rdata - rdata[0]) * float(Scale) / float(Factor)
return acc_gal # acc_gal: Acc. converted unit into gal
# Read data filess
rfile_EW1 = 'D:\\Jobs_2020\\RJA\\20160415波形\\kik\\EHMH011604150003.EW1'
fr_EW1 = open(rfile_EW1, 'r')
EW1_gal = fr_EW1.read()
fr_EW1.close()
rfile_NS1 = 'D:\\Jobs_2020\\RJA\\20160415波形\\kik\\EHMH011604150003.NS1'
fr_NS1 = open(rfile_NS1, 'r')
NS1_gal = fr_NS1.read()
fr_NS1.close()
rfile_UD1 = 'D:\\Jobs_2020\\RJA\\20160415波形\\kik\\EHMH011604150003.UD1'
fr_UD1 = open(rfile_UD1, 'r')
UD1_gal = fr_UD1.read()
fr_UD1.close()
rfile_EW2 = 'D:\\Jobs_2020\\RJA\\20160415波形\\kik\\EHMH011604150003.EW2'
fr_EW2 = open(rfile_EW2, 'r')
EW2_gal = fr_EW2.read()
fr_EW2.close()
rfile_NS2 = 'D:\\Jobs_2020\\RJA\\20160415波形\\kik\\EHMH011604150003.NS2'
fr_NS2 = open(rfile_NS2, 'r')
NS2_gal = fr_NS2.read()
fr_NS2.close()
rfile_UD2 = 'D:\\Jobs_2020\\RJA\\20160415波形\\kik\\EHMH011604150003.UD2'
fr_UD2 = open(rfile_UD2, 'r')
UD2_gal = fr_UD2.read()
fr_UD2.close()
# Store data in array
# _Acc: 2D Array
_Acc1 = [Convert2Acc(EW1_gal), Convert2Acc(NS1_gal), Convert2Acc(UD1_gal)]
Acc1 = np.array(_Acc1).T # Acc: Transposed 2D array to write to .csv file
_Acc2 = [Convert2Acc(EW2_gal), Convert2Acc(NS2_gal), Convert2Acc(UD2_gal)]
Acc2 = np.array(_Acc2).T # Acc: Transposed 2D array to write to .csv file
# Write to .csv file
with open(str(name)+'-1'+'.csv', 'w') as file:
writer = csv.writer(file, lineterminator='\n')
writer.writerows(Acc1)
with open(str(name)+'-2'+'.csv', 'w') as file:
writer = csv.writer(file, lineterminator='\n')
writer.writerows(Acc2)
Something like:
rfile_EW1 = 'D:\\Jobs_2020\\RJA\\20160415波形\\kik\\'+name+'.EW1'
should work.
Do I miss anything here? Why does this code not outputting data to the file I opened? Any ideas?
The following is the essential part of the entire code that has passed the complier without errors, but not outputting data to the file.
#! /usr/bin/python
#Basic imports
import sys
from time import sleep
from datetime import datetime,date,time
import numpy as np
#Create Bridge objects
try:
bridge_1 = Bridge()
outfile = open("prototype.csv", "a")
# Initialize all sensors and discard the readings
lc1_ini = bridge_1.getBridgeValue(0) * 2674.0 - 210.7
sleep(1)
lc1 = bridge_1.getBridgeValue(0) * 2674.0 - 210.7
# create empty array to store the converted digital data
readings_lc1 = np.empty([])
avg_lc1 = np.empty([])
max_samples = 3
readings_lc1 = np.append(readings_lc1 , lc1 )
if len(readings_lc1) == max_samples:
avg_lc1 = np.mean(readings_lc1[1:])
#Write the data to the text file
outfile.write(str(datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S')) + "," + str(round(lc1,2)) + "\n")
outfile.close()