I am totally new to python and just started with it.
I wrote following script go get some data from the SenseHat, but unfortunately it doesn't do what it should.
It should get the temp, humidity and pressure from the SenseHat and write in to a file "data.csv" every 10 minutes.
Without the sleep(600) the script works fine, but logs to much data. With the sleep function it seems it does nothing...
This is my script:
from sense_hat import SenseHat
from datetime import datetime
from csv import writer
from time import sleep
sense = SenseHat()
def get_sense_data():
sense_data = []
sense_data.append(datetime.now())
sense_data.append(sense.get_temperature())
sense_data.append(sense.get_pressure())
sense_data.append(sense.get_humidity())
return sense_data
#while True:
# print(get_sense_data())
with open('data.csv', 'wb') as f:
data_writer = writer(f)
data_writer.writerow(['time','temp','pres','hum'])
while True:
data = get_sense_data()
data_writer.writerow(data)
sleep(600)from sense_hat import SenseHat
from datetime import datetime
from csv import writer
from time import sleep
sense = SenseHat()
def get_sense_data():
sense_data = []
sense_data.append(datetime.now())
sense_data.append(sense.get_temperature())
sense_data.append(sense.get_pressure())
sense_data.append(sense.get_humidity())
return sense_data
#while True:
# print(get_sense_data())
with open('data.csv', 'wb') as f:
data_writer = writer(f)
data_writer.writerow(['time','temp','pres','hum'])
while True:
data = get_sense_data()
data_writer.writerow(data)
sleep(600)
Please could anyone help me and explain to me what is wrong...?
Output to files is buffered. If you want to see the intermediate results of the script while it's sleeping, you need to flush the buffer.
with open('data.csv', 'wb') as f:
data_writer = writer(f)
data_writer.writerow(['time','temp','pres','hum'])
while True:
data = get_sense_data()
data_writer.writerow(data)
f.flush()
sleep(600)
Related
I found and edited some code that records water flow data off of a flowmeter. I have managed to edit the script to run for only 5 mins and write to a file (I tried a .csv file but I am realizing this may not be entirely possible). However, when the script runs, the data file created just lists one "row" of recorded data. The while loop runs every 5 seconds for about 5 mins and provides a timestamp and a flow-rate reading, I'm trying to get the script to continuously record data. Here is the code:
import json
import time
from datetime import datetime, timedelta
import RPi.GPIO as GPIO
import csv
class FlowMeter():
''' Class representing the flow meter sensor which handles input pulses
and calculates current flow rate (L/min) measurement
'''
def __init__(self):
self.flow_rate = 0.0
self.last_time = datetime.now()
def pulseCallback(self, p):
''' Callback that is executed with each pulse
received from the sensor
'''
# Calculate the time difference since last pulse recieved
current_time = datetime.now()
diff = (current_time - self.last_time).total_seconds()
# Calculate current flow rate
hertz = 1. / diff
self.flow_rate = hertz / 7.5
# Reset time of last pulse
self.last_time = current_time
def getFlowRate(self):
''' Return the current flow rate measurement.
If a pulse has not been received in more than one second,
assume that flow has stopped and set flow rate to 0.0
'''
if (datetime.now() - self.last_time).total_seconds() > 1:
self.flow_rate = 0.0
return self.flow_rate
def main():
''' Main function for repeatedly collecting flow rate measurements
and sending them to the SORACOM API
'''
# Configure GPIO pins
INPUT_PIN = 7
GPIO.setmode(GPIO.BOARD)
GPIO.setup(INPUT_PIN, GPIO.IN, pull_up_down=GPIO.PUD_UP)
# Init FlowMeter instance and pulse callback
flow_meter = FlowMeter()
GPIO.add_event_detect(INPUT_PIN,
GPIO.RISING,
callback=flow_meter.pulseCallback,
bouncetime=20)
end_time = datetime.now() + timedelta(minutes=5)
while True:
timestamp = str(datetime.now())
flow_rate = flow_meter.getFlowRate()
time.sleep(5)
print('Timestamp: %s' % timestamp)
print('Flow rate: %f' % flow_rate)
header = ['Time','Flow Rate']
data = [timestamp, flow_rate]
with open('flowrate.csv', 'w', encoding='UTF8', newline='') as f:
writer = csv.writer(f)
writer.writerow(header)
writer.writerow(data)
if datetime.now() > end_time:
break
if __name__ == '__main__':
main()
I've included my "data writing" strings near the bottom in the while loop.
Opening the file with mode 'w' overwrites the whole file every time. So, just write the header at the beginning, and "append" later on:
def main()
with open('flowrate.csv', 'w', encoding='UTF8', newline='') as f:
writer = csv.writer(f)
writer.writerow(header)
... later, in the loop ...
with open('flowrate.csv', 'a', encoding='UTF8', newline='') as f:
writer = csv.writer(f)
writer.writerow(data)
This is only practical if you're really only writing a row every now and then. If you're writing more often, then you should just keep the file open and reuse the writer instance.
Your problem is you're opening the file and effectively truncating (rewriting) it each time. Instead, open the file, then run the loop.
with open('flowrate.csv', 'w', encoding='UTF8', newline='') as f:
writer = csv.writer(f)
writer.writerow(header)
while True:
timestamp = str(datetime.now())
flow_rate = flow_meter.getFlowRate()
time.sleep(5)
print('Timestamp: %s' % timestamp)
print('Flow rate: %f' % flow_rate)
header = ['Time','Flow Rate']
data = [timestamp, flow_rate]
writer.writerow(data)
if datetime.now() > end_time:
break
I'm trying to write huge data to a csv file. When I try normal method it writes 50 data in 1 second but with multiprocessing it's down to 5 data in 1 second.
And I also added this code sys.setrecursionlimit(25000). Because without it's giving error.
I can feel I'm not doing right. What is the right way?
from bs4 import BeautifulSoup
import requests
import lxml
import csv
import cchardet
from multiprocessing import Pool
import sys
import time
sys.setrecursionlimit(25000)
csvfileWrite=open("comments.csv", 'a+', newline='',encoding='utf-8') #declared as a global variable
writer = csv.writer(csvfileWrite, delimiter=';', quotechar='"',
quoting=csv.QUOTE_MINIMAL) #declared as a global variable
def kacYildiz(div): #This function returns a number 0 to 5. Not important.
yildizSayisi=0
yildizYeri=div.find("div",attrs={"class":"RatingPointer-module-1OKF3"})
yildizlar=yildizYeri.find_all("svg")
for yildiz in yildizlar:
sonuc=yildiz.find("path").get("fill")
if(sonuc=="#f28b00"):
yildizSayisi+=1
return yildizSayisi
def takeText(div):
comment=div.find("span",attrs={"itemprop":"description"}).text
return comment
def yorumSayfaSayisi(row): # This function returns a number that how many
pages in the sites comment section. Not important.
yorumKismi="-yorumlari?"
adres=row[0]+yorumKismi
r = requests_session.get(adres)
soup = BeautifulSoup(r.text,"lxml")
sayfaS=soup.find("ul",attrs={"class":"PaginationBar-module-3qhrm"})
sayi=sayfaS.find_all("li")[-1].text
return sayi
def writeToCsv(comments): #writing commets to csv file.
global csvfileWrite
global writer
textToWrite = takeText(comments)
writer.writerow([kacYildiz(comments),textToWrite])
if __name__ == '__main__':
pageNumber=1
requests_session = requests.Session()
comments=list()
csvfile=open('adresler.csv',newline='')
reader = csv.reader(csvfile, delimiter=';', quotechar='|')
for row in reader:
rowNumber=yorumSayfaSayisi(row)
for i in range(1,int(rowNumber)):
comments.clear()
commetAdress="-yorumlari?sayfa={}".format(i)
adress=row[0]+commetAdress
r = requests_session.get(adress)
soup = BeautifulSoup(r.text,"lxml")
page=soup.find_all("div",attrs={"class":"ReviewCard-module-
3Y36S"})
for comment in page:
comments.append(comment)
p = Pool(10)
start = time.process_time()
p.map(writeToCsv, comments)
p.terminate()
p.join()
once try this approach using ThreadPool
from multiprocessing.pool import ThreadPool
def csvYaz(yorumlar):
global csvfileYaz
global yazici
yazi = yorumAl(yorumlar)
yazici.writerow([kacYildiz(yorumlar),yazi])
# ------main-----
for yorum in yorumSayfasi:
yorumlar.append(yorum)
threads = ThreadPool(10).map(csvYaz, yorumlar)
for zz in threads:
print(zz)
I would like to print the duration of the script and save it inside the csv. What would be the best method?
import requests
with open('infile.txt', 'r') as f:
urls = f.readlines()
datalist=[]
for url in urls:
data = requests.get(url)
datalist.append(data.text)
with open('outfile.txt', 'w') as f:
for item in datalist:
f.write("%s\n" % item)
You can use datetime module.
import requests
from datetime import datetime
def run():
with open('infile.txt', 'r') as f:
urls = f.readlines()
datalist=[]
for url in urls:
data = requests.get(url)
datalist.append(data.text)
with open('outfile.txt', 'w') as f:
for item in datalist:
f.write("%s\n" % item)
startTime = datetime.now()
run()
print(datetime.now() - startTime)
One simple way you could do this is by using the built-in time module. Get the time before the execution, get the time after the execution, subtract them and you get the time taken for your script to run. You could then just export it to your csv just like every other text.
import time
# STEP 1: Get start time
start_time = time.time()
# Step 2: Run your code you want to time
result = func_one(some_param)
# Step 3: Calculate total time elapsed
end_time = time.time() - start_time
I'm somewhat new to python and trying to write a program using Spotipy to first get Spotify song IDs from a playlist and then grab info on their sonic characteristics. I'm able to grab a list of dicts containing the song IDs using a while loop, but if I understand correctly the list stays in the while loop.
In that case the next step would be to either save to a variable outside the loop or just export that info to a csv or other file. I'm opting to export for now, though my exported csv files seem to contain just the header "song_id" and not the actual song IDs themselves. I'm sure this is a simple error but I can't seem to spot what it is. Does anyone have any ideas?
import pandas as pd
from spotipy.oauth2 import SpotifyClientCredentials
import spotipy
from pprint import pprint
import csv
sp = spotipy.Spotify(client_credentials_manager=SpotifyClientCredentials())
#get songs from the Pop Connoisseur Playlist
pl_id = 'spotify:playlist:5AvWMqi0lbzSPvDR6WWesu'
offset = 0
while True:
response = sp.playlist_tracks(pl_id, offset=offset, fields='items.track.id,total')
pc_tracks = [x['track']['id'] for x in response['items']]
print(pc_tracks)
offset = offset + len(response['items'])
print(offset, "/", response['total'])
s = 0
with open('popconn.csv', 'w', newline='') as csvfile:
fieldnames = ['song_id']
thewriter = csv.DictWriter(csvfile, fieldnames = fieldnames)
thewriter.writeheader()
for song in pc_tracks:
s += 1
thewriter.writerow({'song_id': song})
if len(response['items']) == 0:
break
Your code is writing to the same file each time it goes around the while True loop. This would overwrite anything you have already written. As your last loop probably has no items, this will leave you with a file containing just the header.
To avoid this, you only need to open your output file once before you start your loop. This would also just write the header once.
For example:
import pandas as pd
from spotipy.oauth2 import SpotifyClientCredentials
import spotipy
from pprint import pprint
import csv
sp = spotipy.Spotify(client_credentials_manager=SpotifyClientCredentials())
#get songs from the Pop Connoisseur Playlist
pl_id = 'spotify:playlist:5AvWMqi0lbzSPvDR6WWesu'
offset = 0
with open('popconn.csv', 'w', newline='') as csvfile:
fieldnames = ['song_id']
thewriter = csv.DictWriter(csvfile, fieldnames=fieldnames)
thewriter.writeheader()
while True:
response = sp.playlist_tracks(pl_id, offset=offset, fields='items.track.id,total')
pc_tracks = [x['track']['id'] for x in response['items']]
print(pc_tracks)
offset = offset + len(response['items'])
print(offset, "/", response['total'])
s = 0
for song in pc_tracks:
s += 1
thewriter.writerow({'song_id': song})
if len(response['items']) == 0:
break
I'm trying to write values into a .csv file every second until interrupted by the user by pressing a key. I'm able to achieve it in a finite loop.
I tried using raw_input but the program would only write the last value before interruption. What should I change in my code?
Here's the code that works for a finite loop:
import time
import csv
class Timestamp:
def __init__(self):
my_file = open('test_csv.csv','w+')
with my_file:
new_file = csv.writer(my_file)
for val in range(0,20):
with open('test_csv.csv','a') as f:
date_now = time.strftime('%d/%m/%y')
time_now = time.strftime('%H:%M:%S')
to_write = [date_now, time_now]
csv_file =csv.writer(f)
csv_file.writerow(to_write)
time.sleep(1)
Timestamp()
You can use threads.
import time
import csv
from threading import Thread
def write_loop(self):
my_file = open('test_csv.csv', 'w+')
with my_file:
new_file = csv.writer(my_file)
for val in range(0, 20):
with open('test_csv.csv', 'a') as f:
date_now = time.strftime('%d/%m/%y')
time_now = time.strftime('%H:%M:%S')
to_write = [date_now, time_now]
csv_file = csv.writer(f)
csv_file.writerow(to_write)
time.sleep(1)
t = Thread(target=write_loop)
t.daemon = True
t.start()
input("Press any key to stop")