I want to know if iowait time are counted in psutil.cpu_percent(), so write a code as below to test
#cat test.py
import os
import psutil
import time
p = psutil.Process(os.getpid())
start = time.time()
times_before_workload = p.cpu_times()
# this function return cpu_percent between two call. so we have to call it once before workload
percent_before_workload = p.cpu_percent()
# I call f open/close many times and read from the file
# hoping cpu will spent time on both user system and iowait
c = 1000
while c:
f = open('/tmp/big_text')
content = f.read(10240)
f.close()
c -= 1
end = time.time()
percent_after_workload = p.cpu_percent()
times_after_workload = p.cpu_times()
user_seconds = times_after_workload.user - times_before_workload.user
system_seconds = times_after_workload.system - times_before_workload.system
iowait_seconds = times_after_workload.iowait - times_before_workload.iowait
# if cpu percent == user_percent + system_percent + iowait_percent then it means yes. iowait are counted
print 'user_percent %s' % round(user_seconds / (end - start) * 100, 2)
print 'system_percent %s' % round(system_seconds / (end - start) * 100, 2)
print 'iowait_percent %s' % round(iowait_seconds / (end - start) * 100, 2)
print 'percent %s' % percent_after_workload
here is this output
#python test.py
user_percent 67.06
system_percent 67.06
iowait_percent 0.0
percent 134.8
The iowait is 0 so still can not verify. So question are
does iowait counted in cpu percent?
how to make iowait happen?(no zero)
No, iowait is not counted in cpu percent.
Run vmtouch to evict pages from memory at the start of each iteration of the loop.
You may need to install vmtouch:
apt update
apt install vmtouch
Run vmtouch in Python:
import subprocess
subprocess.run(['vmtouch', '-e', '/tmp/big_text'], stdout=subprocess.DEVNULL)
Reference: Why iowait of reading file is zero?
I want to create a GUI in python and make it interact with my micro-controller board(MSP430G2).
Would like to know if there is any way to send/receive data between the two.
pip install python-msp430-tools
if __name__ == '__main__':
from msp430.util import hexdump
import time
mmc = MMC()
try:
mmc.connect()
print mmc.info()
#show what's there
hexdump((0, mmc.read(0)))
#speed test
t1 = time.time()
for n in range(10):
mmc.read(n)
t2 = time.time()
dt = t2 - t1
bytes = n * 512
print "%d bytes in %.2f seconds -> %.2f bytes/second" % (bytes, dt, bytes/dt)
finally:
mmc.release()
python-msp430-tools
A have some problems in my audio network (I work with IP audio) - from time to time I have a short gaps in my audio stream. I have a logger which record all the streams. I've write small script with python and ffmpeg (and borrowed some JavaScript for visualizing :)) to find the gaps in logger mp3 files. It's better than nothing but I have a lot of false detections - it's quite annoying to to check all the results manually - script finds from 20 to 200 gaps per hour and usually only 1-10 gaps caused by some fault - all others are short term low audio level in songs, speech, etc. I'm looking for high level machine learning/data mining mechanism to check the gaps automatically to leave only what I want. I can provide a lot of "true" gaps (array with data) and "false" gaps to teach the machine and after that want just to feed it with data stamp with gap inside to compare does it look like a "true" gap or not. What can you recommend for the fastest solution? Note that Python is the only thing I can write a little. :/ At this moment the code of gap searching is following. It finds gaps with duration more than gap_min ms and less than gap_max ms in mp3 file or folder with files.
import numpy as np
import subprocess, os, sys
import ntpath
tolerance=150#100
gap_min=0.007#0.021
gap_max=0.035#0.03
sample_rate=48000
gap_file_duration=3#duration of the output mp3 files with gaps
ffmpeg_path=r'/Applications/ffmpeg'
temp_folder=r'/Users/user/Downloads/'
result_folder=r'/Users/user/Downloads/tmp/'
target_LUFS=-9#in LUFS
def samples_to_timecode(samples):
return '{0:02d}:{1:02d}:{2:02d}.{3:02d}'.format(int(samples / (3600*sample_rate)),
int(samples / (60*sample_rate) % 60),
int(samples / sample_rate % 60),
int(samples % sample_rate))
def timecode_to_samples(timecode):
return sum(f * int(t) for f,t in zip((3600*sample_rate, 60*sample_rate, sample_rate, 1), timecode.split(':')))
def seconds_to_timecode(seconds):
return '{0:02d}:{1:02d}:{2:03f}'.format(int(seconds / (3600)),
int(seconds / (60) % 60),
seconds % 60)#,
#int(seconds % 1000 % 60))
def analyze_bin_file(source_file):
print('Analizing start...')
data = np.memmap(source_file, dtype='h', mode='r')
zero_indexes=np.where(np.logical_and(data>=-tolerance, data<=tolerance))
gap_start=None
gaps_array=[]
for i in range(len(zero_indexes[0])-1):
if zero_indexes[0][i+1]-zero_indexes[0][i] == 1:
if not gap_start: gap_start=i
else:
if gap_start:
if ((zero_indexes[0][i]-zero_indexes[0][gap_start]) >= (gap_min*sample_rate)) and ((zero_indexes[0][i]-zero_indexes[0][gap_start]) <= (gap_max*sample_rate)):
gaps_array.append([float(zero_indexes[0][gap_start])/sample_rate,
float(zero_indexes[0][i])/sample_rate,
samples_to_timecode(zero_indexes[0][gap_start]),
round(float(zero_indexes[0][i]-zero_indexes[0][gap_start])/sample_rate,3)])
print('Gaps found: %s' % len(gaps_array))
gap_start=None
os.remove(source_file)#For some reasons it works badly in Windows. Just comment this line if cause problem. But you should delete temporary bin files manually after that.
print('Analizing done!')
return gaps_array
def execute_cmd(cmd):
p = subprocess.Popen(cmd , shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
return out.rstrip(), err.rstrip(), p.returncode
def prepare_bin_file(source_file):
print('Start preparing binary file...')
result_file_path=temp_folder+ntpath.basename(source_file)+'.bin'
result=execute_cmd('{0} -i {1} -ar {4} -af volume={3} -ac 1 -map 0:a -c:a pcm_s16le -y -f data {2}'.format(ffmpeg_path,
source_file,
result_file_path,
volume,
sample_rate))
if result[2] == 0:
print('Preparing done!')
return result_file_path
else:
print('Error occures while preparing!')
def cut_gaps(mp3_file,gaps_array):
print('Cutting file {0} start...'.format(mp3_file))
result_files=[]
path_list = mp3_file.split(os.sep)
for gap in range(len(gaps_array)):
gap_start=seconds_to_timecode(gaps_array[gap][0]-float(gap_file_duration)/2)
gap_duration=gap_file_duration+gaps_array[gap][3]
result=execute_cmd('{0} -y -i {1} -ss {2} -t {3} -c:a copy {4}'.format(ffmpeg_path,
mp3_file,
gap_start,
gap_duration,
result_folder+path_list[-2]+os.sep+'mp3'+os.sep+ntpath.basename(mp3_file)+'.{0:03d}'.format(gap)+'.mp3'))
#print('Save bin data file {0} of {1} {2}'.format(gap+1, len(gaps_array), 'OK' if (result_bin[-1] == 0) else 'ERROR'))
#print(result_bin)
result_files.append(ntpath.basename(mp3_file)+'.{0:03d}'.format(gap)+'.mp3')
print('Cutting file {0} of {1} {2}'.format(gap+1, len(gaps_array), 'OK' if (result[-1] == 0) else 'ERROR'))
print('Cutting done!')
return result_files
def make_report(source_file, gaps_array, cut_files):
path_list = source_file.split(os.sep)
report=open(result_folder+path_list[-2]+os.sep+ntpath.basename(source_file)+'.html','w')
report.write('<!doctype html><html lang=""><head></head><html><body><script src="https://cdnjs.cloudflare.com/ajax/libs/wavesurfer.js/1.1.2/wavesurfer.min.js"></script>')
report.write('<div>File {0} analizing report<br>'.format(source_file))
report.write('Searching parameters:<br>Gap minimum {0} second<br>Gap maximum {1} second<br>Tolerance value {2}<br>Analyze volume {3} dB<hr><hr></div>'.format(gap_min,
gap_max,
tolerance,
volume))
if len(gaps_array) > 0:
for gap_no in range(len(gaps_array)):
report.write('<div>Gap No {0}<br>Gap start {1}<br>Gap duration {2}ms</div>'.format(gap_no,
gaps_array[gap_no][2],
gaps_array[gap_no][3]*1000))
html="""
<div id='waveform""" + str(gap_no) + """'></div>
<div style='text-align: center'>
<button class='btn btn-primary' onclick='wavesurfer""" + str(gap_no) + """.playPause()'>
<i class='glyphicon glyphicon-play'></i>
Play
</button>
<p class='row'>
<div class='col-xs-1'>
<i class='glyphicon glyphicon-zoom-in'></i>
</div>
<div class='col-xs-10'>
<input id='slider""" + str(gap_no) + """' type='range' min='1' max='4000' value='1' style='width: 100%' />
</div>
<div class='col-xs-1'>
<i class='glyphicon glyphicon-zoom-out'></i>
</div>
</p>
</div>
"""
report.write(html)
script="""
<script>
var wavesurfer""" + str(gap_no) + """ = WaveSurfer.create({
container: '#waveform""" + str(gap_no) + """',
waveColor: 'red',
progressColor: 'purple'
});
wavesurfer""" + str(gap_no) + """.load('./mp3/""" + cut_files[gap_no] + """');
var slider""" + str(gap_no) + """ = document.querySelector('#slider""" + str(gap_no) + """');
slider""" + str(gap_no) + """.oninput = function () {
var zoomLevel = Number(slider""" + str(gap_no) + """.value);
wavesurfer""" + str(gap_no) + """.zoom(zoomLevel);
};
</script>
"""
report.write(script)
else:
report.write('<div>No gaps found!</div>')
report.write('</body></html>')
report.close()
def normalize_file(source):
print('Analizing integrated loudness...')
result = execute_cmd('{0} -nostats -i {1} -filter_complex ebur128 -f null -'.format(ffmpeg_path,
source))
if result[-1] == 0:
summary_index=str(result[1][-255:]).rfind('Summary:')
summary_list=str(result[1][-255:][summary_index:]).split()
I_LUFS = float(summary_list[summary_list.index('I:') + 1])
gainLog = -(I_LUFS - target_LUFS)
volume = 10 ** (gainLog / 20)
print('Analizing complete. I= {0} LUFS. Volume change value={1}.'.format(I_LUFS, volume))
else:
print('Error!')
return volume
def run(source):
if os.path.isfile(source) or os.path.isdir(source):
path_list = source.split(os.sep)
if not os.path.isdir(result_folder+path_list[-2]):
os.makedirs(result_folder+path_list[-2])
if not os.path.isdir(result_folder+path_list[-2]+os.sep+'mp3'):
os.makedirs(result_folder+path_list[-2]+os.sep+'mp3')
else:
print('Error! File of folder {0} not found!'.format(source))
if os.path.isfile(source):
global volume
volume=normalize_file(source)
bin_file=prepare_bin_file(source)
gaps_array=analyze_bin_file(bin_file)
if len(gaps_array):
cut_files=cut_gaps(source, gaps_array)
make_report(source, gaps_array, cut_files)
else:
make_report(source, gaps_array, cut_files=[])
elif os.path.isdir(source):
for file in os.listdir(source):
if file.endswith(".mp3"):
print(source ,file)
run(source+os.sep+file)
src=r'/Users/user/Downloads/2016-08-02'
if len(sys.argv) > 1:
run(sys.argv[1])
else:
run(src)
The result is HTML file with waveforms. The result works properly only in Firefox browser.
False gaps:
Example of the false gap 1
True gaps:
Example of the true gap 1
UPDATE. Because the algorithm is very sensitive to the volume level, I've added volume normalization before analyzing data. It doesn't apply to the output files - it's just normalize data before it being analyzed.
I'm working on my python script for xbmc media application so I can update the text label with the percent string. I have set the label text with "0%" for the start, but I have no idea how to update the label text with "1%", "10%", "20%" and so on.
When I try this:
progressStartTime = datetime.datetime.now()
delta = datetime.datetime.now() - progressStartTime
secondsLeft = int(delta.seconds) / float(percentageComplete) * (100.0 -
percentageComplete)
if secondsLeft > 30:
secondsLeft -= secondsLeft % 10
self.setControlLabel(self.main_loading_time_left, "" % secondsLeft)
I'm having trouble with update the text in the label where I'm getting an
error. The error I'm getting is: ZeroDivisionError: float division
The error are jumping on this line:
secondsLeft = int(delta.seconds) / float(percentageComplete) * (100.0
- percentageComplete)
Can you please help me how I can update the text in the label with the percent string?
Edit: Here is the update code:
percentageComplete = 0
if percentageComplete < 1:
self.getControl(4202).setLabel("1%")
progressStartTime = datetime.datetime.now()
delta = datetime.datetime.now() - progressStartTime
secondsLeft = int(delta.seconds) * (100.0 - percentageComplete)
if percentageComplete > 1:
secondsLeft -= secondsLeft % 10
self.getControl(4202).setLabel(secondsLeft + "%")
#self.setControlLabel(self.main_loading_time_left, "%" % secondsLeft)
You can't determine what time is left when you're at 0%. Exclude that case with a if case.
I'm using the library called psutil to get system/network stats, but I can only get the total uploaded/downloaded bytes on my script.
What would be the way to natively get the network speed using Python?
If you need to know the transfer rate immediately, you should create a thread that does the calculations continuously. I'm not an expert on the subject, but I tried writing a simple program that does what you need:
import threading
import time
from collections import deque
import psutil
def calc_ul_dl(rate, dt=3, interface="WiFi"):
t0 = time.time()
counter = psutil.net_io_counters(pernic=True)[interface]
tot = (counter.bytes_sent, counter.bytes_recv)
while True:
last_tot = tot
time.sleep(dt)
counter = psutil.net_io_counters(pernic=True)[interface]
t1 = time.time()
tot = (counter.bytes_sent, counter.bytes_recv)
ul, dl = [
(now - last) / (t1 - t0) / 1000.0
for now, last in zip(tot, last_tot)
]
rate.append((ul, dl))
t0 = time.time()
def print_rate(rate):
try:
print("UL: {0:.0f} kB/s / DL: {1:.0f} kB/s".format(*rate[-1]))
except IndexError:
"UL: - kB/s/ DL: - kB/s"
# Create the ul/dl thread and a deque of length 1 to hold the ul/dl- values
transfer_rate = deque(maxlen=1)
t = threading.Thread(target=calc_ul_dl, args=(transfer_rate,))
# The program will exit if there are only daemonic threads left.
t.daemon = True
t.start()
# The rest of your program, emulated by me using a while True loop
while True:
print_rate(transfer_rate)
time.sleep(5)
Here you should set the dt argument to whatever seams reasonable for you. I tried using 3 seconds, and this is my output while runnning an online speedtest:
UL: 2 kB/s / DL: 8 kB/s
UL: 3 kB/s / DL: 45 kB/s
UL: 24 kB/s / DL: 1306 kB/s
UL: 79 kB/s / DL: 4 kB/s
UL: 121 kB/s / DL: 3 kB/s
UL: 116 kB/s / DL: 4 kB/s
UL: 0 kB/s / DL: 0 kB/s
The values seems reasonable since my result from the speedtest were DL: 1258 kB/s and UL: 111 kB/s.
The answer provided by Steinar Lima is correct.
But it can be done without threading also:
import time
import psutil
import os
count = 0
qry = ""
ul = 0.00
dl = 0.00
t0 = time.time()
upload = psutil.net_io_counters(pernic=True)["Wireless Network Connection"][0]
download = psutil.net_io_counters(pernic=True)["Wireless Network Connection"][1]
up_down = (upload, download)
while True:
last_up_down = up_down
upload = psutil.net_io_counters(pernic=True)["Wireless Network Connection"][0]
download = psutil.net_io_counters(pernic=True)["Wireless Network Connection"][1]
t1 = time.time()
up_down = (upload, download)
try:
ul, dl = [
(now - last) / (t1 - t0) / 1024.0
for now, last in zip(up_down, last_up_down)
]
t0 = time.time()
except:
pass
if dl > 0.1 or ul >= 0.1:
time.sleep(0.75)
os.system("cls")
print("UL: {:0.2f} kB/s \n".format(ul) + "DL: {:0.2f} kB/s".format(dl))
v = input()
Simple and easy ;)
I added an LCD mod for this code if you want to test it on a raspberry pi but you need to add the psutil and the lcddriver to your project code!!!!
import time
import psutil
import os
import lcddriver
count=0
qry=''
ul=0.00
dl=0.00
t0 = time.time()
upload=psutil.net_io_counters(pernic=True)['wlan0'][0]
download=psutil.net_io_counters(pernic=True)['wlan0'][1]
up_down=(upload,download)
display = lcddriver.lcd()
while True:
last_up_down = up_down
upload=psutil.net_io_counters(pernic=True)['wlan0'][0]
download=psutil.net_io_counters(pernic=True)['wlan0'][1]
t1 = time.time()
up_down = (upload,download)
try:
ul, dl = [(now - last) / (t1 - t0) / 1024.0
for now,last in zip(up_down, last_up_down)]
t0 = time.time()
#display.lcd_display_string(str(datetime.datetime.now().time()), 1)
except:
pass
if dl>0.1 or ul>=0.1:
time.sleep(0.75)
os.system('cls')
print('UL: {:0.2f} kB/s \n'.format(ul)+'DL:{:0.2f} kB/s'.format(dl))
display.lcd_display_string(str('DL:{:0.2f} KB/s '.format(dl)), 1)
display.lcd_display_string(str('UL:{:0.2f} KB/s '.format(ul)), 2)
# if KeyboardInterrupt: # If there is a KeyboardInterrupt (when you press ctrl+c), exit the program and cleanup
# print("Cleaning up!")
# display.lcd_clear()
v=input()
The (effective) network speed is simply bytes transferred in a given time interval, divided by the length of the interval. Obviously there are different ways to aggregate / average the times and they give you different "measures" ... but it all basically boils down to division.
Another and more simple solution (without threading and queues although still based on #Steinar Lima) and for more recent python:
import time
import psutil
def on_calculate_speed(self, interface):
dt = 1 # I find that dt = 1 is good enough
t0 = time.time()
try:
counter = psutil.net_io_counters(pernic=True)[interface]
except KeyError:
return []
tot = (counter.bytes_sent, counter.bytes_recv)
while True:
last_tot = tot
time.sleep(dt)
try:
counter = psutil.net_io_counters(pernic=True)[interface]
except KeyError:
break
t1 = time.time()
tot = (counter.bytes_sent, counter.bytes_recv)
ul, dl = [
(now - last) / (t1 - t0) / 1000.0
for now, last
in zip(tot, last_tot)
]
return [int(ul), int(dl)]
t0 = time.time()
while SomeCondition:
# "wlp2s0" is usually the default wifi interface for linux, but you
# could use any other interface that you want/have.
interface = "wlp2s0"
result_speed = on_calculate_speed(interface)
if len(result_speed) < 1:
print("Upload: - kB/s/ Download: - kB/s")
else:
ul, dl = result_speed[0], result_speed[1]
print("Upload: {} kB/s / Download: {} kB/s".format(ul, dl))
Or you could also fetch the default interface with pyroute2:
while SomeCondition:
ip = IPDB()
interface = ip.interfaces[ip.routes['default']['oif']]["ifname"]
result_speed = on_calculate_speed(interface)
if len(result_speed) < 1:
print("Upload: - kB/s/ Download: - kB/s")
else:
ul, dl = result_speed[0], result_speed[1]
print("Upload: {} kB/s / Download: {} kB/s".format(ul, dl))
ip.release()
i found this tread, and dont have any idea from python, i jst copy and paste codes, and now need a little help, this script, i have jst show the total of bytes send/recived, can modify to show the actual speed?
def network(iface):
stat = psutil.net_io_counters(pernic=True)[iface]
return "%s: Tx%s, Rx%s" % \
(iface, bytes2human(stat.bytes_sent), bytes2human(stat.bytes_recv))
def stats(device):
# use custom font
font_path = str(Path(__file__).resolve().parent.joinpath('fonts', 'C&C Red Alert [INET].ttf'))
font_path2 = str(Path(__file__).resolve().parent.joinpath('fonts', 'Stockholm.ttf'))
font2 = ImageFont.truetype(font_path, 12)
font3 = ImageFont.truetype(font_path2, 11)
with canvas(device) as draw:
draw.text((0, 0), cpu_usage(), font=font2, fill="white")
if device.height >= 32:
draw.text((0, 14), mem_usage(), font=font2, fill="white")
if device.height >= 64:
draw.text((0, 26), "IP: " + getIP("eth0"), font=font2, fill=255)
try:
draw.text((0, 38), network('eth0'), font=font2, fill="white")
except KeyError:
# no wifi enabled/available
pass
The code
# pip install speedtest-cli
import speedtest
speed_test = speedtest.Speedtest()
def bytes_to_mb(bytes):
KB = 1024 # One Kilobyte is 1024 bytes
MB = KB * 1024 # One MB is 1024 KB
return int(bytes/MB)
download_speed = bytes_to_mb(speed_test.download())
print("Your Download speed is", download_speed, "MB")
upload_speed = bytes_to_mb(speed_test.upload())
print("Your Upload speed is", upload_speed, "MB")
The first answer in interface should be change to desired network adapter. To see the name in ubuntu you can use ifconfig, then change interface='wifi' to the device name.
a little change to formatting in python3
def print_rate(rate):
try:
print(('UL: {0:.0f} kB/s / DL: {1:.0f} kB/s').format(*rate[-1]))
except IndexError:
'UL: - kB/s/ DL: - kB/s'