We Keep Coding

cpu_usage() from python psutil library showing 100% of cpu usage, but HWMonitor only 10-11%?

Like above, why function cpu_usage shows 100% of cpu, while in HWMonitor or in Windows Task Manager I can see only 10-11% of usage. I got some code that measuring cpu and ram usage. Memory usage seems to be working like charm, but this cpu is somehow 10x greater than in task manager. Why?
import random
import threading
import psutil
def display_cpu():
global running
running = True
currentProcess = psutil.Process()
# start loop
while running:
print("CPU: ",currentProcess.cpu_percent(interval=1), "%", "| Memory: ", currentProcess.memory_info().rss/(1024*1024), "MB")
def start():
global t
# create thread and start it
t = threading.Thread(target=display_cpu)
t.start()
def stop():
global running
global t
# use `running` to stop loop in thread so thread will end
running = False
# wait for thread's end
t.join()
# ---
def insertion_sort():
nums = []
for i in range(30000):
nums.append(random.randint(1, 10000))
for i in range(1, len(nums)):
item_to_insert = nums[i]
j = i - 1
while j >= 0 and nums[j] > item_to_insert:
nums[j + 1] = nums[j]
j -= 1
nums[j + 1] = item_to_insert
# ---
for i in range(1):
start()
try:
result = insertion_sort()
finally:
stop()

Multiprocessing: callback on condition?

I'm using this code as a template (KILLING IT section)
https://stackoverflow.com/a/36962624/9274778
So I've solved this for now... changed the code to the following
import random
from time import sleep
def worker(i,ListOfData):
print "%d started" % i
#MyCalculations with ListOfData
x = ListOfData * Calcs
if x > 0.95:
return ListOfDataRow, True
else:
return ListOfDataRow, False
callback running only in main
def quit(arg):
if arg[1] == True:
p.terminate() # kill all pool workers
if __name__ == "__main__":
import multiprocessing as mp
Loops = len(ListOfData) / 25
Start = 0
End = 25
pool = mp.Pool()
for y in range(0,Loops)
results = [pool.apply(worker, args=(i,ListOfData[x]),callback = quit)
for y in range(0,len(ListofData))]
for c in results:
if c[1] == True
break
Start = Start + 25
End = End +25
So I chunk my data frame (assume for now my ListOfData is always divisible by 25) and send it off to the multiprocessing. I've found for my PC performance groups of 25 works best. If the 1st set doesn't return a TRUE, then I go to the next chunk.
I couldn't use the async method as it ran all at different times and sometimes I'd get a TRUE back that was further down the list (not what I wanted).

Multiprocessing: How to determine whether a job is waiting or submitted?

Background
A small server which waits for different types of jobs which are represented
as Python functions (async_func and async_func2 in the sample code below).
Each job gets submitted to a Pool with apply_async and takes a different amount of time, i.e. I cannot be sure that a job which was submitted first, also finishes first
I can check whether the job was finished with .get(timeout=0.1)
Question
How I can check whether the job is still waiting in the queue or is already running?
Is using a Queue the correct way or is there a more simple way?
Code
import multiprocessing
import random
import time
def async_func(x):
iterations = 0
x = (x + 0.1) % 1
while (x / 10.0) - random.random() < 0:
iterations += 1
time.sleep(0.01)
return iterations
def async_func2(x):
return(async_func(x + 0.5))
if __name__ == "__main__":
results = dict()
status = dict()
finished_processes = 0
worker_pool = multiprocessing.Pool(4)
jobs = 10
for i in range(jobs):
if i % 2 == 0:
results[i] = worker_pool.apply_async(async_func, (i,))
else:
results[i] = worker_pool.apply_async(async_func2, (i,))
status[i] = 'submitted'
while finished_processes < jobs:
for i in range(jobs):
if status[i] != 'finished':
try:
print('{0}: iterations needed = {1}'.format(i, results[i].get(timeout=0.1)))
status[i] = 'finished'
finished_processes += 1
except:
# how to distinguish between "running but no result yet" and "waiting to run"
status[i] = 'unknown'

Just send the status dict, to the function, since dicts are mutable all you need to do is change a bit your functions:
def async_func2(status, x):
status[x] = 'Started'
return(async_func(x + 0.5))
Of course you can change the status to pending just before calling your apply_async

Running program on multiple cores

I am running a program in Python using threading to parallelise the task. The task is simple string matching, I am matching a large number of short strings to a database of long strings. When I tried to parallelise it, I decided to split the list of short strings into a number of sublists equal to the number of cores and run each of them separately, on a different core. However, when I run the task on 5 or 10 cores, it is about twice slower than just on one core. What could the reason for that be and how can I possibly fix it?
Edit: my code can be seen below
import sys
import os
import csv
import re
import threading
from Queue import Queue
from time import sleep
from threading import Lock
q_in = Queue()
q_out = Queue()
lock = Lock()
def ceil(nu):
if int(nu) == nu:
return int(nu)
else:
return int(nu) + 1
def opencsv(csvv):
with open(csvv) as csvfile:
peptides = []
reader = csv.DictReader(csvfile)
k = 0
lon = ""
for row in reader:
pept = str(row["Peptide"])
pept = re.sub("\((\+\d+\.\d+)\)", "", pept)
peptides.append(pept)
return peptides
def openfasta(fast):
with open(fast, "r") as fastafile:
dic = {}
for line in fastafile:
l = line.strip()
if l[0] == ">":
cur = l
dic[l] = ""
else:
dic[cur] = dic[cur] + l
return dic
def match(text, pattern):
text = list(text.upper())
pattern = list(pattern.upper())
ans = []
cur = 0
mis = 0
i = 0
while True:
if i == len(text):
break
if text[i] != pattern[cur]:
mis += 1
if mis > 1:
mis = 0
cur = 0
continue
cur = cur + 1
i = i + 1
if cur == len(pattern):
ans.append(i - len(pattern))
cur = 0
mis = 0
continue
return ans
def job(pepts, outfile, genes):
c = 0
it = 0
towrite = []
for i in pepts:
# if it % 1000 == 0:
# with lock:
# print float(it) / float(len(pepts))
it = it + 1
found = 0
for j in genes:
m = match(genes[j], i)
if len(m) > 0:
found = 1
remb = m[0]
wh = j
c = c + len(m)
if c > 1:
found = 0
c = 0
break
if found == 1:
towrite.append("\t".join([i, str(remb), str(wh)]) + "\n")
return towrite
def worker(outfile, genes):
s = q_in.qsize()
while True:
item = q_in.get()
print "\r{0:.2f}%".format(1 - float(q_in.qsize()) / float(s))
if item is None:
break #kill thread
pepts = item
q_out.put(job(pepts, outfile, genes))
q_in.task_done()
def main(args):
num_worker_threads = int(args[4])
pept = opencsv(args[1])
l = len(pept)
howman = num_worker_threads
ll = ceil(float(l) / float(howman * 100))
remain = pept
pepties = []
while len(remain) > 0:
pepties.append(remain[0:ll])
remain = remain[ll:]
for i in pepties:
print len(i)
print l
print "Csv file loaded..."
genes = openfasta(args[2])
out = args[3]
print "Fasta file loaded..."
threads = []
with open(out, "w") as outfile:
for pepts in pepties:
q_in.put(pepts)
for i in range(num_worker_threads):
t = threading.Thread(target=worker, args=(outfile, genes, ))
# t.daemon = True
t.start()
threads.append(t)
q_in.join() # run workers
# stop workers
for _ in range(num_worker_threads):
q_in.put(None)
for t in threads:
t.join()
# print(t)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
The important part of the code is within the job function, where short sequences in pepts get matched to long sequences in genes.

This should be because of GIL (Global Interpreter Lock) in CPython.
In CPython, the global interpreter lock, or GIL, is a mutex that prevents multiple native threads from executing Python bytecodes at once.
David Beazley's presentation at PyCon 2010 gave a detailed explanation about GIL. And from page 32 to page 34, he explained why the same multiple-threading code (of CPU-bound computation) could have worse performance when running with multiple cores than when running with single core.
(with single core) Threads alternate execution, but switch far
less frequently than you might imagine
With multiple cores, runnable threads get scheduled simultaneously (on different cores) and battle over the GIL
David's this experiment result visualizes "how thread switching gets more rapid as the number of CPUs increases".
Even though your job function contains some I/O, according to its 3-level nested loops (two in job and one in match), it is more like CPU-bound computation.
Changing your code to multiple-processing will help you utilize multiple cores and may improve the performance. However, how much you could gain depends on the quantity of the computation - whether the benefit from parallelizing the computation could far surpass the overhead introduced by multiple-processing such as inter-process communication.

Python Apply_async not waiting for other Processes to Finish

I have the following sample code that I am trying to use the multiprocessing module on. The following statement had been working previously under other applications, but one process (which receives a very small amount of data just due to the breakup) finishes first and causes the program to finish. Could someone help me understand why this is not waiting for the others?
def mpProcessor(basePath, jsonData, num_procs = mp.cpu_count()):
manager = mp.Manager()
map = manager.dict()
procs = mp.Pool(processes = num_procs, maxtasksperchild = 1)
chunkSize = len(jsonData) / (num_procs)
dataChunk = [(i, i + chunkSize) for i in range(0, len(jsonData), chunkSize)]
count = 1
for i in dataChunk:
print 'test'
s, e = i
procs.apply_async(processJSON, args = (count, basePath, jsonData[s:e]))
count += 1
procs.close()
procs.join()
return map
def processJSON(proc, basePath, records):
print 'Spawning new process: %d' %os.getpid()
outDict = dict()
print len(records)
for i in range(len(records)):
valid = False
idx = 0
while valid == False:
jsonObject = json.loads(records[i][1])['results'][idx]
if jsonObject['kind'] == 'song':
valid = True
break
else:
idx += 1
tunesTrack = Track()
tunesTrack.setTrackId(jsonObject['trackId'])
print 'Finished processing %d records with process %d' %(len(records), os.getpid())

You seem to be reinventing the wheel.
What you are trying to do could be much more easily achieved by using an initializer with the pool and using map rather than apply_async. As it stands your code snippet is not runnable so I can't be sure what the actual problem is. However, the following should simplify your code and make it easier to debug.
import math
import multiprocessing as mp
def pool_init(basePath_):
global basePath, job_count
basePath = basePath_
job_count = 0
print 'Spawning new process: %d' %os.getpid()
def mpProcessor(basePath, jsonData, num_procs=mp.cpu_count()):
pool = mp.Pool(processes=num_procs, initializer=pool_init, initargs=(basePath,))
# could specify a chunksize, but multiprocessing works out the optimal chunksize
return pool.map(processJSON, jsonData)
# change processJSON to work with single records and
# remove proc and basePath args (as not needed)
def processJSON(record):
global job_count
print 'Starting job %d in process: %d' % (job_count, os.getpid())
valid = False
idx = 0
while valid == False:
jsonObject = json.loads(record[1])['results'][idx]
if jsonObject['kind'] == 'song':
valid = True
break
else:
idx += 1
tunesTrack = Track()
tunesTrack.setTrackId(jsonObject['trackId'])
print 'Finished processing job %d with process %d' % (job_count, os.getpid())
job_count += 1