This code:
Looks for an image a
If it finds a it tries to find a match for any image in the array
image_list
If it finds an image in image_list it looks for e
If it finds e it logs it and moves the mouse to x, y checks for pixel colour and then clicks when when it finds a match, clicks.
This is where my problem arises my avoidLog() function is being called 2-3 times per click. I've added print("click") to check if it's actually clicking and it isn't, it's clicking at the right time but for some reason, my log is triggering more than once.
It's logging correctly, just an odd number of times.
I know python is operating in a single thread in my example so I'm not sure why it's looping back round and not clicking. It would make more sense to me if it was clicking multiple times and logging multiple times.
import pyautogui as py
def avoidLog():
avoidLog = open('Avoided.txt', 'a')
avoidLog.write("Found at: " + str(f) + " at: " + str(skipTime))
avoidLog.write("\n")
avoidLog.close()
image_list = []
while True:
if py.locateOnScreen('a.jpg') != None:
for image in image_list:
found = py.locateCenterOnScreen(image)
if found != None:
skipTrigger = py.locateOnScreen('e.jpg')
if skipTrigger != None:
avoidLog()
py.moveTo(x, y)
r = py.pixelMatchesColor(x,y, (r,g,b))
if r == True:
py.sleep(2)
print("click")
py.click()
break
avoidLog() is called whenever e.jpg is located. However, py.click() is only called if pixelMatchesColor is True
Related
Total beginner here.
In this program, I am trying to "monitor" a specific area on my screen. There are several images entering the region (from the right side) with random frequency. (Basically, a "simple" rhythm" game).
I have tried pyautoguy.locateOnScreen, but it was way too slow even though I set the region into a small square. I don't think I can use pixels because the images entering the region are all the same color but have different symbols on them, and each relates to a keyboard letter.
In this last attempt, I tried this:
Create an "endless" while loop, in which I make a screenshot of my desired region/location(bbox), and within that endless loop, I use IF and several ELIF in which I use pyautogui.locate to search a specific image (tiny screenshot of the symbol) on the bigger screenshot made by ImageGrab. I have several images, and each image has a different keyboard response.
I got to about 75-80% accuracy, but when the symbols are close to each other, it always misses. When I checked at which rate the whole loop runs, I get to like 2.9 runs per second, sometimes 4-5. Could anyone give me some pointers on how to double the speed (if that's even possible?)
Thanks!
import time
import pyautogui
import os
import pydirectinput
from PIL import ImageGrab, Image
import time
start_time = time.time()
x = 1 # frames
counter = 0
bbox=(588,786,635,944) #my region I am searching in
def detect(imgDir, haystackImage, confidence=0.90, cache=True):
if (pyautogui.locate(os.path.join('images', imgDir), haystackImage, confidence=confidence)) is not None:
return True
else:
return False
while True:
ScreenGrab = ImageGrab.grab(bbox)
ScreenName = f"screenshot"
Path = f"./images/{ScreenName}.png"
ScreenGrab.save(Path)
hsImage = Image.open('D:/PY/images/screenshot.png')
if detect('D:/PY/images/a.png', hsImage, cache=True):
pydirectinput.press('a')
continue
elif detect('D:/PY/images/b.png', hsImage, cache=True):
pydirectinput.press('b')
continue
elif detect('D:/PY/images/c.png', hsImage, cache=True):
pydirectinput.press('c')
continue
elif detect('D:/PY/images/d.png', hsImage, cache=True):
pydirectinput.press('d')
continue
elif detect('D:/PY/images/e.png', hsImage, cache=True):
pydirectinput.press('e')
continue
else:
pass
counter+=1 #counter for my runs per second
if (time.time() - start_time) > x :
print("FPS: ", counter / (time.time() - start_time))
counter = 0
start_time = time.time()
I have created the following function that allows the user to change the shape of the Python turtle to an image he/she selects from a file dialog file dialog that pops up when a specific button is pressed:
def TurtleShape(iop = None):
# "iop" is supposed to be an image path
try:
manipulateimage.config(state = NORMAL)
flipButton.config(state = NORMAL)
mirrorButton.config(state = NORMAL)
originalButton.config(state = NORMAL)
resetturtle.config(state = NORMAL)
rotateButton.config(state = NORMAL)
global klob
# The following "if-else" statement uses the "iop" argument's value as the value for "klob" if `iop` is NOT `None`
if iop != None:
klob = iop
print("lmcv")
else:
klob = filedialog.askopenfilename()
print("klobby")
global im
im = Image.open(klob)
pictures.append(im)
edited.clear()
print(im)
im.save(klob + '.gif', "GIF")
register_shape(klob + '.gif')
shape(klob + '.gif')
update()
except:
pass
The above function is also supposed to use the iop argument's value as the turtle's image if it is not None.
Now, consider this situation; you draw a bunch of things, set the turtle to an image, and just when you are about to stamp the image, you accidentally press the button that resets the turtle to its normal shape (yes, that button exists in my program). Oh no! How would you get it back without going through all the steps to open and edit it again? Well, that is where my undoHandler function (shown below) comes in. It just essentially undoes the last function called using many stacks, which I created as deques. It is pretty straightforward if you are proficient in Python:
def undoHandler():
if len(function) > 0 and draw.drawing == True:
undoHandler.handling = True
if not hasattr(undoHandler, "counter"):
undoHandler.counter = 0
undoHandler.counter += 1
# clear the canvas
Clear()
# Pop a point object from function deque
function.pop()
penup()
goto(-200, 100)
pendown()
try:
# Execute everything up to point before last function called
for i in function:
# Set canvas and turtle to previous state
tsd = i.recieveshape()
shape(tsd)
mndf = i.recieveheading()
setheading(mndf)
hk = i.getletterheight()
global letter_height
letter_height = hk
rk = i.getletterwidth()
global letter_width
letter_width = rk
milk = i.getspacewidth()
global space_width
space_width = milk
hw = i.getwidth()
width(hw)
op = i.getcolor()
try:
color(op)
except:
for g in colors:
cp = g.getcolor2()
colormode(255)
color(cp)
# Get function wrapped in Point object and execute it
j = i.getfunction()
j()
# Following is the code block where the issue occurs. Basically, if the function being run is equal to `TurtleShape`, then do the following...
if j.__name__ == "TurtleShape":
# `hfl` is a deque that holds all of the `pictures` deque's contents as it is cleared when the turtle is set to its default state
pictures.extend(hfl)
lmcv = pictures.pop()
pictures.append(lmcv)
try:
# Resize image to previous size if user changes it. Otherwise, skip this.
bun = picwidth.pop()
picwidth.append(bun)
mun = picheight.pop()
picheight.append(mun)
clob = lmcv.resize((int(bun), int(mun)), Image.ANTIALIAS)
except:
clob = lmcv
clob.save(klob + str(undoHandler.counter) + ".gif")
# Use the `clob.save` output from above as source image in `TurtleShape` function (this is where issue occurs)
TurtleShape(klob + str(undoHandler.counter) + ".gif")
print("Undone!")
else:
pass
except:
pass
Basically what happens here is that it takes the function (wrapped in a Point object) from a queue through which the main functions go through as they are called. The functions then get appended to the function deque, after which, when undoHandler is called by the user, the screen gets cleared, and latest value is popped from the function deque so that all the other actions except the last one will be executed again. This issue I am facing occurs specifically in the if j.__name__ == "TurtleShape": code block. Basically, for some reason, when the user chooses to undo the resetting of the turtle to its original shape, it works as it should until the TurtleShape function is executed by the undoHandler. For some reason, when the undoHandler executes the TurtleShape function, even when I give a valid argument for the iop attribute of the TurtleShape function (as you can see in the if j.__name__ == "TurtleShape": code block), the else statement is executed first (i.e. the file dialog comes up instead of continuing from the if statement). Only if the user clicks cancel in that dialog will the turtle get set to the previous image.
What is wrong in my code that is leading to this occurrence, and how would I stop this from happening? I have tried changing the klob attribute in the function where the output is saved in the undoHandler function to, for example, "SaveImage", but still no luck. I have also tried to add an if-elif statement in the TurtleShape when it is supposed to choose between iop or a file dialog as the value for klob, but still the issue occurs. Apparently, it executes the elif statement even when it is not necessarily true. Therefore, any help is very much appreciated in remedying this issue! :)
It's happening here:
j = i.getfunction()
j()
If the function you've just gotten is the TurtleShape() function, then you're calling it once with its default arguments (i.e., iop = None). Then you go into your big if j.__name__ == "TurtleShape": statement and call it again inside the if block.
Move that j() call into the else: block of your big if j.__name__ == "TurtleShape": statement, and your problem should go away.
Does that brief explanation make enough sense for you to understand why the problem is happening? Or do you need me to explain a bit more in-depth how calling j() is calling TurtleShape with the parameter iop = None?
I've got a program right now that is meant to display an image, and allow for various functions on keypress (e.g., load a new frame, crop a region of interest, etc). It's also set up so that if too much time passes with no action, it'll automatically attempt to load a new image.
def triggeredbytimer():
global f, x1i, y1i, x2i, y2i, sid, keepgoing
keepgoing = False
print "Inactivity has lead to an automatic refresh. Loading newest image."
f = 0; t = 0
incoming = imagerequest(sid, f, t, x1i, y1i, x2i, y2i)
imagecallback(incoming)
def imagecallback(data):
print "----------------------- imagecallback"
global sid, f, x1i, y1i, x2i, y2i, img, refPt, cropping, keepgoing
keepgoing = True
########### \\// change this to change wait for autorefresh
tmr = Timer(10.0, triggeredbytimer)
tmr.start()
##################################
b = data.b; t = data.tr; sid = data.sid; error = int(data.error)
t = float(t); sid = int(sid)
expandedt = time.strftime("%m-%d-%Y %H:%M:%S", time.localtime(t))
print str(t) + " ----> " + expandedt
print "----------------------- image incoming"
timestr = time.strftime("%H:%M:%S", time.gmtime(t))
print "Loaded image from %s."%timestr
imagedata = bridge.imgmsg_to_cv2(b, "bgr8")
npimg = cv2.imencode('.jpg', imagedata)
cv2.imwrite('temp_image_np.jpg', imagedata)
img = cv2.imread('temp_image_np.jpg')
clone = img.copy()
cv2.namedWindow('image', cv2.WINDOW_NORMAL)
cv2.setMouseCallback("image", mouseclickcb)
cv2.imshow("image", img)
while keepgoing == True:
key = cv2.waitKey(0) & 0xFF
## (here there are a lot of keypress functions, for brevity I included only one)
elif key == ord("t"):
print "Looking for newest frame..."
tmr.cancel()
t = 0; f = 0
incoming = imagerequest(sid, f, t, x1i, y1i, x2i, y2i)
imagecallback(incoming)
(NOTE: The images are fetched from a ROS program via services, and that is where the "imagerequest" function comes from. All of that works fine: the problem is specifically as prescribed.)
To summarize here- since there's a bit of irrelevant clippings in this snippet - an image is loaded by imagecallback. When that function is called, a timer starts (time limit is arbitrary, set low here for testing). When the timer expires, triggeredbytime() requests a new image. It acquires the image via a ROS service, then takes the return from that and uses it as an input to imagecallback(data), thus loading the image and allowing all the keypress functions to still be valid.
So the problem is that whenever the program refreshes with a new frame, the while keepgoing == True loop does not stop, leading to stacked loops that go on and get worse with every refresh. this leads to a single keypress executing multiple times, which quickly overwhelms the program it's pulling images from. (Originally, this was just a while True: loop, but I added the "keepgoing" variable to try to remedy the situation - unsuccessfully, as it were.) I'm not sure how to resolve this issue, and attempting to turn the loop "off" long enough to kill the first loop but allowing the next one to execute has not worked out so far. Is there any way to exit this while keepgoing == True loop cleanly to prevent things from stacking as such, or is there a better way in general to address the problem of loading a new image after a certain amount of time?
Thanks!
EDIT: to be clear, the problem is with ending the 'while' loop when the timer hits 0. setting keepgoing = False doesn't seem to work (probably because I can't 'return' it), so I need something similar to that if it's possible...
On my current Python script I have it counting down from 3 to 1, then taking a picture as you can intepret from this code below, doing this 4 times. However, it seems to sneak in 1 picture before counting down, and leaving the last one out, so the last countdown doesn't seem to matter.
Can anyone tell me why this is?
def start_photobooth():
import config
now = time.strftime("%d-%m-%Y-%H:%M:%S")
try:
for i, filename in enumerate(camera.capture_continuous(config.images_path + now + '-' + '{counter:02d}.jpg')):
print "[" + current_time() + "] [PHOTO] Producing " + filename
surface.fill((0,0,0,0))
surface.set_alpha(288)
textSurfaceObj = fontObj.render('', True, red)
surface.blit(textSurfaceObj, textRectObj)
pygame.display.update()
time.sleep(float(config.countdown))
for y in range(3,0,-1):
surface.fill((0,0,0,0))
surface.set_alpha(288)
textSurfaceObj = fontObj.render(str(y), True, red)
surface.blit(textSurfaceObj, textRectObj)
pygame.display.update()
pygame.time.wait(1000)
if i == total_pics-1:
break
Your code will take a picture at the very beginning of your loop as the capture_continuous method is executed at that point.
Your code will then run its countdown and restart the loop at which point it takes another photo.
What your loop is really doing is just:
Take picture
Countdown
Repeat
You want it to be:
Countdown
Take picture
Repeat
You could therefore change the start of your loop to:
for i in range(total_pics):
remove the if section at the end of your code (as this is handled by the for loop now) and insert a line to take the photo after the countdown. Assuming this is rasperry pi camera then the line would be:
filename = camera.capture("{0}{1}-{2:02d}.jpg".format(config.images_path,now,i))
I'm not familiar with the picamera module, so it may be that you do it this way:
filename = "{0}{1}-{2:02d}.jpg".format(config.images_path,now,i)
camera.capture(filename)
This is a short script using pyscreenshot (https://github.com/ponty/pyscreenshot) to get the screen image then looks for a smaller loaded image (local file 'refresh.png', about 32x32 px) inside of it.
I need to duplicate tabs in Chrome and middle clicking refresh does this wonderfully without any reloads.
I get error C:\slave\WinInstallerMegaPack\src\opencv\modules\core\src\matrix.cpp:113: error: (-215) s >= 0 when it gets to second part of the watch. Getting the whole screen the checking works, but getting just part of the screen then checking is a problem. I have no idea what that error means, even after looking at some source. Checking the whole screen for 'refresh.png' takes a bit over a second. Afterwards it starts failing on the partial screen screengrab check and I can't figure out why or understand this error. Checking just part of the screen would go much faster.
From the only results on Google I'd have to guess that the array has too many variables, but I don't get how. I added some print statements that show everything before and after the problem is 1 variable of len 4.
import Image, numpy
import cv2
import numpy as np
import pyscreenshot as ImageGrab
import os
from pymouse import PyMouse
from time import clock, sleep
def grabScreen(Bbox = None): #unlike rect in pygame, bbox does top left (x,y), bottom right (x,y). So the second x,y isn't width, height, it's width + x, height + y
tempmousexy = m.position() #get current mousexy
m.move(0,0) #move pointer to the corner for screen shot (change this if you put the refresh button in the very top left, weirdo
if Bbox:
screenshot = ImageGrab.grab(bbox=Bbox) #grab part of screen
else:
screenshot = ImageGrab.grab() #grab whole screen
m.move(tempmousexy[0],tempmousexy[1]) #put mouse pointer back after screenshot
return np.array(screenshot)
def saveScreen(savename = 'img.png'): #not used right now, but a cool line
ImageGrab.grab_to_file(savename)
def findMatch(img, template, closenesslimit = .001, method = cv2.TM_CCOEFF_NORMED):
#convert to gray for faster processing - probably still accurate enough
template = cv2.cvtColor(template, cv2.CV_32FC1)
img = cv2.cvtColor(img, cv2.CV_32FC1)
#check for match
m = cv2.matchTemplate(template, img, cv2.cv.CV_TM_SQDIFF_NORMED)
#we want the minimum squared difference - lower means less difference
mn,_,mnLoc,_ = cv2.minMaxLoc(m)
print 'RAW (best match, worst match, bestmXY, worstmXY):', str(cv2.minMaxLoc(m))
#check if match is closer (less than) closenesslimit
print 'Closest match (lower is better):', str(mn)
if mn > closenesslimit: return False
#else
print '~Match found~ (closeness was better - less - than limit)'
#find rectangle
matchx, matchy = mnLoc
matchw, matchh = template.shape[:2]
print (matchx,matchy,matchx+matchw,matchy+matchh) #checking what is about to be returned
return (matchx,matchy,matchx+matchw,matchy+matchh) #x1, y1, x2, y2
def checkScreen():
print 'Checking whole screen'
return findMatch(grabScreen(),templateimg)
def checkArea(checkrect):
print 'Checking area'
return findMatch(grabScreen(checkrect),templateimg)
global m
m = PyMouse() #load PyMouse for mouse moves and position checks
guessxy = 0 #once a good x,y is detected, detection will start out there
startdelay = 3 #delay while waiting for first event
runningdelay = .2 #delay between repeating check events after first event
if os.name == 'nt':
templateimg = cv2.imread('bensmall.png')
else:
templateimg = cv2.imread('alexsmall.png') #changes based on resolution, would be better if this scaled automatically based on screen res
print 'Global mouse set, small template (refresh button image) loaded'
while True:
while not guessxy: #check the whole screen until a location is found
sleep(startdelay)
guessxy = checkScreen()
if not guessxy: print 'Nothing found yet, sleeping', str(startdelay)
print 'GUESSXY:', str(guessxy)
fails = 0
faillimit = 5
while guessxy: #once location is found, guess near this area again
sleep(runningdelay) #sleep at the beginning avoids awkward pauses later when something is found but it has to sleep again before printing about it
runstart = clock()
temppos = m.position()
print 'GUESSXY (limiting screenshot to this area):', str(guessxy)
found = checkArea(guessxy) #increase guessxy area by some small amount on each side later
print found
if found:
guessxy = found
print guessxy
m.move(guessxy[0],guessxy[1]) #remove this line later
print 'found'
else:
fails+=1
print 'Not found,', str(fails)
if fails>= faillimit: #wasn't near the guess point, go back to checking whole screen
print 'Too much fail, resetting...'
guessxy = 0
print clock()-runstart