Hello my goal is to be able to make my bot click the thing of my choosing on any screen size because I think that is the main issue. Ive tried to decrease the confidence level but it just ends up clicking something else with the same general color. ive tested it with an EXACT image and it clicks the correct spot so its not like the coordinates are off or anything its just the image recognition.
These are the images to go by
(X1, NextLesson, Arrow)
from pyautogui import *
import pyautogui
import time
import keyboard
import random
def NextLesson():
keepGoing = True
while keepGoing == True:
counter = 0
nl_coordinates = pyautogui.locateOnScreen('images/nextLesson.png', confidence=0.4)
print(nl_coordinates) # This will print out where it is
if nl_coordinates:
print(f"I can see it at {nl_coordinates}")
pyautogui.click(nl_coordinates)
keepGoing = False
else:
print("I cannot see it.")
def Arrow():
keepGoing = True
while keepGoing == True:
counter = 0
arrow_coordinates = pyautogui.locateOnScreen('images/arrow.png', confidence=0.4)
print(arrow_coordinates) # This will print out where it is
if arrow_coordinates:
print(f"I can see it at {arrow_coordinates}")
pyautogui.click(arrow_coordinates)
keepGoing = False
else:
print("I cannot see it.")
def X1():
keepGoing = True
while keepGoing == True:
counter = 0
x1_coordinates = pyautogui.locateOnScreen('images/x1.png', confidence=0.4)
print(x1_coordinates) # This will print out where it is
if x1_coordinates:
print(f"I can see it at {x1_coordinates}")
pyautogui.click(x1_coordinates)
keepGoing = False
else:
print("I cannot see it.")
while True:
counter = 0
counter2 = 0
true = True
time.sleep(2)
X1()#
time.sleep(8)
NextLesson()#
time.sleep(10)
Arrow()#
print("calibration complete ")
time.sleep(5)
cords = pyautogui.position()
while counter != 1800:
time.sleep(60)
pyautogui.click(cords) #clicking where ouse is at
print("clicked")
counter += 60
print(counter)
if counter == 1800:
time.sleep(5) #stops code for 5 secs
X1() #clicks mouse to x button
print("clicked x")
time.sleep(5) #stops code for 5 secs
NextLesson() #clicks mouse to the assignment button
print("clicked assignemnt")
time.sleep(15) #stops code for 2 secs
Arrow() #clicks mouse to the second assignment button
print("clicked 2nd assignment button ")
time.sleep(5) #waits 5secs to put cursor at position
cords = pyautogui.position() #grabs position
print("grabbed position")
We can use opencv-python to perform multiscale template matching. The idea is to scale the template image and attempt to locate the resized template in the screenshot. The code below (adapted from here) loops over 50 scaling parameters in the range [0.25,2] and selects the one the that provides the best match. You might want to decrease the range or number of scaling parameters for efficiency. It is also better to save the correct scaling parameter and reuse it for multiple images.
import cv2
import pyscreeze
import numpy as np
import imutils
import pyautogui
def template_match_with_scaling(image,gs=True,confidence=0.8):
"""
Locate an image and return a pyscreeze box surrounding it.
Template matching is done by default in grayscale (gs=True)
Detect image if normalized correlation coefficient is > confidence (0.8 is default)
"""
templateim = pyscreeze._load_cv2(image,grayscale=gs) # template image
(tH, tW) = templateim.shape[:2]
screenim_color = pyautogui.screenshot() # screenshot of image
screenim_color = cv2.cvtColor(np.array(screenim_color),cv2.COLOR_RGB2BGR)
if gs is True:
screenim = cv2.cvtColor(np.array(screenim_color),cv2.COLOR_BGR2GRAY)
else:
screenim = screenim_color
#try different scaling parameters and see which one matches best
found = None #bookeeping variable for the maximum correlation coefficient, position and scale
scalingrange = np.linspace(0.25,2,num=50)
for scale in scalingrange:
resizedtemplate = imutils.resize(templateim, width = int(templateim.shape[1]*scale) ) # resizing with imutils maintains the aspect ratio
r = float(resizedtemplate.shape[1])/templateim.shape[1] # recompute scaling factor
result = cv2.matchTemplate(screenim, resizedtemplate, cv2.TM_CCOEFF_NORMED) # template matching using the correlation coefficient
(_, maxVal, _, maxLoc) = cv2.minMaxLoc(result) #returns a 4-tuple which includes the minimum correlation value, the maximum correlation value, the (x, y)-coordinate of the minimum value, and the (x, y)-coordinate of the maximum value
if found is None or maxVal > found[0]:
found = (maxVal, maxLoc, r)
(maxVal, maxLoc, r) = found
if maxVal > confidence:
box = pyscreeze.Box(int(maxLoc[0]), int(maxLoc[1]), int(tW*r), int(tH*r) )
return box
else:
return None
def locate_center_with_scaling(image,gs=True):
loc = template_match_with_scaling(image,gs=gs)
if loc:
return pyautogui.center(loc)
else:
raise Exception("Image not found")
#sample usage
coords = locate_center_with_scaling('images/arrow.png')
The answer I found is taking a more accurate photo of what I need like trying to remove the background as much greyscale=ture and adding more confidence
Related
In the code below as you can see i'm trying to find an image in the screen with a .png(that i didn't add because i don't think it's the problem) but it doesn't find anything
import pyautogui as pt
from time import sleep
class Clicker:
def __init__(self, target_png, speed):
self.target_png = target_png
self.speed = speed
pt.FAILSAFE = True
def nav_to_image(self):
try:
position = pt.locateOnScreen(self.target_png, confidence=.8)
pt.moveTo(position[0] + 15, position[1] + 15, duration=self.speed)
pt.doubleClick()
except:
print('No img found')
return 0
if __name__=='__main__':
sleep(1)
clicker = Clicker('accetta.png', speed=.001)
end = 0
while True:
sleep(1)
if clicker.nav_to_image() == 00:
end += 1
if end > 5:
break
I see that you confidence is set to 80%. It's a little tricky when you're not looking for the exact match. Have you tested the code with an image that doesn't require confidence to work?
Sometimes there is a difference in the hue. Try locateOnScreen(image, grayscale=False) and see if that makes a difference.
It would help if you included both the image you're trying to find and a copy of the screenshot in order to reproduce the error.
I want to write a bot that will simulate mouse movements when user is away for more than 5 minutes and stay put when user takes control i.e. moves the mouse.
I have written following code as a part of the program.
Here is the link to my old program which clicks at given points periodically. The problem with this program is I have to start the program when I want to go somewhere and after returning close the program in order to resume working.
Here is the one of the module I wrote for the new program which detects whether mouse is moving or not.
import win32api
from time import sleep
print("starting engine.")
count = 0
while(True):
savedpos = win32api.GetCursorPos()
if count>20*5:
break
sleep(1)
curpos = win32api.GetCursorPos()
if savedpos == curpos:
savedpos = curpos
print("Mouse is steady.")
else:
print("Mouse is moving.")
count += 1
I wont be writing the code but I have the Idea to solve the problem
you use pyautogui.position() to keep checking the position and if it doesnt change position for 300 seconds you move it
I wrote following code referring other Stackoverflow posts.
This code waits 4 minutes for mouse being steady. Whenever mouse is moved, timer is reset to zero.
import win32api
from time import sleep
import pyautogui as gui
print("starting engine.")
count = 0
savedpos = win32api.GetCursorPos()
def actions():
print(gui.size())
while True:
#toast.show_toast("ROBOT V2 !", "Robot is active.", threaded=False, icon_path=None, duration=2)
gui.click()
gui.press('home')
gui.moveTo(541, 142, 0.25) # Money Transfer dropdown
gui.click()
sleep(0.5)
gui.moveTo(541, 172, 0.25) # Money Transfer link
gui.click()
sleep(5)
cond()
def cond():
count = 0
while True:
savedpos = win32api.GetCursorPos()
sleep(0.5)
curpos = win32api.GetCursorPos()
if savedpos == curpos:
savedpos = curpos
print("Mouse is steady. Elapsed time: ", (count+1)/2, " seconds.")
count += 1
if count >= 480: # if count is greater than 60 it means timeout will occur after mouse is steady for more than
# 240 seconds i.e. 4 minutes. (approx.)
print("User away for more than 4 minutes, taking control of the system.")
actions()
break
else:
pass
else:
print("Mouse is moving.")
count = 0
cond()
Im trying to loop a function in itself in a special condition but as it re runs the function refreshes the variable "j" also. So i want to use that functions repeadetly to check connection and refresh the page so if i put variables to the out of the functions it will run once and makes it top value so the looping second time will fail. How to fix that?
def checkConnection():
onPage = pyautogui.locateCenterOnScreen('onpage.png', grayscale = True, confidence = 0.8)
noConnectionError = pyautogui.locateCenterOnScreen('noconnection.png', grayscale = True, confidence = 0.8)
sleep(2)
if onInstagram != None:
print("[INFO] : NETWORK STATUS : SUCCESS")
elif noConnectionError != None:
print("[WARNING] : NO CONNECTION !!!")
print("[INFO] TYRING TO RECONNECT...")
sleep(1)
j = 0
while j < 3:
print("[INFIO] REFRESHING PAGE... TRIAL : "+ str(j+1))
refresh()
sleep(3)
checkConnection()
print("[WARNING] : TRY AGAIN FAILED")
sleep(1)
print("[WARNING] : *****QUIT*****")
sleep(5)
quit()
I'm making a simulation of a traffic light and a car. When the light turns green, the car should move. (I'm aware that actual stoplights don't jump from green to red, or from red to yellow, but...just go with it). The program takes input from the user as to how long the stoplight should loop; it starts by remaining red for 3 seconds and then looping through the other colors.
So my problem is that when I try to move the car (represented by a Rectangle object here) and wheels (two Circle objects), the graphics window appears to be creating an entirely new rectangle, despite the fact that I haven't called the clone() method or anything anywhere in my program. Try running the program for 8+ seconds to see.
Why is the car rectangle making copies of itself instead of just moving the original?
Code below:
import sys
from graphics import *
import time
def drawCar(win):
car = Rectangle(Point(0,510), Point(100,555))
car.setFill(color_rgb(255,0,0))
wheel1 = Circle(Point(15,565),10)
wheel2 = Circle(Point(75,565),10)
wheel1.setFill(color_rgb(0,0,0))
wheel2.setFill(color_rgb(0,0,0))
car.draw(win)
wheel1.draw(win)
wheel2.draw(win)
def drawTrack(win):
rect = Rectangle(Point(0,575),Point(500,600))
rect.setFill(color_rgb(0,0,0))
rect.draw(win)
drawCar(win)
def loop(x):
# opens and titles a graphics window
win = GraphWin("Traffic Light", 500,600)
# creates 3 black circles for the window
red = Circle(Point(250,100),80)
yellow = Circle(Point(250,260),80)
green = Circle(Point(250,420),80)
# draw the car and track
drawTrack(win)
corner1 = Point(0,510)
corner2 = Point(100,555)
car = Rectangle(corner1,corner2)
car.setFill(color_rgb(255,0,0))
wheel1 = Circle(Point(15,565),10)
wheel2 = Circle(Point(75,565),10)
wheel1.setFill(color_rgb(0,0,0))
wheel2.setFill(color_rgb(0,0,0))
car.draw(win)
wheel1.draw(win)
wheel2.draw(win)
# sets default colors of the circles
red.setFill(color_rgb(255,0,0))
yellow.setFill(color_rgb(0,0,0))
green.setFill(color_rgb(0,0,0))
red.draw(win)
yellow.draw(win)
green.draw(win)
redCount = 1 # red light counter is automatically set to 1 because it starts with red by default
yelCount = 0 # yellow and green light counters are set to 0
greenCount = 0
redSet = True
yellowSet = False
greenSet = False
start = time.time()
end = time.time() + x
time.sleep(2) # wait 2 seconds while showing the red light (since the loop waits 1 additional second on the red), then begin the color-changing
while (time.time() - start < end):
if(time.time() + 1 > end): # if the time it will take to rest on the next light will make it go overtime
break # then stop
if redSet:
print("Red, changing to yellow")
red.setFill(color_rgb(0,0,0))
yellow.setFill(color_rgb(255,255,0))
yelCount += 1
redSet = False
yellowSet = True
time.sleep(1) # hold the yellow light for 1 second
elif yellowSet:
yellow.setFill(color_rgb(0,0,0))
green.setFill(color_rgb(0,255,0))
greenCount += 1
yellowSet = False
greenSet = True
time.sleep(1) # hold green light for 1 second
#corner1.move(60,0)
#corner2.move(60,0)
car.move(60,0)
wheel1.move(60,0)
wheel2.move(60,0)
print("Corners moved")
elif greenSet:
green.setFill(color_rgb(0,0,0))
red.setFill(color_rgb(255,0,0))
greenSet = False
redSet = True
time.sleep(1)
redCount += 1
else:
break
print("Red light hit ", redCount)
print("Yellow light hit ", yelCount)
print("Green light hit ", greenCount)
# if the user clicks anywhere in the window
win.getMouse()
# then we close the window
win.close()
def main():
# prompts the user for a time limit
x = float(input("How many seconds do you want the traffic light simulation to last? "))
# prints confirmation
print("Starting the loop for ", x, " seconds:")
# begins the loop
loop(x)
main()
The problem is because you're drawing the car (and wheels) twice.
Fortunately the fix to prevent that is simple:
def drawTrack(win):
rect = Rectangle(Point(0,575),Point(500,600))
rect.setFill(color_rgb(0,0,0))
rect.draw(win)
# drawCar(win) # Don't do this.
Note: After doing this, there will be no calls at all the function drawCar() so you could remove it. Another alternative would be to leave it in and replace the lines of code in the initialization part of the loop() function that do the same thing with a call to it (thereby making the code more modular).
I keep getting an invalid syntax error with this. So, how can I resolve this and where can I find related documentation in the future.
import cv2
import numpy as np
drawing = False # true if mouse is pressed
mode = True # if True, draw rectangle. Press 'm' to toggle to curve
ix,iy = -1,-1
class DessinerLigne:
def dessinerLigne(self):
# Create a black image
self.img=np.zeros((512,512,3),np.uint8)
def draw_circle(event,x,y,flags,param):
global ix,iy,drawing,mode
if event == cv2.EVENT_LBUTTONDOWN:
drawing = True
ix,iy = x,y
elif event == cv2.EVENT_MOUSEMOVE:
if drawing == True:
if mode == True:
cv2.rectangle(img,(ix,iy),(x,y),(0,255,0),-1)
else:
cv2.circle(img,(x,y),5,(0,0,255),-1)
elif event == cv2.EVENT_LBUTTONUP:
drawing = False
if mode == True:
cv2.rectangle(img,(ix,iy),(x,y),(0,255,0),-1)
else:
cv2.circle(img,(x,y),5,(0,0,255),-
img = np.zeros((512,512,3), np.uint8)
cv2.imshow("Image", self.img)
# If q is pressed then exit program
self.k=cv2.waitKey(0)
if self.k==ord('q'):
cv2.destroyAllWindows()
if __name__=="__main__":
DL=DessinerLigne()
DL.dessinerLigne()
There clearly are multiple issues with this script. Ones that need immediate attention are:
There's an indentation error in the definition of dessinerLigne class.
Change:
class DessinerLigne:
def dessinerLigne(self):
# Create a black image
self.img=np.zeros((512,512,3),np.uint8)
to:
class DessinerLigne:
def dessinerLigne(self):
# Create a black image
self.img=np.zeros((512,512,3),np.uint8)
and the indentation error should be fixed.
There's an incomplete line of code in line 32.
Is line 33 a part of the method draw_circle()? If so, it has be properly indented. Add 4 whitespaces in front of it.
You seem to have pasted the code from somewhere. During this process, it is very likely that some invisible control characters that might break the syntax may have arrived. Use an editor that has 'show invisible' features to resolve this issue.