Develop Reference

python Opencv template match error with win32api

So i have been trying to use this code to get a screenshot from a window, with this screenshot i will use opencv to match template, but every time that i try to match template i always get the same error, i have tried with a few other things like image grab from pillow, pyautogui and i am now trying with windowsAPI. i will say in advance this code is not mine.
Can anyone help me with this?
the error:
test = cv.matchTemplate(screenshot, target, cv.TM_CCORR_NORMED)
cv2.error: OpenCV(4.6.0) :-1: error: (-5:Bad argument) in function 'matchTemplate'
> Overload resolution failed:
> - templ is not a numpy array, neither a scalar
> - Expected Ptr<cv::UMat> for argument 'templ'
the source code:
import cv2 as cv
import numpy as np
import pyautogui
from time import time
import win32gui, win32ui, win32con
from PIL import ImageGrab
game = 'LOST ARK (64-bit, DX9) v.2.5.3.1'
target = 'box.png'
def list_window_names():
def winEnumHandler(hwnd, ctx):
if win32gui.IsWindowVisible(hwnd):
print(hex(hwnd), win32gui.GetWindowText(hwnd))
win32gui.EnumWindows(winEnumHandler, None)
def windowcapture():
hwnd = win32gui.FindWindow(None, game)
rect = win32gui.GetWindowRect(hwnd)
w = rect[2] - rect[0]
h = rect[3] - rect[1]
#hwnd = win32gui.FindWindow(None, windowname)
wDc = win32gui.GetWindowDC(hwnd)
dcObj = win32ui.CreateDCFromHandle(wDc)
cDC = dcObj.CreateCompatibleDC()
dataBitMap = win32ui.CreateBitmap()
dataBitMap.CreateCompatibleBitmap(dcObj, w, h)
cDC.SelectObject(dataBitMap)
cDC.BitBlt((0, 0), (w, h), dcObj, (0, 0), win32con.SRCCOPY)
dataBitMap.SaveBitmapFile(cDC, 'debug.bmp')
signedIntsArray = dataBitMap.GetBitmapBits(True)
img = np.fromstring(signedIntsArray, dtype='uint8')
img.shape = (h, w, 4)
dcObj.DeleteDC()
cDC.DeleteDC()
win32gui.ReleaseDC(hwnd, wDc)
win32gui.DeleteObject(dataBitMap.GetHandle())
return img
list_window_names()
loop_time = time()
while True:
#screenshot = ImageGrab.grab(bbox = (w,h,x,y))
#screenshot = pyautogui.screenshot()
#screenshot = np.array(screenshot)
#screenshot = cv.cvtColor(screenshot, cv.COLOR_RGB2BGR)
#print('top:{} , left:{}, w:{} ,h: {}'.format(w,h,x,y))
#screenshot.show()
#screenshot = np.array(screenshot)
#screenshot = screenshot[:, :, ::-1].copy()
#screenshot = screenshot[..., :3].copy()
screenshot = windowcapture()
cv.imshow("computer vision",screenshot)
method = cv.TM_CCORR
result = cv.matchTemplate(screenshot, target, method)
# debug the loop rate
print('FPS {}'.format(1 / (time() - loop_time)))
loop_time = time()
if cv.waitKey(1) == ord('q'):
cv.destroyAllWindows()
break
print('Done.')

How do I get rid of black bar in python window?

I'm working on a Object Detection project for the game Cuphead using OpenCV and Python. Now I'm trying to capture objects in real time but when the detection window displays I get this rare black bar on the top and I don't know how to get rid of it, here's what I see, on the left my object detection window and in the right the Cuphead game window.
Here's the code for the class used for this:
import numpy as np
import win32gui, win32ui, win32con
class WindowCapture:
# define monitor's width and height
w = 0
h = 0
hwnd = None
# constructor
def __init__(self, window_name):
if window_name is None: # if we don't pass any window names capture desktop
self.hwnd = win32gui.GetDesktopWindow()
else:
# Find the game window
self.hwnd = win32gui.FindWindow(None, window_name)
if not self.hwnd:
raise Exception("Window not founnd: {}".format(window_name))
# define window's widht and height. the resolution we'll work with
window_rect = win32gui.GetWindowRect(self.hwnd)
self.w = window_rect[2] - window_rect[0]
self.h = window_rect[3] - window_rect[1]
def get_screenshot(self):
# get the window image data
wDC = win32gui.GetWindowDC(self.hwnd)
dcObj = win32ui.CreateDCFromHandle(wDC)
cDC = dcObj.CreateCompatibleDC()
dataBitMap = win32ui.CreateBitmap()
dataBitMap.CreateCompatibleBitmap(dcObj, self.w, self.h)
cDC.SelectObject(dataBitMap)
cDC.BitBlt((0,0), (self.w, self.h), dcObj, (0,0), win32con.SRCCOPY)
# create the screenshot image that we want to return to be processed
signedIntsArray = dataBitMap.GetBitmapBits(True)
img = np.fromstring(signedIntsArray, dtype='uint8')
img.shape = (self.h, self.w, 4)
# Free Resources
dcObj.DeleteDC()
cDC.DeleteDC()
win32gui.ReleaseDC(self.hwnd, wDC)
win32gui.DeleteObject(dataBitMap.GetHandle())
# get rid of the alpha channel in the img
img = img[..., :3]
img = np.ascontiguousarray(img)
return img

It seems img.shape = (self.h, self.w, 4) causes the problem. As GetWindowRect and #IInspectable said,
In Windows Vista and later, the Window Rect now includes the area
occupied by the drop shadow.

OpenCv window cascading the screen when it's clicked on

I wrote this code to capture screen footage by taking screenshots everything second, however the OpenCv window cascades the screen when it's clicked on.
When you hover over the Opencv window, it shows me the screen without a hundred duplicates. What is exactly the bug? Picture:
The code for using the class I made:
from window_capture import Window_Capture
from time import time
import cv2 as cv
import numpy
loop_time = time()
wincap = Window_Capture()
while True:
screenshot = wincap.screenshare()
cv.imshow('Screen', screenshot)
print("FPS {}".format(round(1 / (time() - loop_time))))
loop_time = time()
if cv.waitKey(1) == ord('q'):
print("Capture Ended")
cv.destroyWindow(screenshot)
The class for screen capturing:
import pyautogui
import numpy
import cv2 as cv
from time import time
from PIL import ImageGrab
import win32gui, win32ui, win32con
from win32api import GetSystemMetrics
import time
class Window_Capture:
w = 0
h = 0
hwnd = None
def __init__(self, window_name=None):
# checking is windowname exists
if window_name is None:
print("No window set to capture: Setting to active Window")
time.sleep(3)
self.hwnd = win32gui.GetDesktopWindow()
print(win32gui.GetWindowText(win32gui.GetDesktopWindow()))
else:
# self.hwnd = win32gui.FindWindow(None, window_name)
self.hwnd = win32gui.FindWindow(None, window_name)
print("Window Found: Setting to preferred Window")
# define monitor height and width and hwnd
self.w = GetSystemMetrics(0)
self.h = GetSystemMetrics(1)
def screenshare(self):
wDC = win32gui.GetWindowDC(self.hwnd)
dcObj = win32ui.CreateDCFromHandle(wDC)
cDC = dcObj.CreateCompatibleDC()
dataBitMap = win32ui.CreateBitmap()
dataBitMap.CreateCompatibleBitmap(dcObj, self.w, self.h)
cDC.SelectObject(dataBitMap)
cDC.BitBlt((0, 0), (self.w, self.h), dcObj, (0, 0), win32con.SRCCOPY)
signedIntsArray = dataBitMap.GetBitmapBits(True)
img = numpy.frombuffer(signedIntsArray, dtype='uint8')
img.shape = (self.h, self.w, 4)
dcObj.DeleteDC()
cDC.DeleteDC()
win32gui.ReleaseDC(self.hwnd, wDC)
win32gui.DeleteObject(dataBitMap.GetHandle())
img = img[..., :3]
img = numpy.ascontiguousarray(img)
return img
This bug is preventing me from moving onto the next part my image detection project.

Better solution is :
import time
import cv2
import numpy as np
from mss import mss
#import pyautogui
#width, height= pyautogui.size()
width=900
height=900
mon = {'top': 0, 'left': 0, 'width': width, 'height': height}
with mss() as sct:
# mon = sct.monitors[0]
while True:
last_time = time.time()
img = sct.grab(mon)
print('fps: {0}'.format(1 / (time.time()-last_time)))
cv2.imshow('test', np.array(img))
if cv2.waitKey(25) & 0xFF == ord('q'):
cv2.destroyAllWindows()
break
Note:
When you hover then you won't be seeing many windows beacuse you see the complete window without the opencv window opended and else you see many of them

you see a cascade because you take a screenshot of the window itself, repeatedly.
it's like standing between two mirrors.

Obtain image data of separate application using Python3 on Linux

Summary: How can I get the image data (a screenshot) of an application by its window name in Linux for further processing using Python 3.X
I need to do some image detection on GUIs running in both Linux and Windows. I have working code for Windows 10 (below) that gets an applications image data using the win32 API and puts it into a numpy array for using with OpenCV. I now need a similar approach for Linux.
Importantly, it would be greatly beneficial if the window did not have to be visible for the image data to be obtained (the windows will never be minimised, but they might be obscured by other applications, and I don't want to have to force the windows to be the active windows for the sake of what is essentially a screenshot)
I have come across this thread using the Gdk API, which works, but only for the application that invokes it. I have also read this thread, using PIL might be the way to go, but i'd appreciate some input on this matter
Example of what I need, working on Windows 10
import numpy as np
import win32gui, win32ui, win32con
import cv2 as cv
def get_img_data(window_name):
window = win32gui.FindWindow(None, window_name)
if not window:
raise Exception('Window not found: {}'.format(window_name))
# get the window size
window_rect = win32gui.GetWindowRect(window)
w = window_rect[2] - window_rect[0]
h = window_rect[3] - window_rect[1]
# account for the window border and titlebar and cut them off
border_pixels = 8
titlebar_pixels = 30
w = w - (border_pixels * 2)
h = h - titlebar_pixels - border_pixels
cropped_x = border_pixels
cropped_y = titlebar_pixels
# set the cropped coordinates offset so we can translate screenshot
# images into actual screen positions
offset_x = window_rect[0] + cropped_x
offset_y = window_rect[1] + cropped_y
wDC = win32gui.GetWindowDC(window)
dcObj = win32ui.CreateDCFromHandle(wDC)
cDC = dcObj.CreateCompatibleDC()
dataBitMap = win32ui.CreateBitmap()
dataBitMap.CreateCompatibleBitmap(dcObj, w, h)
cDC.SelectObject(dataBitMap)
cDC.BitBlt((0, 0), (w, h), dcObj, (cropped_x, cropped_y), win32con.SRCCOPY)
signedIntsArray = dataBitMap.GetBitmapBits(True)
img = np.fromstring(signedIntsArray, dtype='uint8')
img.shape = (h, w, 4)
# free resources
dcObj.DeleteDC()
cDC.DeleteDC()
win32gui.ReleaseDC(window, wDC)
win32gui.DeleteObject(dataBitMap.GetHandle())
img = img[...,:3]
img = np.ascontiguousarray(img)
# cv.imwrite('./test.png', img) # uncomment to save the screenshot
return img
img_data = get_img_data('Task Manager')

Taking fast screenshot Winapi and Opencv

I need to take very fast screenshots of a game for an OpenCV project I am working on. I can use PIL easily for example:
def take_screenshot1(hwnd):
rect = win32gui.GetWindowRect(hwnd)
img = ImageGrab.grab(bbox=rect)
img_np = np.array(img)
return cv2.cvtColor(img_np, cv2.COLOR_RGB2BGR)
But it takes on average of 0.05 seconds which isn't fast enough for real time capture.
I can use the answer posted here, but that only saves the bitmap to a file. That is over 10 times faster than by using PIL, but I am unsure of any methods within OpenCV to convert it to a bgr/hsv image.
def take_screenshot(hwnd):
wDC = win32gui.GetWindowDC(hwnd)
dcObj=win32ui.CreateDCFromHandle(wDC)
cDC=dcObj.CreateCompatibleDC()
dataBitMap = win32ui.CreateBitmap()
dataBitMap.CreateCompatibleBitmap(dcObj, 500, 500)
cDC.SelectObject(dataBitMap)
cDC.BitBlt((0, 0), (500, 500), dcObj, (0, 0), win32con.SRCCOPY)
dataBitMap.SaveBitmapFile(cDC, "foo.png")
dcObj.DeleteDC()
cDC.DeleteDC()
win32gui.ReleaseDC(hwnd, wDC)
win32gui.DeleteObject(dataBitMap.GetHandle())
im = cv2.imread("foo.png")
return cv2.cvtColor(im, cv2.COLOR_RGB2BGR)
EDIT:The size of the window is 500x500, so it is saving the same area in both examples.
Even if I save the image, and then reopen it with OpenCV it is still faster than PIL, but surely there is an easier way?
EDIT: Ok so using the comments and doing some research on winapi I now can access the bitmap data directly as follows:
def take_screenshot1(hwnd):
wDC = win32gui.GetWindowDC(hwnd)
dcObj=win32ui.CreateDCFromHandle(wDC)
cDC=dcObj.CreateCompatibleDC()
dataBitMap = win32ui.CreateBitmap()
dataBitMap.CreateCompatibleBitmap(dcObj, 500, 500)
cDC.SelectObject(dataBitMap)
cDC.BitBlt((0, 0), (500, 500), dcObj, (0, 0), win32con.SRCCOPY)
im = dataBitMap.GetBitmapBits(True) # Tried False also
img = np.array(im)
cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
print(img)
dcObj.DeleteDC()
cDC.DeleteDC()
win32gui.ReleaseDC(hwnd, wDC)
win32gui.DeleteObject(dataBitMap.GetHandle())
But i'm not sure how to convert the returned bitmap to a form that OpenCV understands, as there are no methods to convert bitmap to rgb/bgr in OpenCV

I'll just show the code that works for me.
import time
import win32gui
import win32ui
import win32con
import win32api
import numpy as np
import cv2
def window_capture():
hwnd = 0
hwndDC = win32gui.GetWindowDC(hwnd)
mfcDC = win32ui.CreateDCFromHandle(hwndDC)
saveDC = mfcDC.CreateCompatibleDC()
saveBitMap = win32ui.CreateBitmap()
MoniterDev = win32api.EnumDisplayMonitors(None, None)
w = MoniterDev[0][2][2]
h = MoniterDev[0][2][3]
saveBitMap.CreateCompatibleBitmap(mfcDC, w, h)
saveDC.SelectObject(saveBitMap)
saveDC.BitBlt((0, 0), (w, h), mfcDC, (0, 0), win32con.SRCCOPY)
im = saveBitMap.GetBitmapBits(True) # Tried False also
img = np.frombuffer(im, dtype=np.uint8).reshape((h, w, 4))
cv2.imshow("demo", img)
cv2.waitKey(100)
beg = time.time()
for i in range(100):
window_capture()
end = time.time()
print(end - beg)
cv2.destroyAllWindows()