I need to take very fast screenshots of a game for an OpenCV project I am working on. I can use PIL easily for example:
def take_screenshot1(hwnd):
rect = win32gui.GetWindowRect(hwnd)
img = ImageGrab.grab(bbox=rect)
img_np = np.array(img)
return cv2.cvtColor(img_np, cv2.COLOR_RGB2BGR)
But it takes on average of 0.05 seconds which isn't fast enough for real time capture.
I can use the answer posted here, but that only saves the bitmap to a file. That is over 10 times faster than by using PIL, but I am unsure of any methods within OpenCV to convert it to a bgr/hsv image.
def take_screenshot(hwnd):
wDC = win32gui.GetWindowDC(hwnd)
dcObj=win32ui.CreateDCFromHandle(wDC)
cDC=dcObj.CreateCompatibleDC()
dataBitMap = win32ui.CreateBitmap()
dataBitMap.CreateCompatibleBitmap(dcObj, 500, 500)
cDC.SelectObject(dataBitMap)
cDC.BitBlt((0, 0), (500, 500), dcObj, (0, 0), win32con.SRCCOPY)
dataBitMap.SaveBitmapFile(cDC, "foo.png")
dcObj.DeleteDC()
cDC.DeleteDC()
win32gui.ReleaseDC(hwnd, wDC)
win32gui.DeleteObject(dataBitMap.GetHandle())
im = cv2.imread("foo.png")
return cv2.cvtColor(im, cv2.COLOR_RGB2BGR)
EDIT:The size of the window is 500x500, so it is saving the same area in both examples.
Even if I save the image, and then reopen it with OpenCV it is still faster than PIL, but surely there is an easier way?
EDIT: Ok so using the comments and doing some research on winapi I now can access the bitmap data directly as follows:
def take_screenshot1(hwnd):
wDC = win32gui.GetWindowDC(hwnd)
dcObj=win32ui.CreateDCFromHandle(wDC)
cDC=dcObj.CreateCompatibleDC()
dataBitMap = win32ui.CreateBitmap()
dataBitMap.CreateCompatibleBitmap(dcObj, 500, 500)
cDC.SelectObject(dataBitMap)
cDC.BitBlt((0, 0), (500, 500), dcObj, (0, 0), win32con.SRCCOPY)
im = dataBitMap.GetBitmapBits(True) # Tried False also
img = np.array(im)
cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
print(img)
dcObj.DeleteDC()
cDC.DeleteDC()
win32gui.ReleaseDC(hwnd, wDC)
win32gui.DeleteObject(dataBitMap.GetHandle())
But i'm not sure how to convert the returned bitmap to a form that OpenCV understands, as there are no methods to convert bitmap to rgb/bgr in OpenCV
I'll just show the code that works for me.
import time
import win32gui
import win32ui
import win32con
import win32api
import numpy as np
import cv2
def window_capture():
hwnd = 0
hwndDC = win32gui.GetWindowDC(hwnd)
mfcDC = win32ui.CreateDCFromHandle(hwndDC)
saveDC = mfcDC.CreateCompatibleDC()
saveBitMap = win32ui.CreateBitmap()
MoniterDev = win32api.EnumDisplayMonitors(None, None)
w = MoniterDev[0][2][2]
h = MoniterDev[0][2][3]
saveBitMap.CreateCompatibleBitmap(mfcDC, w, h)
saveDC.SelectObject(saveBitMap)
saveDC.BitBlt((0, 0), (w, h), mfcDC, (0, 0), win32con.SRCCOPY)
im = saveBitMap.GetBitmapBits(True) # Tried False also
img = np.frombuffer(im, dtype=np.uint8).reshape((h, w, 4))
cv2.imshow("demo", img)
cv2.waitKey(100)
beg = time.time()
for i in range(100):
window_capture()
end = time.time()
print(end - beg)
cv2.destroyAllWindows()
Related
So i have been trying to use this code to get a screenshot from a window, with this screenshot i will use opencv to match template, but every time that i try to match template i always get the same error, i have tried with a few other things like image grab from pillow, pyautogui and i am now trying with windowsAPI. i will say in advance this code is not mine.
Can anyone help me with this?
the error:
test = cv.matchTemplate(screenshot, target, cv.TM_CCORR_NORMED)
cv2.error: OpenCV(4.6.0) :-1: error: (-5:Bad argument) in function 'matchTemplate'
> Overload resolution failed:
> - templ is not a numpy array, neither a scalar
> - Expected Ptr<cv::UMat> for argument 'templ'
the source code:
import cv2 as cv
import numpy as np
import pyautogui
from time import time
import win32gui, win32ui, win32con
from PIL import ImageGrab
game = 'LOST ARK (64-bit, DX9) v.2.5.3.1'
target = 'box.png'
def list_window_names():
def winEnumHandler(hwnd, ctx):
if win32gui.IsWindowVisible(hwnd):
print(hex(hwnd), win32gui.GetWindowText(hwnd))
win32gui.EnumWindows(winEnumHandler, None)
def windowcapture():
hwnd = win32gui.FindWindow(None, game)
rect = win32gui.GetWindowRect(hwnd)
w = rect[2] - rect[0]
h = rect[3] - rect[1]
#hwnd = win32gui.FindWindow(None, windowname)
wDc = win32gui.GetWindowDC(hwnd)
dcObj = win32ui.CreateDCFromHandle(wDc)
cDC = dcObj.CreateCompatibleDC()
dataBitMap = win32ui.CreateBitmap()
dataBitMap.CreateCompatibleBitmap(dcObj, w, h)
cDC.SelectObject(dataBitMap)
cDC.BitBlt((0, 0), (w, h), dcObj, (0, 0), win32con.SRCCOPY)
dataBitMap.SaveBitmapFile(cDC, 'debug.bmp')
signedIntsArray = dataBitMap.GetBitmapBits(True)
img = np.fromstring(signedIntsArray, dtype='uint8')
img.shape = (h, w, 4)
dcObj.DeleteDC()
cDC.DeleteDC()
win32gui.ReleaseDC(hwnd, wDc)
win32gui.DeleteObject(dataBitMap.GetHandle())
return img
list_window_names()
loop_time = time()
while True:
#screenshot = ImageGrab.grab(bbox = (w,h,x,y))
#screenshot = pyautogui.screenshot()
#screenshot = np.array(screenshot)
#screenshot = cv.cvtColor(screenshot, cv.COLOR_RGB2BGR)
#print('top:{} , left:{}, w:{} ,h: {}'.format(w,h,x,y))
#screenshot.show()
#screenshot = np.array(screenshot)
#screenshot = screenshot[:, :, ::-1].copy()
#screenshot = screenshot[..., :3].copy()
screenshot = windowcapture()
cv.imshow("computer vision",screenshot)
method = cv.TM_CCORR
result = cv.matchTemplate(screenshot, target, method)
# debug the loop rate
print('FPS {}'.format(1 / (time() - loop_time)))
loop_time = time()
if cv.waitKey(1) == ord('q'):
cv.destroyAllWindows()
break
print('Done.')
I'm working on a Object Detection project for the game Cuphead using OpenCV and Python. Now I'm trying to capture objects in real time but when the detection window displays I get this rare black bar on the top and I don't know how to get rid of it, here's what I see, on the left my object detection window and in the right the Cuphead game window.
Here's the code for the class used for this:
import numpy as np
import win32gui, win32ui, win32con
class WindowCapture:
# define monitor's width and height
w = 0
h = 0
hwnd = None
# constructor
def __init__(self, window_name):
if window_name is None: # if we don't pass any window names capture desktop
self.hwnd = win32gui.GetDesktopWindow()
else:
# Find the game window
self.hwnd = win32gui.FindWindow(None, window_name)
if not self.hwnd:
raise Exception("Window not founnd: {}".format(window_name))
# define window's widht and height. the resolution we'll work with
window_rect = win32gui.GetWindowRect(self.hwnd)
self.w = window_rect[2] - window_rect[0]
self.h = window_rect[3] - window_rect[1]
def get_screenshot(self):
# get the window image data
wDC = win32gui.GetWindowDC(self.hwnd)
dcObj = win32ui.CreateDCFromHandle(wDC)
cDC = dcObj.CreateCompatibleDC()
dataBitMap = win32ui.CreateBitmap()
dataBitMap.CreateCompatibleBitmap(dcObj, self.w, self.h)
cDC.SelectObject(dataBitMap)
cDC.BitBlt((0,0), (self.w, self.h), dcObj, (0,0), win32con.SRCCOPY)
# create the screenshot image that we want to return to be processed
signedIntsArray = dataBitMap.GetBitmapBits(True)
img = np.fromstring(signedIntsArray, dtype='uint8')
img.shape = (self.h, self.w, 4)
# Free Resources
dcObj.DeleteDC()
cDC.DeleteDC()
win32gui.ReleaseDC(self.hwnd, wDC)
win32gui.DeleteObject(dataBitMap.GetHandle())
# get rid of the alpha channel in the img
img = img[..., :3]
img = np.ascontiguousarray(img)
return img
It seems img.shape = (self.h, self.w, 4) causes the problem. As GetWindowRect and #IInspectable said,
In Windows Vista and later, the Window Rect now includes the area
occupied by the drop shadow.
I need to make a screen capture of a specific application. If I grab the Windows desktop (hwnd = None), everything works, but I get a black screen when I try to grab the screen of a specific application, e.g. hwnd = win32gui.FindWindow(None, 'Albion Online Client').
import cv2 as cv
import numpy as np
import os
from time import time
import win32gui
import win32ui
import win32con
os.chdir(os.path.dirname(os.path.abspath(__file__)))
def window_capture():
w = 1920 # set this
h = 1080 # set this
#hwnd = None ### Everything works
hwnd = win32gui.FindWindow(None, 'Albion Online Client') ### Black Screen
wDC = win32gui.GetWindowDC(hwnd)
dcObj = win32ui.CreateDCFromHandle(wDC)
cDC = dcObj.CreateCompatibleDC()
dataBitMap = win32ui.CreateBitmap()
dataBitMap.CreateCompatibleBitmap(dcObj, w, h)
cDC.SelectObject(dataBitMap)
cDC.BitBlt((0, 0), (w, h), dcObj, (0, 0), win32con.SRCCOPY)
#Save screenshoot
#dataBitMap.SaveBitmapFile(cDC, 'debug.bmp' )
signedIntsArray = dataBitMap.GetBitmapBits(True)
img = np.fromstring(signedIntsArray, dtype='uint8')
img.shape = (h, w, 4)
# Free Resources
dcObj.DeleteDC()
cDC.DeleteDC()
win32gui.ReleaseDC(hwnd, wDC)
win32gui.DeleteObject(dataBitMap.GetHandle())
img = img[..., :3]
img = np.ascontiguousarray(img)
return img
# initialize the WindowCapture clas
loop_time = time()
while(True):
screenshot = window_capture()
cv.imshow('Computer Vision', screenshot)
# debug the loop rate
print('FPS {}'.format(1 / (time() - loop_time)))
loop_time = time()
# press 'q' with the output window focused to exit.
# waits 1 ms every loop to process key presses
if cv.waitKey(1) == ord('q'):
cv.destroyAllWindows()
break
print('Done.')
That's an OpenGL or Direct3D application. It doesn't draw to its own window. It creates textures and surfaces, and has the graphics card render its display list directly into the frame buffer.
I am trying to crop an image in python in circular shape. And I also want to paste that image on the top of another image, and then then then save the image in a desired format.
This is the image that I want to crop in circular way
This is how the image should be looked like after cropping
This is the image on which I want to paste the circular shaped image
This is my expected output
Here is the code, as far I tried
from PIL import Image, ImageDraw, ImageFilter
im1 = Image.open('rocket.jpg')
im2 = Image.open('lena.jpg')
width, height = im1.size
print(height, width)
mask_im = Image.new("L", im2.size, 0)
draw = ImageDraw.Draw(mask_im)
draw.ellipse((150, 40, 250, 100), fill=255)
mask_im.save('mask_circle.jpg', quality=95)
back_im = im1.copy()
back_im.paste(im2, (0, 0), mask_im)
back_im.save('rocket_pillow_paste_mask_circle.jpg', quality=95)
mask_im_blur = mask_im.filter(ImageFilter.GaussianBlur(10))
mask_im_blur.save('mask_circle_blur.jpg', quality=95)
back_im = im1.copy()
back_im.paste(im2, (0, 0), mask_im_blur)
back_im.save('rocket_pillow_paste_mask_circle_blur.jpg', quality=95)
from PIL import Image, ImageDraw, ImageFilter
im1 = Image.open('rocket.jpg')
im2 = Image.open('lena.jpg')
#height, width, channels = im1.shape
width, height = im1.size
print(height, width)
# 
mask_im = Image.new("L", im2.size, 0)
draw = ImageDraw.Draw(mask_im)
draw.ellipse((140, 50, 260, 170), fill=255)
mask_im.save('mask_circle.jpg', quality=95)
back_im = im1.copy()
back_im.paste(im2, (0, 0), mask_im)
back_im.save('rocket_pillow_paste_mask_circle.jpg', quality=95)
# 
mask_im_blur = mask_im.filter(ImageFilter.GaussianBlur(10))
mask_im_blur.save('mask_circle_blur.jpg', quality=95)
back_im = im1.copy()
back_im.paste(im2, (0, 0), mask_im_blur)
back_im.save('rocket_pillow_paste_mask_circle_blur.jpg', quality=95)
# 
Summary: How can I get the image data (a screenshot) of an application by its window name in Linux for further processing using Python 3.X
I need to do some image detection on GUIs running in both Linux and Windows. I have working code for Windows 10 (below) that gets an applications image data using the win32 API and puts it into a numpy array for using with OpenCV. I now need a similar approach for Linux.
Importantly, it would be greatly beneficial if the window did not have to be visible for the image data to be obtained (the windows will never be minimised, but they might be obscured by other applications, and I don't want to have to force the windows to be the active windows for the sake of what is essentially a screenshot)
I have come across this thread using the Gdk API, which works, but only for the application that invokes it. I have also read this thread, using PIL might be the way to go, but i'd appreciate some input on this matter
Example of what I need, working on Windows 10
import numpy as np
import win32gui, win32ui, win32con
import cv2 as cv
def get_img_data(window_name):
window = win32gui.FindWindow(None, window_name)
if not window:
raise Exception('Window not found: {}'.format(window_name))
# get the window size
window_rect = win32gui.GetWindowRect(window)
w = window_rect[2] - window_rect[0]
h = window_rect[3] - window_rect[1]
# account for the window border and titlebar and cut them off
border_pixels = 8
titlebar_pixels = 30
w = w - (border_pixels * 2)
h = h - titlebar_pixels - border_pixels
cropped_x = border_pixels
cropped_y = titlebar_pixels
# set the cropped coordinates offset so we can translate screenshot
# images into actual screen positions
offset_x = window_rect[0] + cropped_x
offset_y = window_rect[1] + cropped_y
wDC = win32gui.GetWindowDC(window)
dcObj = win32ui.CreateDCFromHandle(wDC)
cDC = dcObj.CreateCompatibleDC()
dataBitMap = win32ui.CreateBitmap()
dataBitMap.CreateCompatibleBitmap(dcObj, w, h)
cDC.SelectObject(dataBitMap)
cDC.BitBlt((0, 0), (w, h), dcObj, (cropped_x, cropped_y), win32con.SRCCOPY)
signedIntsArray = dataBitMap.GetBitmapBits(True)
img = np.fromstring(signedIntsArray, dtype='uint8')
img.shape = (h, w, 4)
# free resources
dcObj.DeleteDC()
cDC.DeleteDC()
win32gui.ReleaseDC(window, wDC)
win32gui.DeleteObject(dataBitMap.GetHandle())
img = img[...,:3]
img = np.ascontiguousarray(img)
# cv.imwrite('./test.png', img) # uncomment to save the screenshot
return img
img_data = get_img_data('Task Manager')