After trying out various approaches... I have stumbled upon this page to take full-page screenshot with chromedriver, selenium and python.
The original code is here. (and I copy the code in this posting below)
It uses PIL and it works great! However, there is one issue... which is it captures fixed headers and repeats for the whole page and also misses some parts of the page during page change. sample url to take a screenshot:
http://www.w3schools.com/js/default.asp
How to avoid the repeated headers with this code... Or is there any better option which uses python only... ( i don't know java and do not want to use java).
Please see the screenshot of the current result and sample code below.
test.py
"""
This script uses a simplified version of the one here:
https://snipt.net/restrada/python-selenium-workaround-for-full-page-screenshot-using-chromedriver-2x/
It contains the *crucial* correction added in the comments by Jason Coutu.
"""
import sys
from selenium import webdriver
import unittest
import util
class Test(unittest.TestCase):
""" Demonstration: Get Chrome to generate fullscreen screenshot """
def setUp(self):
self.driver = webdriver.Chrome()
def tearDown(self):
self.driver.quit()
def test_fullpage_screenshot(self):
''' Generate document-height screenshot '''
#url = "http://effbot.org/imagingbook/introduction.htm"
url = "http://www.w3schools.com/js/default.asp"
self.driver.get(url)
util.fullpage_screenshot(self.driver, "test.png")
if __name__ == "__main__":
unittest.main(argv=[sys.argv[0]])
util.py
import os
import time
from PIL import Image
def fullpage_screenshot(driver, file):
print("Starting chrome full page screenshot workaround ...")
total_width = driver.execute_script("return document.body.offsetWidth")
total_height = driver.execute_script("return document.body.parentNode.scrollHeight")
viewport_width = driver.execute_script("return document.body.clientWidth")
viewport_height = driver.execute_script("return window.innerHeight")
print("Total: ({0}, {1}), Viewport: ({2},{3})".format(total_width, total_height,viewport_width,viewport_height))
rectangles = []
i = 0
while i < total_height:
ii = 0
top_height = i + viewport_height
if top_height > total_height:
top_height = total_height
while ii < total_width:
top_width = ii + viewport_width
if top_width > total_width:
top_width = total_width
print("Appending rectangle ({0},{1},{2},{3})".format(ii, i, top_width, top_height))
rectangles.append((ii, i, top_width,top_height))
ii = ii + viewport_width
i = i + viewport_height
stitched_image = Image.new('RGB', (total_width, total_height))
previous = None
part = 0
for rectangle in rectangles:
if not previous is None:
driver.execute_script("window.scrollTo({0}, {1})".format(rectangle[0], rectangle[1]))
print("Scrolled To ({0},{1})".format(rectangle[0], rectangle[1]))
time.sleep(0.2)
file_name = "part_{0}.png".format(part)
print("Capturing {0} ...".format(file_name))
driver.get_screenshot_as_file(file_name)
screenshot = Image.open(file_name)
if rectangle[1] + viewport_height > total_height:
offset = (rectangle[0], total_height - viewport_height)
else:
offset = (rectangle[0], rectangle[1])
print("Adding to stitched image with offset ({0}, {1})".format(offset[0],offset[1]))
stitched_image.paste(screenshot, offset)
del screenshot
os.remove(file_name)
part = part + 1
previous = rectangle
stitched_image.save(file)
print("Finishing chrome full page screenshot workaround...")
return True
This answer improves upon prior answers by am05mhz and Javed Karim.
It assumes headless mode, and that a window-size option was not initially set. Before calling this function, ensure the page has loaded fully or sufficiently.
It attempts to set the width and height both to what is necessary. The screenshot of the entire page can sometimes include a needless vertical scrollbar. One way to generally avoid the scrollbar is by taking a screenshot of the body element instead. After saving a screenshot, it reverts the size to what it was originally, failing which the size for the next screenshot may not set correctly.
Ultimately this technique may still not work perfectly well for some examples.
from selenium import webdriver
def save_screenshot(driver: webdriver.Chrome, path: str = '/tmp/screenshot.png') -> None:
# Ref: https://stackoverflow.com/a/52572919/
original_size = driver.get_window_size()
required_width = driver.execute_script('return document.body.parentNode.scrollWidth')
required_height = driver.execute_script('return document.body.parentNode.scrollHeight')
driver.set_window_size(required_width, required_height)
# driver.save_screenshot(path) # has scrollbar
driver.find_element_by_tag_name('body').screenshot(path) # avoids scrollbar
driver.set_window_size(original_size['width'], original_size['height'])
If using Python older than 3.6, remove the type annotations from the function definition.
Screenshots are limited to the viewport but you can get around this by capturing the body element, as the webdriver will capture the entire element even if it is larger than the viewport. This will save you having to deal with scrolling and stitching images, however you might see problems with footer position (like in the screenshot below).
Tested on Windows 8 and Mac High Sierra with Chrome Driver.
from selenium import webdriver
url = 'https://stackoverflow.com/'
path = '/path/to/save/in/scrape.png'
driver = webdriver.Chrome()
driver.get(url)
el = driver.find_element_by_tag_name('body')
el.screenshot(path)
driver.quit()
Returns: (full size: https://i.stack.imgur.com/ppDiI.png)
How it works: set browser height as longest as you can...
#coding=utf-8
import time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
def test_fullpage_screenshot(self):
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--start-maximized')
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.get("yoururlxxx")
time.sleep(2)
#the element with longest height on page
ele=driver.find_element("xpath", '//div[#class="react-grid-layout layout"]')
total_height = ele.size["height"]+1000
driver.set_window_size(1920, total_height) #the trick
time.sleep(2)
driver.save_screenshot("screenshot1.png")
driver.quit()
if __name__ == "__main__":
test_fullpage_screenshot()
from selenium import webdriver
driver = webdriver.Firefox()
driver.get('https://developer.mozilla.org/')
element = driver.find_element_by_tag_name('body')
element_png = element.screenshot_as_png
with open("test2.png", "wb") as file:
file.write(element_png)
This works for me. It saves the entire page as screenshot.
For more information you can read up the api docs:
http://selenium-python.readthedocs.io/api.html
The key is to turn on the headless mode!
No stitching required and no need for loading the page twice.
Full working code:
URL = 'http://www.w3schools.com/js/default.asp'
options = webdriver.ChromeOptions()
options.headless = True
driver = webdriver.Chrome(options=options)
driver.get(URL)
S = lambda X: driver.execute_script('return document.body.parentNode.scroll'+X)
driver.set_window_size(S('Width'),S('Height')) # May need manual adjustment
driver.find_element_by_tag_name('body').screenshot('web_screenshot.png')
driver.quit()
This is practically the same code as posted by #Acumenus with slight improvements.
Summary of my findings
I decided to post this anyway because I did not find an explanation about what is happening when the headless mode is turned off (the browser is displayed) for screenshot taking purposes.
As I tested (with Chrome WebDriver), if the headless mode is turned on, the screenshot is saved as desired. However, if the headless mode is turned off, the saved screenshot has approximately the correct width and height, but the outcome varies case-by-case. Usually, the upper part of the page which is visible by the screen is saved, but the rest of the image is just plain white. There was also a case with trying to save this Stack Overflow thread by using the above link; even the upper part was not saved which interestingly now was transparent while the rest still white. The last case I noticed was only once with the given W3Schools link; there where no white parts but the upper part of the page repeated until the end, including the header.
I hope this will help for many of those who for some reason are not getting the expected result as I did not see anyone explicitly explaining about the requirement of headless mode with this simple approach.
Only when I discovered the solution to this problem myself, I found a post by #vc2279 mentioning that the window of a headless browser can be set to any size (which seems to be true for the opposite case too). Although, the solution in my post improves upon that that it does not require repeated browser/driver opening or page reloading.
Further suggestions
If for some pages it does not work for you, I suggest trying to add time.sleep(seconds) before getting the size of the page. Another case would be if the page requires scrolling until the bottom to load further content, which can be solved by the scheight method from this post:
scheight = .1
while scheight < 9.9:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight/%s);" % scheight)
scheight += .01
Also, note that for some pages the content may not be in any of the top-level HTML tags like <html> or <body>, for example, YouTube uses <ytd-app> tag.
As a last note, I found one page that "returned" a screenshot still with the horizontal scrollbar, the size of the window needed manual adjustment, i.e., the image width needed to be increased by 18 pixels, like so: S('Width')+18.
After knowing the approach of #Moshisho.
My full standalone working script is... (added sleep 0.2 after each scroll and position)
import sys
from selenium import webdriver
import util
import os
import time
from PIL import Image
def fullpage_screenshot(driver, file):
print("Starting chrome full page screenshot workaround ...")
total_width = driver.execute_script("return document.body.offsetWidth")
total_height = driver.execute_script("return document.body.parentNode.scrollHeight")
viewport_width = driver.execute_script("return document.body.clientWidth")
viewport_height = driver.execute_script("return window.innerHeight")
print("Total: ({0}, {1}), Viewport: ({2},{3})".format(total_width, total_height,viewport_width,viewport_height))
rectangles = []
i = 0
while i < total_height:
ii = 0
top_height = i + viewport_height
if top_height > total_height:
top_height = total_height
while ii < total_width:
top_width = ii + viewport_width
if top_width > total_width:
top_width = total_width
print("Appending rectangle ({0},{1},{2},{3})".format(ii, i, top_width, top_height))
rectangles.append((ii, i, top_width,top_height))
ii = ii + viewport_width
i = i + viewport_height
stitched_image = Image.new('RGB', (total_width, total_height))
previous = None
part = 0
for rectangle in rectangles:
if not previous is None:
driver.execute_script("window.scrollTo({0}, {1})".format(rectangle[0], rectangle[1]))
time.sleep(0.2)
driver.execute_script("document.getElementById('topnav').setAttribute('style', 'position: absolute; top: 0px;');")
time.sleep(0.2)
print("Scrolled To ({0},{1})".format(rectangle[0], rectangle[1]))
time.sleep(0.2)
file_name = "part_{0}.png".format(part)
print("Capturing {0} ...".format(file_name))
driver.get_screenshot_as_file(file_name)
screenshot = Image.open(file_name)
if rectangle[1] + viewport_height > total_height:
offset = (rectangle[0], total_height - viewport_height)
else:
offset = (rectangle[0], rectangle[1])
print("Adding to stitched image with offset ({0}, {1})".format(offset[0],offset[1]))
stitched_image.paste(screenshot, offset)
del screenshot
os.remove(file_name)
part = part + 1
previous = rectangle
stitched_image.save(file)
print("Finishing chrome full page screenshot workaround...")
return True
driver = webdriver.Chrome()
''' Generate document-height screenshot '''
url = "http://effbot.org/imagingbook/introduction.htm"
url = "http://www.w3schools.com/js/default.asp"
driver.get(url)
fullpage_screenshot(driver, "test1236.png")
Not sure if people are still having this issue.
I've done a small hack that works pretty well and that plays nicely with dynamic zones. Hope it helps
# 1. get dimensions
browser = webdriver.Chrome(chrome_options=options)
browser.set_window_size(default_width, default_height)
browser.get(url)
time.sleep(sometime)
total_height = browser.execute_script("return document.body.parentNode.scrollHeight")
browser.quit()
# 2. get screenshot
browser = webdriver.Chrome(chrome_options=options)
browser.set_window_size(default_width, total_height)
browser.get(url)
browser.save_screenshot(screenshot_path)
Why not just getting the width and height of the page and then resize the driver? So will be something like this
total_width = driver.execute_script("return document.body.offsetWidth")
total_height = driver.execute_script("return document.body.scrollHeight")
driver.set_window_size(total_width, total_height)
driver.save_screenshot("SomeName.png")
This is going to make a screenshot of your entire page without the need to merge together different pieces.
You can achieve this by changing the CSS of the header before the screenshot:
topnav = driver.find_element_by_id("topnav")
driver.execute_script("arguments[0].setAttribute('style', 'position: absolute; top: 0px;')", topnav)
EDIT: Put this line after your window scroll:
driver.execute_script("document.getElementById('topnav').setAttribute('style', 'position: absolute; top: 0px;');")
So in your util.py it will be:
driver.execute_script("window.scrollTo({0}, {1})".format(rectangle[0], rectangle[1]))
driver.execute_script("document.getElementById('topnav').setAttribute('style', 'position: absolute; top: 0px;');")
If the site is using the header tag, you can do it with find_element_by_tag_name("header")
I changed code for Python 3.6, maybe it will be useful for someone:
from selenium import webdriver
from sys import stdout
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import unittest
#from Login_Page import Login_Page
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from io import BytesIO
from PIL import Image
def testdenovoUIavailable(self):
binary = FirefoxBinary("C:\\Mozilla Firefox\\firefox.exe")
self.driver = webdriver.Firefox(firefox_binary=binary)
verbose = 0
#open page
self.driver.get("http://yandex.ru")
#hide fixed header
#js_hide_header=' var x = document.getElementsByClassName("topnavbar-wrapper ng-scope")[0];x[\'style\'] = \'display:none\';'
#self.driver.execute_script(js_hide_header)
#get total height of page
js = 'return Math.max( document.body.scrollHeight, document.body.offsetHeight, document.documentElement.clientHeight, document.documentElement.scrollHeight, document.documentElement.offsetHeight);'
scrollheight = self.driver.execute_script(js)
if verbose > 0:
print(scrollheight)
slices = []
offset = 0
offset_arr=[]
#separate full screen in parts and make printscreens
while offset < scrollheight:
if verbose > 0:
print(offset)
#scroll to size of page
if (scrollheight-offset)<offset:
#if part of screen is the last one, we need to scroll just on rest of page
self.driver.execute_script("window.scrollTo(0, %s);" % (scrollheight-offset))
offset_arr.append(scrollheight-offset)
else:
self.driver.execute_script("window.scrollTo(0, %s);" % offset)
offset_arr.append(offset)
#create image (in Python 3.6 use BytesIO)
img = Image.open(BytesIO(self.driver.get_screenshot_as_png()))
offset += img.size[1]
#append new printscreen to array
slices.append(img)
if verbose > 0:
self.driver.get_screenshot_as_file('screen_%s.jpg' % (offset))
print(scrollheight)
#create image with
screenshot = Image.new('RGB', (slices[0].size[0], scrollheight))
offset = 0
offset2= 0
#now glue all images together
for img in slices:
screenshot.paste(img, (0, offset_arr[offset2]))
offset += img.size[1]
offset2+= 1
screenshot.save('test.png')
Source : https://pypi.org/project/Selenium-Screenshot/
from Screenshot import Screenshot_Clipping
from selenium import webdriver
import time
ob = Screenshot_Clipping.Screenshot()
driver = webdriver.Chrome()
url = "https://www.bbc.com/news/world-asia-china-51108726"
driver.get(url)
time.sleep(1)
img_url = ob.full_Screenshot(driver, save_path=r'.', image_name='Myimage.png')
driver.quit()
For Chrome, it's also possible to use the Chrome DevTools Protocol:
import base64
...
page_rect = browser.driver.execute_cdp_cmd("Page.getLayoutMetrics", {})
screenshot = browser.driver.execute_cdp_cmd(
"Page.captureScreenshot",
{
"format": "png",
"captureBeyondViewport": True,
"clip": {
"width": page_rect["contentSize"]["width"],
"height": page_rect["contentSize"]["height"],
"x": 0,
"y": 0,
"scale": 1
}
})
with open(path, "wb") as file:
file.write(base64.urlsafe_b64decode(screenshot["data"]))
Credits
This works both in headless and non-headless mode.
Full page screenshots are not a part of the W3C spec. However, many web drivers implement their own endpoints to get a real full page screenshot. I found this method using geckodriver to be superior to the injected "screenshot, scroll, stitch" method, and far better than resizing the window in headless mode.
Example:
from selenium import webdriver
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.firefox.options import Options
options = Options()
options.headless = True
service = Service('/your/path/to/geckodriver')
driver = webdriver.Firefox(options=options, service=service)
driver.get('https://www.nytimes.com/')
driver.get_full_page_screenshot_as_file('example.png')
driver.close()
geckodriver (Firefox)
If you're using geckodriver, you can hit these methods:
driver.get_full_page_screenshot_as_file
driver.save_full_page_screenshot
driver.get_full_page_screenshot_as_png
driver.get_full_page_screenshot_as_base64
I've tested and confirmed these to be working on Selenium 4.07. I don't believe these functions are included in Selenium 3.
The best documentation I could find on these is in this merge
chromedriver (Chromium)
It appears that chromedriver has implemented their own full page screenshot functionality:
https://chromium-review.googlesource.com/c/chromium/src/+/2300980
and the Selenium team appears to be aiming for support in Selenium 4:
https://github.com/SeleniumHQ/selenium/issues/8168
My first answer on StackOverflow. I'm a newbie.
The other answers quoted by the fellow expert coders are awesome & I'm not even in the competition. I'd just like to quote the steps taken from the following link: pypi.org
Refer full-page screenshot section.
open your command prompt and navigate to the directory where Python is installed
cd "enter the directory"
install the module using pip
pip install Selenium-Screenshot
The above module works for python 3.
once the module is installed, try the following code by creating a separate file in python IDLE
from Screenshot import Screenshot_Clipping
from selenium import webdriver
ob = Screenshot_Clipping.Screenshot()
driver = webdriver.Chrome()
url = "https://github.com/sam4u3/Selenium_Screenshot/tree/master/test"
driver.get(url)
# the line below makes taking & saving screenshots very easy.
img_url=ob.full_Screenshot(driver, save_path=r'.', image_name='Myimage.png')
print(img_url)
driver.close()
driver.quit()
For Python using Selenium 4 and Chrome Driver
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
import time
import shutil
def take_full_page_screenshot():
#Install chrome driver
chrome_driver_path = ChromeDriverManager().install()
service = Service(chrome_driver_path)
service.start()
#setup chrome options
options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--incognito')
options.add_argument('--start-maximized')
options.add_argument('--disable-gpu')
driver = webdriver.Chrome(chrome_driver_path, options=options)
#open url and wait for the page to load
driver.get('https://www.stackoverflow.com')
time.sleep(2)
#find the element with longest height on page
element = driver.find_element(By.TAG_NAME, 'body')
total_height = element.size["height"]+1000
#set the window dimensions
driver.set_window_size(1920, total_height)
#save screenshot
driver.save_screenshot("screenshot.png")
#quit driver
driver.quit()
if __name__ == '__main__':
take_full_page_screenshot()
element=driver.find_element_by_tag_name('body')
element_png = element.screenshot_as_png
with open("test2.png", "wb") as file:
file.write(element_png)
There was an error in the code suggested earlier in line 2. Here is the corrected one. Being a noob here, not able to edit my own post as yet.
Sometimes the baove doesn't get best results. So can use another method to get height of all elements and sum them to set the capture height as below:
element=driver.find_elements_by_xpath("/html/child::*/child::*")
eheight=set()
for e in element:
eheight.add(round(e.size["height"]))
print (eheight)
total_height = sum(eheight)
driver.execute_script("document.getElementsByTagName('html')[0].setAttribute('style', 'height:"+str(total_height)+"px')")
element=driver.find_element_by_tag_name('body')
element_png = element.screenshot_as_png
with open(fname, "wb") as file:
file.write(element_png)
BTW, it works on FF.
You can use Splinter
Splinter is an abstraction layer on top of existing browser automation tools such as Selenium
There is a new feature browser.screenshot(..., full=True) in new version 0.10.0.
full=True option will make full screen capture for you.
easy by python, but slowly
import os
from selenium import webdriver
from PIL import Image
def full_screenshot(driver: webdriver):
driver.execute_script(f"window.scrollTo({0}, {0})")
total_width = driver.execute_script("return document.body.offsetWidth")
total_height = driver.execute_script("return document.body.parentNode.scrollHeight")
viewport_width = driver.execute_script("return document.body.clientWidth")
viewport_height = driver.execute_script("return window.innerHeight")
rectangles = []
i = 0
while i < total_height:
ii = 0
top_height = i + viewport_height
if top_height > total_height:
top_height = total_height
while ii < total_width:
top_width = ii + viewport_width
if top_width > total_width:
top_width = total_width
rectangles.append((ii, i, top_width, top_height))
ii = ii + viewport_width
i = i + viewport_height
stitched_image = Image.new('RGB', (total_width, total_height))
previous = None
part = 0
for rectangle in rectangles:
if not previous is None:
driver.execute_script("window.scrollTo({0}, {1})".format(rectangle[0], rectangle[1]))
file_name = "part_{0}.png".format(part)
driver.get_screenshot_as_file(file_name)
screenshot = Image.open(file_name)
if rectangle[1] + viewport_height > total_height:
offset = (rectangle[0], total_height - viewport_height)
else:
offset = (rectangle[0], rectangle[1])
stitched_image.paste(screenshot, offset)
del screenshot
os.remove(file_name)
part = part + 1
previous = rectangle
return stitched_image
I'm currently using this approach:
def take_screenshot(self, driver, screenshot_name = "debug.png"):
elem = driver.find_element_by_tag_name('body')
total_height = elem.size["height"] + 1000
driver.set_window_size(1920, total_height)
time.sleep(2)
driver.save_screenshot(screenshot_name)
return driver
If you are trying to do this post ~2021, you need to edit the find element command from:
element = driver.find_element_by_tag('body')
to:
from selenium.webdriver.common.by import By
...
element = driver.find_element(By.TAG_NAME, "body")
Slightly modify #ihightower and #A.Minachev's code, and make it work in mac retina:
import time
from PIL import Image
from io import BytesIO
def fullpage_screenshot(driver, file, scroll_delay=0.3):
device_pixel_ratio = driver.execute_script('return window.devicePixelRatio')
total_height = driver.execute_script('return document.body.parentNode.scrollHeight')
viewport_height = driver.execute_script('return window.innerHeight')
total_width = driver.execute_script('return document.body.offsetWidth')
viewport_width = driver.execute_script("return document.body.clientWidth")
# this implementation assume (viewport_width == total_width)
assert(viewport_width == total_width)
# scroll the page, take screenshots and save screenshots to slices
offset = 0 # height
slices = {}
while offset < total_height:
if offset + viewport_height > total_height:
offset = total_height - viewport_height
driver.execute_script('window.scrollTo({0}, {1})'.format(0, offset))
time.sleep(scroll_delay)
img = Image.open(BytesIO(driver.get_screenshot_as_png()))
slices[offset] = img
offset = offset + viewport_height
# combine image slices
stitched_image = Image.new('RGB', (total_width * device_pixel_ratio, total_height * device_pixel_ratio))
for offset, image in slices.items():
stitched_image.paste(image, (0, offset * device_pixel_ratio))
stitched_image.save(file)
fullpage_screenshot(driver, 'test.png')
I have modified jeremie-s' answer so that it only get the url once.
browser = webdriver.Chrome(chrome_options=options)
browser.set_window_size(default_width, default_height)
browser.get(url)
height = browser.execute_script("return document.body.parentNode.scrollHeight")
# 2. get screenshot
browser.set_window_size(default_width, height)
browser.save_screenshot(screenshot_path)
browser.quit()
Got it!!! works like a charm
For NodeJS, but the concept is the same:
await driver.executeScript(`
document.documentElement.style.display = "table";
document.documentElement.style.width = "100%";
document.body.style.display = "table-row";
`);
await driver.findElement(By.css('body')).takeScreenshot();
I have modified the answer given by #ihightower, instead of saving the screenshot in that function, return the total height and total width of the webpage and then set the window size to total height and total width.
from PIL import Image
from io import BytesIO
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
def open_url(url):
options = Options()
options.headless = True
driver = webdriver.Chrome(chrome_options=options)
driver.maximize_window()
driver.get(url)
save_screenshot(driver, 'screen.png')
def save_screenshot(driver, file_name):
height, width = scroll_down(driver)
driver.set_window_size(width, height)
img_binary = driver.get_screenshot_as_png()
img = Image.open(BytesIO(img_binary))
img.save(file_name)
# print(file_name)
print(" screenshot saved ")
def scroll_down(driver):
total_width = driver.execute_script("return document.body.offsetWidth")
total_height = driver.execute_script("return document.body.parentNode.scrollHeight")
viewport_width = driver.execute_script("return document.body.clientWidth")
viewport_height = driver.execute_script("return window.innerHeight")
rectangles = []
i = 0
while i < total_height:
ii = 0
top_height = i + viewport_height
if top_height > total_height:
top_height = total_height
while ii < total_width:
top_width = ii + viewport_width
if top_width > total_width:
top_width = total_width
rectangles.append((ii, i, top_width, top_height))
ii = ii + viewport_width
i = i + viewport_height
previous = None
part = 0
for rectangle in rectangles:
if not previous is None:
driver.execute_script("window.scrollTo({0}, {1})".format(rectangle[0], rectangle[1]))
time.sleep(0.5)
# time.sleep(0.2)
if rectangle[1] + viewport_height > total_height:
offset = (rectangle[0], total_height - viewport_height)
else:
offset = (rectangle[0], rectangle[1])
previous = rectangle
return (total_height, total_width)
open_url("https://www.medium.com")
This works for me
s = Service("/opt/homebrew/bin/chromedriver")
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--start-maximized')
driver = webdriver.Chrome(chrome_options=chrome_options, service=s)
highest_ele = driver.find_element(By.XPATH, '//*[#id="react-app"]/div[3]/div[3]/span/span/span[2]')
total_height = highest_ele.location['y']
driver.set_window_size(height=total_height, width=1920)
time.sleep(1)
driver.save_screenshot('~/shot.png') # replace your path
Related
I have This code for get screenshot of website with python .
import time
from PIL import Image
from selenium import webdriver
options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.headless = True
driver = webdriver.Chrome(options=options)
driver.set_window_size(1920, 1080)
URL = "https://google.com"
driver.get(URL)
time.sleep(5)
el = driver.save_screenshot('1.png')
driver.quit()
im = Image.open( "1.png")
width, height = im.size
left = 1
top = height / 7
right = 1920
bottom = 1 * height
im1 = im.crop((left, top, right, bottom))
im1 = im1.save("1.png")
it works with Centos 7 command line ( python3 main.py ) and get screenshot of page .
but when i run shell_exec("python3 main.py"); in php , do proccess until line 6 driver = webdriver.Chrome(options=options) and line 6 does not work .
i search a lot and did'nt found any solution .
please help me with that code .
I just solve my problem with this option :
options.add_argument("--crash-dumps-dir=/tmp")
I am working on a little selenium project, but I got some issues.
So what I need to do is to click on a link to open it in a new tab, and whenever I have taken the information I need to close that tab and go into the next one. driver.close() does not work as it gives me the error: Message: no such window: target window already closed. So I intstead tried this (saw this while researching):
driver.find_element_by_tag_name('html').send_keys(Keys.CONTROL + 'w'), and I also tried with adding Keys.F4, but nothing worked.
It seems to work for other people, so why not for me?
Code:
def cpuFunc():
i = 0
print("Launching CPU")
cpu = webdriver.Chrome('chromedriver.exe',options=option)
cpu.get('https://www.komplett.se/category/11204/datorutrustning/datorkomponenter/processor')
cpu.find_element_by_xpath('/html/body/div[1]/div[2]/div[1]/div/div/div[2]/form/div/div[1]/button').click()
#while i < 10:
# cpu.find_element_by_tag_name('html').send_keys(Keys.END)
# i += 1
# time.sleep(0.5)
#print("At bottom: CPU")
cpu.find_element_by_tag_name('body').send_keys(Keys.CONTROL + Keys.HOME)
time.sleep(0.5)
link = cpu.find_element_by_xpath(f'/html/body/main/div/div[2]/div[5]/div[2]/form/div[1]/a')
ActionChains(cpu).key_down(Keys.CONTROL).click(link).key_up(Keys.CONTROL).perform()
time.sleep(1)
window = cpu.window_handles[-1]
cpu.switch_to.window(window)
title = cpu.find_element_by_xpath("/html/body/div[2]/main/div[2]/div[2]/div[3]/section/div/section/div[1]/h1/span").text
price = cpu.find_element_by_xpath("/html/body/div[2]/main/div[2]/div[2]/div[3]/section/div/section/div[3]/div[2]/div[1]/div/div/div[1]/div[1]/div[1]/span").text
btn = cpu.find_element_by_xpath('/html/body/div[2]/main/div[2]/div[2]/div[3]/div/div[2]/div/section[2]/button')
time.sleep(0.5)
cpu.execute_script("arguments[0].click();", btn)
core = cpu.find_element_by_xpath("/html/body/div[2]/main/div[2]/div[2]/div[3]/div/div[2]/div/section[2]/div/div/div/table[2]/tbody/tr[2]/td").text
thread = cpu.find_element_by_xpath("/html/body/div[2]/main/div[2]/div[2]/div[3]/div/div[2]/div/section[2]/div/div/div/table[2]/tbody/tr[3]/td").text
cache = cpu.find_element_by_xpath("/html/body/div[2]/main/div[2]/div[2]/div[3]/div/div[2]/div/section[2]/div/div/div/table[2]/tbody/tr[4]/td").text
clock = cpu.find_element_by_xpath("/html/body/div[2]/main/div[2]/div[2]/div[3]/div/div[2]/div/section[2]/div/div/div/table[2]/tbody/tr[7]/td").text
turbo = cpu.find_element_by_xpath("/html/body/div[2]/main/div[2]/div[2]/div[3]/div/div[2]/div/section[2]/div/div/div/table[2]/tbody/tr[8]/td").text
socket = cpu.find_element_by_xpath("/html/body/div[2]/main/div[2]/div[2]/div[3]/div/div[2]/div/section[2]/div/div/div/table[2]/tbody/tr[9]/td").text
wattage = cpu.find_element_by_xpath("/html/body/div[2]/main/div[2]/div[2]/div[3]/div/div[2]/div/section[2]/div/div/div/table[2]/tbody/tr[10]/td").text
cpu.find_element_by_tag_name('html').send_keys(Keys.CONTROL + 'w') # Here it shall close
time.sleep(60000)
enter code here
You can simply use ActionChains & Keys.
from selenium.webdriver import ActionChains
from selenium.webdriver.common.keys import Keys
If MacOs:
step_1 = ActionChains(cpu)
step_1.send_keys(Keys.COMMAND + 'w')
If Windows:
step_1 = ActionChains(cpu)
step_1.send_keys(Keys.CONTROL + 'w')
Hope it helps, if issues please comment.
Driver.close() worked for me after also fixed a lot of those full xpaths which are easily breakable and added webdriver waits for stability in finding elements.
wait = WebDriverWait(cpu, 10)
cpu.get('https://www.komplett.se/category/11204/datorutrustning/datorkomponenter/processor')
wait.until(EC.element_to_be_clickable((By.XPATH, "//*[#class='btn-large primary'][#type='submit']"))).click()
link = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "form > div:nth-child(1) > a")))
ActionChains(cpu).key_down(Keys.CONTROL).click(link).key_up(Keys.CONTROL).perform()
time.sleep(1)
window = cpu.window_handles[-1]
cpu.switch_to.window(window)
title = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "div.product-main-info__info > h1 > span"))).text
price = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "div.product-price > span"))).text
print(title,price)
btn = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "section.product-section.technical-details.col-xs-12 > button")))
btn.click()
table2 = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "table:nth-child(2) > tbody")))
core = table2.find_element_by_xpath("./tr[2]/td").text
thread = table2.find_element_by_xpath("./tr[3]/td").text
cache = table2.find_element_by_xpath("./tr[4]/td").text
clock = table2.find_element_by_xpath("./tr[7]/td").text
turbo = table2.find_element_by_xpath("./tr[8]/td").text
socket = table2.find_element_by_xpath("./tr[9]/td").text
wattage = table2.find_element_by_xpath("./tr[10]/td").text
cpu.close()
print(core,thread,cache,clock,turbo,socket,wattage)
Import
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
I am attempting to get some data from the EPA website, unfortunately i am not able to capture all of the data points, I theorize this is due to a combination of scrolling and waiting for the tag to become visible. however I have been working on this since yesterday with no luck.
from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from selenium.webdriver.common.keys import Keys
import numpy as np
options = webdriver.ChromeOptions()
path = '/Users/<user>/Applications/chromedriver'
options.set_headless(True)
driver = webdriver.Chrome(chrome_options= options, executable_path=path)
url = 'https://edap.epa.gov/public/single/?appid=73b2b6a5-70c6-4820-b3fa-186ac094f10d&obj=b5bf280c-3488-4e46-84f6-58e2a0c34108&opt=noanimate%2Cnoselections&select=clearall'
driver.set_window_size(1920, 1080)
driver.get(url)
SCROLL_PAUSE_TIME = 0.5
# Get scroll height
last_height = driver.execute_script("return document.body.scrollHeight")
rin_data = []
while True:
# Scroll down to bottom
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CLASS_NAME,"qv-st-value-overflow")))
soup = BeautifulSoup(driver.page_source, "html.parser")
tableURL = soup.select('.qv-st-value-overflow')
for rin_val in tableURL:
rin_data.append(rin_val.get_text())
# Wait to load page
time.sleep(SCROLL_PAUSE_TIME)
# Calculate new scroll height and compare with last scroll height
new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
It use Websocket not Ajax to get the data and you need to scroll the table[ng-style="tableStyles.content"] not body but it need custom scroll or using mouse wheel scroll. The function is taken from here
SCROLL_PAUSE_TIME = 2
driver.get(url)
# add mouse wheel function to the page
driver.execute_script('''
window.scrollTable = function() {
var element = document.querySelector('table[ng-style="tableStyles.content"]')
var box = element.getBoundingClientRect();
var deltaY = box.height;
var clientX = box.left + (box.width / 2);
var clientY = box.top + (box.height / 2);
var target = element.ownerDocument.elementFromPoint(clientX, clientY);
for (var e = target; e; e = e.parentElement) {
if (e === element) {
target.dispatchEvent(new WheelEvent('wheel', {view: window, bubbles: true, cancelable: true, clientX: clientX, clientY: clientY, deltaY: deltaY}));
}
}
}
''')
rin_data = []
while True:
WebDriverWait(driver, 30).until(
EC.presence_of_element_located((By.CSS_SELECTOR, 'tr[class^="qv-st-data-row"]'))
)
last_position = driver.find_element_by_css_selector(".scrollbar-thumb").get_attribute('style')
rows = driver.find_elements_by_css_selector('tr[class^="qv-st-data-row"]')
for row in rows:
rin_data.append(row.text)
# Scroll down the table
driver.execute_script('scrollTable()')
# Wait to load content from Websocket, maybe need to increase
time.sleep(SCROLL_PAUSE_TIME)
# Calculate new scroll position and compare with last scroll position
new_position = driver.find_element_by_css_selector(".scrollbar-thumb").get_attribute('style')
if new_position == last_position:
break
Note, in this case you don't need to use BeautifulSoup
Problem Description
Currently working on a project which requires me to take browse a url and take a screenshot of the webpage.
After looking various resources i found 3 ways to do so.I will be mentioning all 3 methods iam currently using.
Method - 1 : PhantomJS
from selenium import webdriver
import time
import sys
print 'Without Headless'
_start = time.time()
br = webdriver.PhantomJS()
br.get('http://' + sys.argv[1])
br.save_screenshot('screenshot-phantom.png')
br.quit
_end = time.time()
print 'Total time for non-headless {}'.format(_end - _start)
Method-2 : Headless Browser
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
print 'Headless'
_start = time.time()
options = Options()
options.add_argument("--headless") # Runs Chrome in headless mode.
options.add_argument('--no-sandbox') # # Bypass OS security model
options.add_argument('start-maximized')
options.add_argument('disable-infobars')
options.add_argument("--disable-extensions")
driver = webdriver.Chrome(chrome_options=options, executable_path='/usr/bin/chromedriver')
driver.get('http://' + sys.argv[1])
driver.save_screenshot('screenshot-headless.png')
driver.quit()
_end = time.time()
print 'Total time for headless {}'.format(_end - _start)
Method - 3 :PyQT
import argparse
import sys
import logging
import sys
import time
import os
import urlparse
from selenium import webdriver
from PyQt4.QtCore import *
from PyQt4.QtGui import *
from PyQt4.QtWebKit import *
class Screenshot(QWebView):
def __init__(self):
self.app = QApplication(sys.argv)
QWebView.__init__(self)
self._loaded = False
self.loadFinished.connect(self._loadFinished)
def capture(self, url, output_file):
_logger.info('Received url {}'.format(url))
_start = time.time()
try:
#Check for http/https
if url[0:3] == 'http' or url[0:4] == 'https':
self.url = url
else:
url = 'http://' + url
self.load(QUrl(url))
self.wait_load(url)
# set to webpage size
frame = self.page().mainFrame()
self.page().setViewportSize(frame.contentsSize())
# render image
image = QImage(self.page().viewportSize(), QImage.Format_ARGB32)
painter = QPainter(image)
frame.render(painter)
painter.end()
_logger.info('Saving screenshot {} for {}'.format(output_file,url))
image.save(os.path.join(os.path.dirname(os.path.realpath(__file__)),'data',output_file))
except Exception as e:
_logger.error('Error in capturing screenshot {} - {}'.format(url,e))
_end = time.time()
_logger.info('Time took for processing url {} - {}'.format(url,_end - _start))
def wait_load(self,url,delay=1,retry_count=60):
# process app events until page loaded
while not self._loaded and retry_count:
_logger.info('wait_load for url {} retry_count {}'.format(url,retry_count))
self.app.processEvents()
time.sleep(delay)
retry_count -=1
_logger.info('wait_load for url {} expired'.format(url))
self._loaded = False
def _loadFinished(self, result):
self._loaded = True
Issue Faced:
These 3 methods while using,all of them are getting stuck due to one or other error.One such issue faced is asked here Error Question on Stackoverflow.
So out of these 3 methods to take screenshot of a webpage in Python,which is effecient and will work on large scale deployment.
Taken from https://gist.github.com/fabtho/13e4a2e7cfbfde671b8fa81bbe9359fb and rewritten in Python 3
This method will technically work but it will not look good, as many websites will have cookie acceptance pop-ups that will appear in every screenshot, so depending on which website you use, you may wish to remove these first using selenium before beginning the screenshotting process.
from PIL import Image
from io import BytesIO
verbose = 1
browser = webdriver.Chrome(executable_path='C:/yourpath/chromedriver.exe')
browser.get('http://stackoverflow.com/questions/37906704/taking-a-whole-page-screenshot-with-selenium-marionette-in-python')
# from here http://stackoverflow.com/questions/1145850/how-to-get-height-of-entire-document-with-javascript
js = 'return Math.max( document.body.scrollHeight, document.body.offsetHeight, document.documentElement.clientHeight, document.documentElement.scrollHeight, document.documentElement.offsetHeight);'
scrollheight = browser.execute_script(js)
if verbose > 0:
print(scrollheight)
slices = []
offset = 0
while offset < scrollheight:
if verbose > 0:
print(offset)
browser.execute_script("window.scrollTo(0, %s);" % offset)
img = Image.open(BytesIO(browser.get_screenshot_as_png()))
offset += img.size[1]
slices.append(img)
if verbose > 0:
browser.get_screenshot_as_file('%s/screen_%s.png' % ('/tmp', offset))
print(scrollheight)
screenshot = Image.new('RGB', (slices[0].size[0], offset))
offset = 0
for img in slices:
screenshot.paste(img, (0, offset))
offset += img.size[1]
screenshot.save('screenshot.png')
browser.quit()```
I've written a script in python with selenium to handle the infinite scrolling webpage. The problem I'm facing is that It scrolls few times then quits the browser. It never reaches the bottom. I tried with Explicit Wait as well but that gives even fewer scrolling. How can I reach the bottom when there will be no more scrolling to do.
This is my try:
import time
from selenium import webdriver
from urllib.parse import urljoin
url = "https://www.instagram.com/explore/tags/travelphotoawards/"
driver = webdriver.Chrome()
driver.get(url)
last_len = len(driver.find_elements_by_css_selector(".v1Nh3 a"))
new_len = last_len
while True:
last_len = new_len
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(5)
items = driver.find_elements_by_css_selector(".v1Nh3 a")
new_len = len(items)
if last_len == new_len:break
driver.quit()
Edit:
If I try like below, I can do the scrolling as many times as I want but that is not a good idea to cope with:
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
url = "https://www.instagram.com/explore/tags/travelphotoawards/"
driver = webdriver.Chrome()
driver.get(url)
for scroll in range(1,10): #I can do the scrolling as many times as I want but it is fully hardcoded
item = driver.find_element_by_tag_name("body")
item.send_keys(Keys.END)
elems = driver.find_elements_by_css_selector(".v1Nh3 a")
time.sleep(3)
driver.quit()
I hope there is any way to do the scrolling automatically until it reaches the bottom.
So few thing here. In case of infinite scrolling I would follow few things
Disable images so that the scrolling is faster
Never trust a condition to be true if it is not consistent. Test it for it continuously for a period and if the condition is consistent then trust it
Try to not scroll way too long, infinite scrolling can cause browser to clog too much memory and sometimes even crash
Dump data in batches after every scroll. So on first page load, I would dump all page date. Then every scroll, I would just dump the delta part. This can be easily done using an xpath.
Below is a updated script which will do better for you. Do remember nothing is perfect, so you need to make your script adapt to failures
import time
from selenium import webdriver
from urllib.parse import urljoin
option = webdriver.ChromeOptions()
chrome_prefs = {}
option.experimental_options["prefs"] = chrome_prefs
chrome_prefs["profile.default_content_settings"] = {"images": 2}
chrome_prefs["profile.managed_default_content_settings"] = {"images": 2}
driver = webdriver.Chrome(chrome_options=option)
url = "https://www.instagram.com/explore/tags/travelphotoawards/"
driver.get(url)
last_len = len(driver.find_elements_by_css_selector(".v1Nh3 a"))
new_len = last_len
consistent = 0
while True:
last_len = new_len
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(5)
items = driver.find_elements_by_css_selector(".v1Nh3 a")
new_len = len(items)
if last_len == new_len:
consistent += 1
if consistent == 3:
break
else:
consistent = 0
driver.quit()
Every time there is a scroll older images disappear. You might get the same number or even smaller number of images after the scroll.
Each image has unique href, you can compare the last image href to the previous last image
last_href = driver.find_elements_by_css_selector('.v1Nh3 > a')[-1].get_attribute('href')
new_href = last_href
while True:
last_href = new_href
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(5)
new_href = driver.find_elements_by_css_selector('.v1Nh3 > a')[-1].get_attribute('href')
if last_href != new_href:
break