Develop Reference

bbox works when inputting exact coordinates within module but fails with variable containing identical data

I'm having an issue trying to make a screen grab of an area defined by lines in a config file:
The following code:
def coordstore(): # check line 1 of config file (screencap name)
f=open('config.txt')
line=f.readlines()
coordstore.x1,coordstore.y1,coordstore.x2,coordstore.y2 = (line[4]).strip(),(line[5]).strip(),(line[6]).strip(),(line[7]).strip() # note: confusing. 0=line 1, 1=line2 etc.
coordstore.coords = coordstore.x1+',',coordstore.y1+',',coordstore.x2+',',coordstore.y2
coordstore.stripped = ' '.join((coordstore.coords))
print(coordstore.stripped)
return(coordstore.stripped)
coordstore()
def screenshot():
import pyscreenshot as ImageGrab2
# part of the screen
if __name__ == '__main__':
im = ImageGrab2.grab(bbox=(coordstore.stripped)) # X1,Y1,X2,Y2
im.save('tc.png')
screenshot()
prints exactly this value: 10, 20, 100, 300
it then fails with this Traceback:
Traceback (most recent call last):
File "file location", line 82, in <module>
screenshot()
File "file location", line 80, in screenshot
im = ImageGrab2.grab(bbox=(coordstore.stripped)) # X1,Y1,X2,Y2
File "PYCHARM\venv\lib\site-packages\pyscreenshot\__init__.py", line 67, in grab
to_file=False, childprocess=childprocess, backend=backend, bbox=bbox)
File "PYCHARM\venv\lib\site-packages\pyscreenshot\__init__.py", line 38, in _grab
x1, y1, x2, y2 = bbox
ValueError: too many values to unpack (expected 4)
If I manually replace (bbox=(coordstore.stripped)) with (bbox=(10, 20, 100, 300)) which is literally identical to what the variable prints, the code works perfectly but is not useful for my needs. What am I not seeing? Thanks for the help!
UPDATE:
I have tried another approach.
def coordstore(): # check line 1 of config file (screencap name)
f=open('config.txt')
line=f.readlines()
coordstore.x1 = (line[4]).strip()
coordstore.y1 = (line[5]).strip()
coordstore.x2 = (line[6]).strip()
coordstore.y2 = (line[7]).strip()
print(coordstore.x1+',', coordstore.y1+',', coordstore.x2+',', coordstore.y2)
return(coordstore.x1, coordstore.y1, coordstore.x2, coordstore.y2)
coordstore()
def screenshot():
import pyscreenshot as ImageGrab2
# part of the screen
if __name__ == '__main__':
im = ImageGrab2.grab(bbox=(coordstore.x1+',', coordstore.y1+',', coordstore.x2+',', coordstore.y2+',')) # X1,Y1,X2,Y2
im.save('tc.png')
screenshot()
This prints
10, 20, 100, 300
but returns the following Traceback errors:
Traceback (most recent call last):
File "module location", line 84, in <module>
screenshot()
File "module location", line 82, in screenshot
im = ImageGrab2.grab(bbox=(coordstore.x1+',', coordstore.y1+',', coordstore.x2+',', coordstore.y2+',')) # X1,Y1,X2,Y2
File "\PYCHARM\venv\lib\site-packages\pyscreenshot\__init__.py", line 67, in grab
to_file=False, childprocess=childprocess, backend=backend, bbox=bbox)
File "\PYCHARM\venv\lib\site-packages\pyscreenshot\__init__.py", line 42, in _grab
raise ValueError('bbox y2<=y1')
ValueError: bbox y2<=y1

As you have mentioned in comments the value of coordstore.stripped is string. And expected argument by ImageGrab2.grab is type of tuple so, first you have to convert it into tuple.
Update the method like this:
def screenshot():
import pyscreenshot as ImageGrab2
# part of the screen
if __name__ == '__main__':
im = ImageGrab2.grab(bbox=tuple(map(int, coordstore.stripped.split(', ')))) # X1,Y1,X2,Y2
im.save('tc.png')

Scrapy: Item Loader and KeyError even when Key is defined

Intention / expected behaviour
Return the text of the links from page: https://www.bezrealitky.cz/vypis/nabidka-prodej/byt/praha
In CSV format and in the shell.
Error
I get a KeyError: 'title', even though I have defined the key in the item.py itemloader.
Full Traceback
Traceback (most recent call last):
File "C:\Users\phili\Anaconda3\envs\py35\lib\site-packages\scrapy\utils\defer.py", line 102, in iter_errback
yield next(it)
File "C:\Users\phili\Anaconda3\envs\py35\lib\site-packages\scrapy\spidermiddlewares\offsite.py", line 29, in process_spider_output
for x in result:
File "C:\Users\phili\Anaconda3\envs\py35\lib\site-packages\scrapy\spidermiddlewares\referer.py", line 22, in <genexpr>
return (_set_referer(r) for r in result or ())
File "C:\Users\phili\Anaconda3\envs\py35\lib\site-packages\scrapy\spidermiddlewares\urllength.py", line 37, in <genexpr>
return (r for r in result or () if _filter(r))
File "C:\Users\phili\Anaconda3\envs\py35\lib\site-packages\scrapy\spidermiddlewares\depth.py", line 58, in <genexpr>
return (r for r in result or () if _filter(r))
File "C:\Users\phili\Documents\Python Scripts\Scrapy Spiders\bezrealitky\bezrealitky\spiders\bezrealitky_spider.py", line 33, in parse
yield loader.load_item()
File "C:\Users\phili\Anaconda3\envs\py35\lib\site-packages\scrapy\loader\__init__.py", line 115, in load_item
value = self.get_output_value(field_name)
File "C:\Users\phili\Anaconda3\envs\py35\lib\site-packages\scrapy\loader\__init__.py", line 122, in get_output_value
proc = self.get_output_processor(field_name)
File "C:\Users\phili\Anaconda3\envs\py35\lib\site-packages\scrapy\loader\__init__.py", line 144, in get_output_processor
self.default_output_processor)
File "C:\Users\phili\Anaconda3\envs\py35\lib\site-packages\scrapy\loader\__init__.py", line 154, in _get_item_field_attr
value = self.item.fields[field_name].get(key, default)
KeyError: 'title'
Spider.py
def parse(self, response):
for records in response.xpath('//*[starts-with(#class,"record")]'):
loader = BaseItemLoader(selector=records)
loader.add_xpath('title', './/div[#class="details"]/h2/a[#href]/text()')
yield loader.load_item()
Item.py - Itemloader
class BaseItemLoader(ItemLoader):
title_in = MapCompose(unidecode)
Conclusion
I am a bit at a loss, as I think I followed the Scrapy manual and defined the item loader and the key by "title_in", but then when I yield the value to it I get the KeyError. I check in the shell that the Xpath provides the text I want, so at least that is working. Hoping to get some help!

Even if you use ItemLoader you should define Item class first and then pass it to the item loader either defining it as loader's property:
class CustomItemLoader(ItemLoader):
default_item_class = MyItem
or passing its instance to loader's constructor:
l = CustomItemLoader(item=Item())
otherwise item loader knows nothing about the item and its fields.

Usage of python-readability

(https://github.com/buriy/python-readability)
I am struggling using this library and I can't find any documentation for it. (Is there any?)
There are some kind of useable pieces calling help(Document) but there is still something wrong.
My code so far:
from readability.readability import Document
import requests
url = 'http://www.somepage.com'
html = requests.get(url, verify=False).content
readable_article = Document(html, negative_keywords='test_keyword').summary()
with open('test.html', 'w', encoding='utf-8') as test_file:
test_file.write(readable_article)
According to the help(Document) output, it should be possible to use a list for the input of the negative_keywords.
readable_article = Document(html, negative_keywords=['test_keyword1', 'test-keyword2').summary()
Gives me a bunch of errors I don't understand:
Traceback (most recent call last): File
"/usr/lib/python3.4/site-packages/readability/readability.py", line
163, in summary
candidates = self.score_paragraphs() File "/usr/lib/python3.4/site-packages/readability/readability.py", line
300, in score_paragraphs
candidates[parent_node] = self.score_node(parent_node) File "/usr/lib/python3.4/site-packages/readability/readability.py", line
360, in score_node
content_score = self.class_weight(elem) File "/usr/lib/python3.4/site-packages/readability/readability.py", line
348, in class_weight
if self.negative_keywords and self.negative_keywords.search(feature): AttributeError: 'list' object
has no attribute 'search' Traceback (most recent call last): File
"/usr/lib/python3.4/site-packages/readability/readability.py", line
163, in summary
candidates = self.score_paragraphs() File "/usr/lib/python3.4/site-packages/readability/readability.py", line
300, in score_paragraphs
candidates[parent_node] = self.score_node(parent_node) File "/usr/lib/python3.4/site-packages/readability/readability.py", line
360, in score_node
content_score = self.class_weight(elem) File "/usr/lib/python3.4/site-packages/readability/readability.py", line
348, in class_weight
if self.negative_keywords and self.negative_keywords.search(feature): AttributeError: 'list' object
has no attribute 'search'
Could some one give me please a hint on the error or how to deal with it?

There's an error in the library code. If you look at compile_pattern:
def compile_pattern(elements):
if not elements:
return None
elif isinstance(elements, (list, tuple)):
return list(elements)
elif isinstance(elements, regexp_type):
return elements
else:
# assume string or string like object
elements = elements.split(',')
return re.compile(u'|'.join([re.escape(x.lower()) for x in elements]), re.U)
You can see that it only returns a regex if the elements is not None, not a list or tuple, and not a regular expression.
Later on, though, it assumes that self.negative_keywords is a regular expression. So, I suggest you input your list as a string in the form of "test_keyword1,test_keyword2". This will make sure that compile_pattern returns a regular expression which should fix the error.

Handling exceptions in while loop - Python

Here is my code (almost full version for #cdhowie :)):
def getResult(method, argument=None):
result = None
while True:
print('### loop')
try:
print ('### try hard...')
if argument:
result = method(argument)
else:
result = method()
break
except Exception as e:
print('### GithubException')
if 403 == e.status:
print('Warning: ' + str(e.data))
print('I will try again after 10 minutes...')
else:
raise e
return result
def getUsernames(locations, gh):
usernames = set()
for location in locations:
print location
result = getResult(gh.legacy_search_users, location)
for user in result:
usernames.add(user.login)
print user.login,
return usernames
# "main.py"
gh = Github()
locations = ['Washington', 'Berlin']
# "main.py", line 12 is bellow
usernames = getUsernames(locations, gh)
The problem is, that exception is raised, but I can't handle it. Here is an output:
### loop
### try hard...
Traceback (most recent call last):
File "main.py", line 12, in <module>
usernames = getUsernames(locations, gh)
File "/home/ciembor/projekty/github-rank/functions.py", line 39, in getUsernames
for user in result:
File "/usr/lib/python2.7/site-packages/PyGithub-1.8.0-py2.7.egg/github/PaginatedList.py", line 33, in __iter__
newElements = self.__grow()
File "/usr/lib/python2.7/site-packages/PyGithub-1.8.0-py2.7.egg/github/PaginatedList.py", line 45, in __grow
newElements = self._fetchNextPage()
File "/usr/lib/python2.7/site-packages/PyGithub-1.8.0-py2.7.egg/github/Legacy.py", line 37, in _fetchNextPage
return self.get_page(page)
File "/usr/lib/python2.7/site-packages/PyGithub-1.8.0-py2.7.egg/github/Legacy.py", line 48, in get_page
None
File "/usr/lib/python2.7/site-packages/PyGithub-1.8.0-py2.7.egg/github/Requester.py", line 69, in requestAndCheck
raise GithubException.GithubException(status, output)
github.GithubException.GithubException: 403 {u'message': u'API Rate Limit Exceeded for 11.11.11.11'}
Why it doesn't print ### handling exception?

Take a close look at the stack trace in the exception:
Traceback (most recent call last):
File "main.py", line 12, in <module>
usernames = getUsernames(locations, gh)
File "/home/ciembor/projekty/github-rank/functions.py", line 39, in getUsernames
for user in result:
File "/usr/lib/python2.7/site-packages/PyGithub-1.8.0-py2.7.egg/github/PaginatedList.py", line 33, in __iter__
newElements = self.__grow()
...
The exception is being thrown from code being called by the line for user in result: after getResult finishes executing. This means that the API you're using is using lazy evaluation, so the actual API request doesn't quite happen when you expect it to.
In order to catch and handle this exception, you'll need to wrap the code inside the getUsernames function with a try/except handler.

Python TypeErrors: "' list' object is not callable" and "'function' object is unsubscriptable"

I have the following code:
from random import randint,choice
add=lambda x:lambda y:x+y
sub=lambda x:lambda y:x-y
mul=lambda x:lambda y:x*y
ops=[[add,'+'],[sub,'-'],[mul,'*']]
def gen_expression(length,left,right):
expr=[]
for i in range(length):
op=choice(ops)
expr.append([op[0](randint(left,right)),op[1]])
return expr
def eval_expression (expr,x):
for i in expr:
x=i[0](x)
return x
def eval_expression2 (expr,x):
for i in expr:
x=i(x)
return x
[snip , see end of post]
def genetic_arithmetic(get,start,length,left,right):
batch=[]
found = False
for i in range(30):
batch.append(gen_expression(length,left,right))
while not found:
batch=sorted(batch,key=lambda y:abs(eval_expression(y,start)-get))
print evald_expression_tostring(batch[0],start)+"\n\n"
#combine
for w in range(len(batch)):
rind=choice(range(length))
batch.append(batch[w][:rind]+choice(batch)[rind:])
#mutate
for w in range(len(batch)):
rind=choice(range(length))
op=choice(ops)
batch.append(batch[w][:rind]+[op[0](randint(left,right)),op[1]]+batch[w][rind+1:])
found=(eval_expression(batch[0],start)==get)
print "\n\n"+evald_expression_tostring(batch[0],start)
When I try to call to call genetic_artihmetic with eval_expression as the sorting key, I get this:
Traceback (most recent call last):
File "<pyshell#113>", line 1, in <module>
genetic_arithmetic(0,10,10,-10,10)
File "/home/benikis/graming/py/genetic_number.py", line 50, in genetic_arithmetic
batch=sorted(batch,key=lambda y:abs(eval_expression(y,start)-get))
File "/home/benikis/graming/py/genetic_number.py", line 50, in <lambda>
batch=sorted(batch,key=lambda y:abs(eval_expression(y,start)-get))
File "/home/benikis/graming/py/genetic_number.py", line 20, in eval_expression
x=i[0](x)
TypeError: 'function' object is unsubscriptable
And when I try the same with eval_expression2 as the sorting,the error is this:
Traceback (most recent call last):
File "<pyshell#114>", line 1, in <module>
genetic_arithmetic(0,10,10,-10,10)
File "/home/benikis/graming/py/genetic_number.py", line 50, in genetic_arithmetic
batch=sorted(batch,key=lambda y:abs(eval_expression2(y,start)-get))
File "/home/benikis/graming/py/genetic_number.py", line 50, in <lambda>
batch=sorted(batch,key=lambda y:abs(eval_expression2(y,start)-get))
File "/home/benikis/graming/py/genetic_number.py", line 25, in eval_expression2
x=i(x)
TypeError: 'list' object is not callable
As far as i can wrap my head around this, my guess is that sorted() is trying to recursively sort the sublists,maybe? What is really going on here?
Python version is 2.6 - the one in the debian stable repos.
[snip] here:
def expression_tostring(expr):
expr_str=len(expr)*'('+'x '
for i in expr :
if i[1]=='*':
n=i[0](1)
else:
n=i[0](0)
expr_str+=i[1]+' '+str(n)+') '
return expr_str
def evald_expression_tostring(expr,x):
exprstr=expression_tostring(expr).replace('x',str(x))
return exprstr+ ' = ' + str(eval_expression(expr,x))

x=i[0](x) #here i is a function so you can't perform indexing operation on it
x=i(x) #here i is a list so you can't call it as a function
in both cases the value of i is fetched from expr, may be expr contains different type of object than what you're assuming here.

Try this modification:
def gen_expression(length,left,right):
expr=[]
for i in range(length):
op=choice(ops)
expr.append([op[0], randint(left,right),op[1]])
return expr
def eval_expression (expr,x):
for i in expr:
x=i[0](i[1])
return x
You had expr.append([op[0](randint(left,right)),op[1]]) which will put the return value of the calling the function into the 0th index.