Scraping list from webpage - python

I am trying to extract items in 'Symbol' column for this webpage:
https://chartink.com/screener/2-short-trend
My code is like this:
from requests_html import HTMLSession
def stockList(url):
session = HTMLSession()
r = session.get(url)
r.html.render(sleep=1)
stock = [{item.text} for item in r.html.xpath('//*[#class="//*[#id="DataTables_Table_0"]/tbody/tr[1]/td[3]/a"]')]
return stock
listStock = stockList('https://chartink.com/screener/2-short-trend')
print(listStock)
Error:
Traceback (most recent call last):
File "C:\Users\kashk\PycharmProjects\test\venv\Scripts\CIscreen2pcShort.py", line 12, in <module>
listStock = stockList('https://chartink.com/screener/2-short-trend')
File "C:\Users\kashk\PycharmProjects\test\venv\Scripts\CIscreen2pcShort.py", line 10, in stockList
stock = [{item.text} for item in r.html.xpath('//*[#class="//*[#id="DataTables_Table_0"]/tbody/tr[1]/td[3]/a"]')]
File "C:\Users\kashk\PycharmProjects\test\venv\lib\site-packages\requests_html.py", line 255, in xpath
selected = self.lxml.xpath(selector)
File "src\lxml\etree.pyx", line 1597, in lxml.etree._Element.xpath
File "src\lxml\xpath.pxi", line 305, in lxml.etree.XPathElementEvaluator.__call__
File "src\lxml\xpath.pxi", line 225, in lxml.etree._XPathEvaluatorBase._handle_result
lxml.etree.XPathEvalError: Invalid predicate
Can you pls suggest where am I going wrong.
Also, is there a way to send this output to a dataframe.

File "src\lxml\etree.pyx", line 1597, in lxml.etree._Element.xpath
File "src\lxml\xpath.pxi", line 305, in lxml.etree.XPathElementEvaluator.call
File "src\lxml\xpath.pxi", line 225, in lxml.etree._XPathEvaluatorBase._handle_result
lxml.etree.XPathEvalError: Invalid expression

Related

dom.getElementsByTagNameNS("urn:oasis:names:tc:SAML:1.0:assertion", 'Assertion')[0].toxml() IndexError: list index out of range

We have used "Office365-REST-Python-Client 2.3.11" library to upload file on sharepoint.
Suddenly code stopped working and started giving below issue.
code:
import json
from office365.runtime.auth.user_credential import UserCredential
from office365.runtime.http.request_options import RequestOptions
from office365.sharepoint.client_context import ClientContext
site_url = "https://{your-tenant-prefix}.sharepoint.com"
ctx = ClientContext(site_url).with_credentials(UserCredential("{username}", "{password}"))
request = RequestOptions("{0}/_api/web/".format(site_url))
response = ctx.execute_request_direct(request)
json = json.loads(response.content)
web_title = json['d']['Title']
print("Web title: {0}".format(web_title))
Error:
Traceback (most recent call last):
File "test_upload.py", line 81, in <module>
response = ctx.execute_request_direct(request)
File "/home/ubuntu/atlas/national_kpi_table/venv/lib/python3.8/site-packages/office365/runtime/client_runtime_context.py", line 131, in execute_request_direct
return self.pending_request().execute_request_direct(self._normalize_request(request))
File "/home/ubuntu/atlas/national_kpi_table/venv/lib/python3.8/site-packages/office365/runtime/odata/odata_request.py", line 36, in execute_request_direct
return super(ODataRequest, self).execute_request_direct(request)
File "/home/ubuntu/atlas/national_kpi_table/venv/lib/python3.8/site-packages/office365/runtime/client_request.py", line 91, in execute_request_direct
self.context.authenticate_request(request)
File "/home/ubuntu/atlas/national_kpi_table/venv/lib/python3.8/site-packages/office365/sharepoint/client_context.py", line 230, in authenticate_request
self._auth_context.authenticate_request(request)
File "/home/ubuntu/atlas/national_kpi_table/venv/lib/python3.8/site-packages/office365/runtime/auth/authentication_context.py", line 89, in authenticate_request
self._provider.authenticate_request(request)
File "/home/ubuntu/atlas/national_kpi_table/venv/lib/python3.8/site-packages/office365/runtime/auth/providers/saml_token_provider.py", line 77, in authenticate_request
self.ensure_authentication_cookie()
File "/home/ubuntu/atlas/national_kpi_table/venv/lib/python3.8/site-packages/office365/runtime/auth/providers/saml_token_provider.py", line 84, in ensure_authentication_cookie
self._cached_auth_cookies = self.get_authentication_cookie()
File "/home/ubuntu/atlas/national_kpi_table/venv/lib/python3.8/site-packages/office365/runtime/auth/providers/saml_token_provider.py", line 97, in get_authentication_cookie
token = self._acquire_service_token_from_adfs(user_realm.STSAuthUrl)
File "/home/ubuntu/atlas/national_kpi_table/venv/lib/python3.8/site-packages/office365/runtime/auth/providers/saml_token_provider.py", line 140, in _acquire_service_token_from_adfs
assertion_node = dom.getElementsByTagNameNS("urn:oasis:names:tc:SAML:1.0:assertion", 'Assertion')[0].toxml()
IndexError: list index out of range
Any help appreciated.

Change the wrapper object to a different class in pywinauto

I'm trying to set a value on this dropdown but am having trouble doing so. I've tried using the .select(index) method and that doesn't do anything. I've tried doing .typekeys("{DOWN 2}"), which usually works, but I think because another dropdown is selected before hand or something, it's not working. My usual workaround for this is to do .expand() and then .type_keys("{DOWN 2}"), and then .type_keys("{ENTER}"). I can't do this last workaround, because the control is not being wrapped as a combobox, as it should be.
Is there a way I can change it's wrapper? I tried:
from pywinauto import controls
test = controls.uia_controls.ComboBoxWrapper(formJobStock.child_window(auto_id='cboStockSize'))
test.expand()
but I get:
Traceback (most recent call last):
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\application.py", line 250, in __resolve_control
ctrl = wait_until_passes(
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\timings.py", line 458, in wait_until_passes
raise err
pywinauto.timings.TimeoutError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\controls\uia_controls.py", line 143, in expand
if self.is_expanded():
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\controls\uiawrapper.py", line 561, in is_expanded
state = self.get_expand_state()
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\controls\uia_controls.py", line 188, in get_expand_state
return super(ComboBoxWrapper, self).get_expand_state()
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\controls\uiawrapper.py", line 556, in get_expand_state
return self.iface_expand_collapse.CurrentExpandCollapseState
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\controls\uiawrapper.py", line 132, in __get__
value = self.fget(obj)
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\controls\uiawrapper.py", line 210, in iface_expand_collapse
return uia_defs.get_elem_interface(elem, "ExpandCollapse")
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\uia_defines.py", line 233, in get_elem_interface
cur_ptrn = element_info.GetCurrentPattern(ptrn_id)
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\application.py", line 379, in __getattribute__
ctrls = self.__resolve_control(self.criteria)
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\application.py", line 261, in __resolve_control
raise e.original_exception
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\timings.py", line 436, in wait_until_passes
func_val = func(*args, **kwargs)
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\application.py", line 222, in __get_ctrl
ctrl = self.backend.generic_wrapper_class(findwindows.find_element(**ctrl_criteria))
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\findwindows.py", line 84, in find_element
elements = find_elements(**kwargs)
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\findwindows.py", line 305, in find_elements
elements = findbestmatch.find_best_control_matches(best_match, wrapped_elems)
File "C:\Users\mflanagan\AppData\Local\Programs\Python\Python39\lib\site-packages\pywinauto\findbestmatch.py", line 536, in find_best_control_matches
raise MatchError(items = name_control_map.keys(), tofind = search_text)
pywinauto.findbestmatch.MatchError: Could not find 'element' in 'dict_keys(['Edit', 'Edit0', 'Edit1', 'Edit2', 'Open', 'Button', 'OpenButton'])'
where formJobStock is a window specification:
formJobStock = window.child_window(auto_id='frmJobStock')
and I know it's found because I've done formJobStock.wrapper_object() and it comes up correctly.
It looks like I'm not passing in the element parameter correctly to the controls.controls_uia.ComboBoxWrapper class init call...any idea on how to do that correctly?

jinja2.exceptions.UndefinedError: 'loop' is undefined odoo 13

I am new to odoo, while generating an invoice an error occurred like Failed to render template %r using values %r. And I have added the full trace of the error below.
Traceback (most recent call last):
File "/opt/odoo13/addons/payment/controllers/portal.py", line 104, in payment_status_poll
tx_to_process._post_process_after_done()
File "/opt/odoo13/addons/payment/models/payment_acquirer.py", line 872, in _post_process_after_done
self._reconcile_after_transaction_done()
File "/opt/odoo13/addons/sale/models/payment.py", line 112, in _reconcile_after_transaction_done
sales_orders._send_order_confirmation_mail()
File "/opt/odoo13/addons/sale/models/sale.py", line 817, in _send_order_confirmation_mail
order.with_context(force_send=True).message_post_with_template(template_id, composition_mode='comment', email_layout_xmlid="mail.mail_notification_paynow")
File "/opt/odoo13/addons/mass_mailing/models/mail_thread.py", line 37, in message_post_with_template
return super(MailThread, no_massmail).message_post_with_template(template_id, **kwargs)
File "/opt/odoo13/addons/mail/models/mail_thread.py", line 1998, in message_post_with_template
update_values = composer.onchange_template_id(template_id, kwargs['composition_mode'], self._name, res_id)['value']
File "/opt/odoo13/addons/mail/wizard/mail_compose_message.py", line 405, in onchange_template_id
values = self.generate_email_for_composer(template_id, [res_id])[res_id]
File "/opt/odoo13/addons/mail/wizard/mail_compose_message.py", line 538, in generate_email_for_composer
template_values = self.env['mail.template'].with_context(tpl_partners_only=True).browse(template_id).generate_email(res_ids, fields=fields)
File "/opt/odoo13/addons/mail/models/mail_template.py", line 401, in generate_email
generated_field_values = Template._render_template(
File "/opt/odoo13/addons/mail/models/mail_template.py", line 290, in _render_template
raise UserError(_("Failed to render template %r using values %r") % (template, variables)

Sending raw transaction from web3py: TypeError: <lambda>() missing 4 required positional arguments: 'hash', 'r', 's', and 'v'

I am trying to send raw transaction by web3py using this code:
t = w3.eth.account.sign_transaction(test_contract.functions.edit("test").buildTransaction(
{
"nonce": w3.eth.get_transaction_count(w3.eth.default_account)
}
), pkey)
w3.eth.send_raw_transaction(t)
But, where python comes to the last line, I have this error in console:
Traceback (most recent call last):
File "***/main.py", line 64, in <module>
w3.eth.send_raw_transaction(t)
File "***/venv/lib/python3.9/site-packages/web3/module.py", line 53, in caller
(method_str, params), response_formatters = method.process_params(module, *args, **kwargs) # noqa: E501
File "***/venv/lib/python3.9/site-packages/web3/method.py", line 194, in process_params
_apply_request_formatters(params, self.request_formatters(method)))
File "***/venv/lib/python3.9/site-packages/eth_utils/functional.py", line 45, in inner
return callback(fn(*args, **kwargs))
File "***/venv/lib/python3.9/site-packages/web3/method.py", line 50, in _apply_request_formatters
formatted_params = pipe(params, request_formatters)
File "cytoolz/functoolz.pyx", line 667, in cytoolz.functoolz.pipe
File "cytoolz/functoolz.pyx", line 642, in cytoolz.functoolz.c_pipe
File "cytoolz/functoolz.pyx", line 254, in cytoolz.functoolz.curry.__call__
File "cytoolz/functoolz.pyx", line 250, in cytoolz.functoolz.curry.__call__
File "***/venv/lib/python3.9/site-packages/web3/_utils/abi.py", line 799, in map_abi_data
return pipe(data, *pipeline)
File "cytoolz/functoolz.pyx", line 667, in cytoolz.functoolz.pipe
File "cytoolz/functoolz.pyx", line 642, in cytoolz.functoolz.c_pipe
File "cytoolz/functoolz.pyx", line 254, in cytoolz.functoolz.curry.__call__
File "cytoolz/functoolz.pyx", line 250, in cytoolz.functoolz.curry.__call__
File "***/venv/lib/python3.9/site-packages/web3/_utils/abi.py", line 833, in data_tree_map
return recursive_map(map_to_typed_data, data_tree)
File "***/venv/lib/python3.9/site-packages/web3/_utils/decorators.py", line 30, in wrapped
wrapped_val = to_wrap(*args)
File "***/venv/lib/python3.9/site-packages/web3/_utils/formatters.py", line 89, in recursive_map
items_mapped = map_collection(recurse, data)
File "***/venv/lib/python3.9/site-packages/web3/_utils/formatters.py", line 76, in map_collection
return datatype(map(func, collection))
File "***/venv/lib/python3.9/site-packages/web3/_utils/formatters.py", line 88, in recurse
return recursive_map(func, item)
File "***/venv/lib/python3.9/site-packages/web3/_utils/decorators.py", line 30, in wrapped
wrapped_val = to_wrap(*args)
File "***/venv/lib/python3.9/site-packages/web3/_utils/formatters.py", line 89, in recursive_map
items_mapped = map_collection(recurse, data)
File "***/venv/lib/python3.9/site-packages/web3/_utils/formatters.py", line 76, in map_collection
return datatype(map(func, collection))
File "***/venv/lib/python3.9/site-packages/web3/_utils/abi.py", line 855, in __new__
return super().__new__(cls, *iterable)
File "***/venv/lib/python3.9/site-packages/web3/_utils/formatters.py", line 88, in recurse
return recursive_map(func, item)
File "***/venv/lib/python3.9/site-packages/web3/_utils/decorators.py", line 30, in wrapped
wrapped_val = to_wrap(*args)
File "***/venv/lib/python3.9/site-packages/web3/_utils/formatters.py", line 89, in recursive_map
items_mapped = map_collection(recurse, data)
File "***/venv/lib/python3.9/site-packages/web3/_utils/formatters.py", line 76, in map_collection
return datatype(map(func, collection))
TypeError: <lambda>() missing 4 required positional arguments: 'hash', 'r', 's', and 'v'
I am using infura custom node, that's why I cant send transaction by contract.functions.method.transact(). Don't know to do with this error, spent a lot of time reading docs and got nothing.
How can I fix that?
sign_transaction returns SignedTransaction object while send_raw_transaction accepts raw transaction bytes. So change your last line to:
w3.eth.send_raw_transaction(t.rawTransaction)
You also probably want to save the result to a variable to track the transaction later.
You need to sign the transaction before sending with your account that has ETH balance.
You need to use Signing middleware.
>>> from web3 import Web3, EthereumTesterProvider
>>> w3 = Web3(EthereumTesterProvider)
>>> from web3.middleware import construct_sign_and_send_raw_middleware
>>> from eth_account import Account
>>> acct = Account.create('KEYSMASH FJAFJKLDSKF7JKFDJ 1530')
>>> w3.middleware_onion.add(construct_sign_and_send_raw_middleware(acct))
>>> w3.eth.default_account = acct.address
# Now you can send a tx from acct.address without having to build and sign each raw transaction

Python HTMLParser Not Reading Whole File

from HTMLParser import HTMLParser
class HTMLParserDos(HTMLParser):
full_text = ""
def handle_data(self, data):
self.full_text += data
return self.full_text
h = HTMLParserDos()
file = open('emails.txt', 'r')
h.feed(file.read())
file.close()
print h.container
This code is getting an error:
Traceback (most recent call last): File "/Users/laurenstrom/Google
Drive/PYTHON/RANDO_CALRISSIAN/html_parse", line 15, in
h.feed(file.read()) File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/HTMLParser.py",
line 108, in feed
self.goahead(0) File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/HTMLParser.py",
line 148, in goahead
k = self.parse_starttag(i) File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/HTMLParser.py",
line 229, in parse_starttag
endpos = self.check_for_whole_start_tag(i) File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/HTMLParser.py",
line 304, in check_for_whole_start_tag
self.error("malformed start tag") File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/HTMLParser.py",
line 115, in error
raise HTMLParseError(message, self.getpos()) HTMLParseError: malformed start tag, at line 7, column 18
I'm not sure what I'm missing about .feed() but I can't seem to find anything about why it won't just read the whole file.
Your are asking the HTML parser to parse a file most of which isn't HTML. It is tripping over line 7 of your file. Which is :
Return-Path: <Tom#sjnetworkconsulting.com>
I would imagine it is seeing the < and assuming that is HTML which of course it is not.

Categories

Resources