I have following code which indexes data to elasticsearch using python,
from elasticsearch import Elasticsearch
from elasticsearch import helpers
import requests
from requests.auth import AuthBase
requests.packages.urllib3.disable_warnings()
class TokenAuth(AuthBase):
def __init__(self, token):
self.token = token
def __call__(self, r):
r.headers['Authorization :Bearer'] = f'{self.token}'
return r
es = Elasticsearch('https://localhost:9200/user/type',ca_certs=False,verify_certs=False,auth=TokenAuth(''))
#requests.get('https://httpbin.org/get', auth=TokenAuth('12345abcde-token'))
res = helpers.bulk(es, "ldif2.json", chunk_size=1, request_timeout=200)
It follows token based autheentication , but whan i run this progam i get below error message ,how do i solve this.
Traceback (most recent call last):
File "bulk_index.py", line 20, in <module>
res = helpers.bulk(es, "ldif2.json", chunk_size=1, request_timeout=200)
File "C:\Users\mkumaru\AppData\Local\Programs\Python\Python37\lib\site-packages\elasticsearch\helpers\actions.py", line 300, in bulk
for ok, item in streaming_bulk(client, actions, *args, **kwargs):
File "C:\Users\mkumaru\AppData\Local\Programs\Python\Python37\lib\site-packages\elasticsearch\helpers\actions.py", line 230, in streaming_bulk
**kwargs
File "C:\Users\mkumaru\AppData\Local\Programs\Python\Python37\lib\site-packages\elasticsearch\helpers\actions.py", line 116, in _process_bulk_chunk
raise e
File "C:\Users\mkumaru\AppData\Local\Programs\Python\Python37\lib\site-packages\elasticsearch\helpers\actions.py", line 112, in _process_bulk_chunk
resp = client.bulk("\n".join(bulk_actions) + "\n", *args, **kwargs)
File "C:\Users\mkumaru\AppData\Local\Programs\Python\Python37\lib\site-packages\elasticsearch\client\utils.py", line 84, in _wrapped
return func(*args, params=params, **kwargs)
File "C:\Users\mkumaru\AppData\Local\Programs\Python\Python37\lib\site-packages\elasticsearch\client\__init__.py", line 1498, in bulk
headers={"content-type": "application/x-ndjson"},
File "C:\Users\mkumaru\AppData\Local\Programs\Python\Python37\lib\site-packages\elasticsearch\transport.py", line 353, in perform_request
timeout=timeout,
File "C:\Users\mkumaru\AppData\Local\Programs\Python\Python37\lib\site-packages\elasticsearch\connection\http_urllib3.py", line 239, in perform_request
self._raise_error(response.status, raw_data)
File "C:\Users\mkumaru\AppData\Local\Programs\Python\Python37\lib\site-packages\elasticsearch\connection\base.py", line 168, in _raise_error
status_code, error_message, additional_info
elasticsearch.exceptions.AuthenticationException: AuthenticationException(401, 'Access denied')```
I think es should be like this.
es = Elasticsearch("http://127.0.0.1:9200", http_auth=('user', 'passwd'))
Related
I am trying to read a cloud storage file into a Pandas dataframe locally and then load it into a Big Query table using Python but I get the following error:
google.auth.exceptions.RefreshError: ('invalid_grant: Bad Request', '{\n "error": "invalid_grant",\n "error_description": "Bad Request"\n}')
My code is:
import pandas as pd
from google.cloud import storage
from google.oauth2 import service_account
import gcsfs
scopes=["https://www.googleapis.com/auth/bigquery","https://www.googleapis.com/auth/cloud-platform"]
file='<<filename>>.json'
credentials = service_account.Credentials.from_service_account_file(file, scopes=scopes)
def run(file):
client = storage.Client.from_service_account_json(file)
bucket = client.bucket('my_bucket')
blobs = bucket.list_blobs()
list_temp_raw = []
for file in blobs:
filename = file.name
temp = pd.read_csv('gs://<<>my_bucket>>/' + filename, encoding='utf-8')
list_temp_raw.append(temp)
df = pd.concat(list_temp_raw, encoding='utf-8')
df.to_gbq('<<dataset>>.<<table>>',if_exists='append',credentials=credentials)
run(file)
It is the temp = pd.read_csv('gs://<<>my_bucket>>/' + filename, encoding='utf-8') causing the error:
Traceback (most recent call last):
File "test-load-bq.py", line 23, in <module>
run(file)
File "test-load-bq.py", line 18, in run
temp = pd.read_csv('gs://project-test/' + filename, encoding='utf-8')
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/pandas/io/parsers.py", line 676, in parser_f
return _read(filepath_or_buffer, kwds)
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/pandas/io/parsers.py", line 431, in _read
filepath_or_buffer, encoding, compression
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/pandas/io/common.py", line 192, in get_filepath_or_buffer
filepath_or_buffer, encoding=encoding, compression=compression, mode=mode
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/pandas/io/gcs.py", line 17, in get_filepath_or_buffer
filepath_or_buffer = fs.open(filepath_or_buffer, mode)
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/fsspec/spec.py", line 936, in open
**kwargs
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/gcsfs/core.py", line 1287, in _open
**kwargs,
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/gcsfs/core.py", line 1412, in __init__
**kwargs,
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/fsspec/spec.py", line 1257, in __init__
self.details = fs.info(path)
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/fsspec/asyn.py", line 121, in wrapper
return maybe_sync(func, self, *args, **kwargs)
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/fsspec/asyn.py", line 100, in maybe_sync
return sync(loop, func, *args, **kwargs)
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/fsspec/asyn.py", line 71, in sync
raise exc.with_traceback(tb)
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/fsspec/asyn.py", line 55, in f
result[0] = await future
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/gcsfs/core.py", line 825, in _info
return await self._get_object(path)
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/gcsfs/core.py", line 607, in _get_object
res = await self._call("GET", "b/{}/o/{}", bucket, key, json_out=True)
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/gcsfs/core.py", line 503, in _call
self.maybe_refresh()
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/gcsfs/core.py", line 404, in maybe_refresh
self.credentials.refresh(req)
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/google/oauth2/credentials.py", line 198, in refresh
self._scopes,
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/google/oauth2/_client.py", line 248, in refresh_grant
response_data = _token_endpoint_request(request, token_uri, body)
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/google/oauth2/_client.py", line 124, in _token_endpoint_request
_handle_error_response(response_body)
File "/Users/willcharles/Documents/project/venv/lib/python3.6/site-packages/google/oauth2/_client.py", line 60, in _handle_error_response
raise exceptions.RefreshError(error_details, response_body)
google.auth.exceptions.RefreshError: ('invalid_grant: Bad Request', '{\n "error": "invalid_grant",\n "error_description": "Bad Request"\n}')
I have given my service account the following roles:
Storage Admin & BigQuery Admin
Any ideas why the error is happening?
Below is my Code in raspberry PI's python(Thonny Idle).
Kindly Ignore the Url, it is not the real address.
Code
from firebase import firebase
firebase = firebase.FirebaseApplication('https://testing123123-iot.firebaseio.com',authentication=None)
data = {
'Name':'Hi',
'Email':'hihi.com',
'Phone':512232131
}
result = firebase.post('/testing123123-iot:/Customer', data)
print(result)
Error
Traceback (most recent call last):
File "/home/pi/Documents/PythonCode/TestingFirebase-1.py", line 17, in
result = firebase.post('/testing-iot:/Customer', data)
File "/usr/local/lib/python3.7/dist-packages/firebase/decorators.py", line 19, in wrapped
return f(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/firebase/firebase.py", line 329, in post
connection=connection)
File "/usr/local/lib/python3.7/dist-packages/firebase/decorators.py", line 19, in wrapped
return f(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/firebase/firebase.py", line 97, in make_post_request
timeout=timeout)
File "/usr/local/lib/python3.7/dist-packages/requests/sessions.py", line 340, in post
return self.request('POST', url, data=data, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/requests/sessions.py", line 279, in request
resp = self.send(prep, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
File "/usr/local/lib/python3.7/dist-packages/requests/sessions.py", line 374, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/requests/adapters.py", line 174, in send
timeout=timeout
File "/usr/local/lib/python3.7/dist-packages/requests/packages/urllib3/connectionpool.py", line 417, in urlopen
conn = self._get_conn(timeout=pool_timeout)
File "/usr/local/lib/python3.7/dist-packages/requests/packages/urllib3/connectionpool.py", line 232, in _get_conn
return conn or self._new_conn()
File "/usr/local/lib/python3.7/dist-packages/requests/packages/urllib3/connectionpool.py", line 547, in _new_conn
strict=self.strict)
TypeError: init() got an unexpected keyword argument 'strict'
use json.dumps :
import json
data = {
'Name':'Hi',
'Email':'hihi.com',
'Phone':512232131
}
sent = json.dumps(data)
result = firebase.post('/testing123123-iot:/Customer', sent)
print(result)
Having an issue with the api for Odoo v13. I am able to get the server info but for some reason the uid is not being returned
import xmlrpc.client
url ="localhost:8069"
db = "pnv3"
username = "test"
password = "test"
common = xmlrpc.client.ServerProxy('{}/xmlrpc/2/common'.format(url))
print(common.version())
uid = common.authenticate(db, username, password, url)
print(uid)
getting this error
Traceback (most recent call last):
File "C:/Users/Web Content/.PyCharmCE2019.3/config/scratches/scratch.py", line 11, in <module>
uid = common.authenticate(db, username, password, url)
File "C:\Users\Web Content\AppData\Local\Programs\Python\Python37\lib\xmlrpc\client.py", line 1112, in __call__
return self.__send(self.__name, args)
File "C:\Users\Web Content\AppData\Local\Programs\Python\Python37\lib\xmlrpc\client.py", line 1452, in __request
verbose=self.__verbose
File "C:\Users\Web Content\AppData\Local\Programs\Python\Python37\lib\xmlrpc\client.py", line 1154, in request
return self.single_request(host, handler, request_body, verbose)
File "C:\Users\Web Content\AppData\Local\Programs\Python\Python37\lib\xmlrpc\client.py", line 1170, in single_request
return self.parse_response(resp)
File "C:\Users\Web Content\AppData\Local\Programs\Python\Python37\lib\xmlrpc\client.py", line 1342, in parse_response
return u.close()
File "C:\Users\Web Content\AppData\Local\Programs\Python\Python37\lib\xmlrpc\client.py", line 656, in close
raise Fault(**self._stack[0])
xmlrpc.client.Fault: <Fault 1: 'Traceback (most recent call last):\n File "/odoo/odoo-server/odoo/modules/registry.py", line 59, in __new__\n return cls.registries[db_name]\n File "/odoo/odoo-server/odoo/tools/func.py", line 69, in wrapper\n return func(self, *args, **kwargs)\n File "/odoo/odoo-server/odoo/tools/lru.py", line 44, in __getitem__\n a = self.d[obj].me\nKeyError: \'pnv3\'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/odoo/odoo-server/odoo/addons/base/controllers/rpc.py", line 63, in xmlrpc_2\n response = self._xmlrpc(service)\n File "/odoo/odoo-server/odoo/addons/base/controllers/rpc.py", line 43, in _xmlrpc\n result = dispatch_rpc(service, method, params)\n File "/odoo/odoo-server/odoo/http.py", line 138, in dispatch_rpc\n result = dispatch(method, params)\n File "/odoo/odoo-server/odoo/service/common.py", line 61, in dispatch\n return g[exp_method_name](*params)\n File "/odoo/odoo-server/odoo/service/common.py", line 30, in exp_authenticate\n res_users = odoo.registry(db)[\'res.users\']\n File "/odoo/odoo-server/odoo/__init__.py", line 104, in registry\n return modules.registry.Registry(database_name)\n File "/odoo/odoo-server/odoo/modules/registry.py", line 61, in __new__\n return cls.new(db_name)\n File "/odoo/odoo-server/odoo/modules/registry.py", line 73, in new\n registry.init(db_name)\n File "/odoo/odoo-server/odoo/modules/registry.py", line 141, in init\n with closing(self.cursor()) as cr:\n File "/odoo/odoo-server/odoo/modules/registry.py", line 492, in cursor\n return self._db.cursor()\n File "/odoo/odoo-server/odoo/sql_db.py", line 649, in cursor\n return Cursor(self.__pool, self.dbname, self.dsn, serialized=serialized)\n File "/odoo/odoo-server/odoo/sql_db.py", line 186, in __init__\n self._cnx = pool.borrow(dsn)\n File "/odoo/odoo-server/odoo/sql_db.py", line 532, in _locked\n return fun(self, *args, **kwargs)\n File "/odoo/odoo-server/odoo/sql_db.py", line 600, in borrow\n **connection_info)\n File "/usr/local/lib/python3.7/dist-packages/psycopg2/__init__.py", line 130, in connect\n conn = _connect(dsn, connection_factory=connection_factory, **kwasync)\npsycopg2.OperationalError: FATAL: database "pnv3" does not exist\n\n'>
Process finished with exit cod
1
Databse does exist, have triple checked my password, not sure what else to do at this point.
The url to Odoo server should include protocol part "http://" in the beginning. Strange that you get the version info at all. What do you get as output for the version?
Also you pass the url as the last parameter to authenticate method and this is not required. This should still not give the error you received.
Try your code with these two fixes and report if this helps:
import xmlrpc.client
url ="http://localhost:8069"
db = "pnv3"
username = "test"
password = "test"
common = xmlrpc.client.ServerProxy('{}/xmlrpc/2/common'.format(url))
print(common.version())
uid = common.authenticate(db, username, password, {})
print(uid)
Identical code works on my machine...
I am working on a college project, but I need to make the code below work with socks4 instead of tor/socks5. I have tried modifying SOCKS5Agent to SOCKS4Agent but then I receive and error:
Original code: https://stackoverflow.com/a/33944924/11219616
My code:
import scrapy.core.downloader.handlers.http11 as handler
from twisted.internet import reactor
from txsocksx.http import SOCKS4Agent
from twisted.internet.endpoints import TCP4ClientEndpoint
from scrapy.core.downloader.webclient import _parse
class TorScrapyAgent(handler.ScrapyAgent):
_Agent = SOCKS4Agent
def _get_agent(self, request, timeout):
proxy = request.meta.get('proxy')
if proxy:
proxy_scheme, _, proxy_host, proxy_port, _ = _parse(proxy)
if proxy_scheme == 'socks4':
endpoint = TCP4ClientEndpoint(reactor, proxy_host, proxy_port)
return self._Agent(reactor, proxyEndpoint=endpoint)
return super(TorScrapyAgent, self)._get_agent(request, timeout)
class TorHTTPDownloadHandler(handler.HTTP11DownloadHandler):
def download_request(self, request, spider):
agent = TorScrapyAgent(contextFactory=self._contextFactory, pool=self._pool,
maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
warnsize=getattr(spider, 'download_warnsize', self._default_warnsize))
return agent.download_request(request)
I get the error:
Traceback (most recent call last):
File "C:\Python27\lib\site-packages\twisted\internet\defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "C:\Python27\lib\site-packages\twisted\python\failure.py", line 491, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "C:\Python27\lib\site-packages\scrapy\core\downloader\middleware.py", line 43, in process_request
defer.returnValue((yield download_func(request=request,spider=spider)))
File "C:\Python27\lib\site-packages\ometa\protocol.py", line 53, in dataReceived
self._parser.receive(data)
File "C:\Python27\lib\site-packages\ometa\tube.py", line 41, in receive
status = self._interp.receive(data)
File "C:\Python27\lib\site-packages\ometa\interp.py", line 48, in receive
for x in self.next:
File "C:\Python27\lib\site-packages\ometa\interp.py", line 177, in apply
for x in self._apply(f, ruleName, argvals):
File "C:\Python27\lib\site-packages\ometa\interp.py", line 110, in _apply
for x in rule():
File "C:\Python27\lib\site-packages\ometa\interp.py", line 256, in parse_Or
for x in self._eval(subexpr):
File "C:\Python27\lib\site-packages\ometa\interp.py", line 241, in parse_And
for x in self._eval(subexpr):
File "C:\Python27\lib\site-packages\ometa\interp.py", line 440, in parse_Action
val = eval(expr.data, self.globals, self._localsStack[-1])
File "<string>", line 1, in <module>
File "C:\Python27\lib\site-packages\txsocksx\client.py", line 276, in serverResponse
raise e.socks4ErrorMap.get(status)()
RequestRejectedOrFailed
I'm using django-rq, the django bindings for python-rq, to try generate a PDF asynchronously. The class TemplateProcesser initializes with two arguments and automatically generates the PDF in the __init__ function. This works fine synchronously, outside of django-rq, but with django-rq it fails with this error:
Error:
AttributeError: type object 'TemplateProcesser' has no attribute 'rsplit'
From this call:
django_rq.enqueue(TemplateProcesser, nail_order=serializer.object, user_photo=base64_image)
Any idea on how to correctly include the uninstantiated class in django-rq?
Class:
class TemplateProcesser(object):
def __init__(self, nail_order, user_photo, *args, **kwargs):
self.nail_order = nail_order
self.user_photo = user_photo
...
self.procces_template()
...
StackTrace:
Traceback (most recent call last):
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/worker.py", line 426, in perform_job
rv = job.perform()
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/job.py", line 386, in perform
self._result = self.func(*self.args, **self.kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/job.py", line 154, in func
return import_attribute(self.func_name)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/utils.py", line 168, in import_attribute
module_name, attribute = name.rsplit('.', 1)
AttributeError: type object 'TemplateProcesser' has no attribute 'rsplit'
Traceback after metaperture's answer:
Traceback (most recent call last):
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django/core/handlers/base.py", line 114, in get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django/views/generic/base.py", line 69, in view
return self.dispatch(request, *args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django/views/decorators/csrf.py", line 57, in wrapped_view
return view_func(*args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rest_framework/views.py", line 400, in dispatch
response = self.handle_exception(exc)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rest_framework/views.py", line 397, in dispatch
response = handler(request, *args, **kwargs)
File "/Users/admin/dev/ncla-web/ncla/api/views.py", line 91, in post
django_rq.enqueue(self.template_processor_factory, **parameter_dict)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django_rq/queues.py", line 162, in enqueue
return get_queue().enqueue(func, *args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/queue.py", line 213, in enqueue
description=description, depends_on=depends_on)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django_rq/queues.py", line 42, in enqueue_call
return self.original_enqueue_call(*args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/django_rq/queues.py", line 37, in original_enqueue_call
return super(DjangoRQ, self).enqueue_call(*args, **kwargs)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/queue.py", line 176, in enqueue_call
return self.enqueue_job(job)
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/queue.py", line 232, in enqueue_job
job.save()
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/job.py", line 360, in save
connection.hmset(key, self.dump())
File "/Users/admin/dev/ncla-web/env/lib/python2.7/site-packages/rq/job.py", line 329, in dump
obj['data'] = dumps(self.job_tuple)
File "/Users/admin/dev/ncla-web/env/bin/../lib/python2.7/copy_reg.py", line 70, in _reduce_ex
raise TypeError, "can't pickle %s objects" % base.__name__
TypeError: can't pickle BytesIO objects
Looks like python-rq expects a function as the first argument. I would try:
def template_processor_factory(*args, **kwargs):
return TemplateProcessor(*args, **kwargs)
django_rq.enqueue(template_processer_factory, nail_order=serializer.object,
user_photo=base64_image)