import json
from six.moves.urllib_parse import urlencode
from six.moves.urllib_request import urlopen
from django.core.management.base import CommandError
def call(method, data, post=False):
"""
Calls `method` from the DISQUS API with data either in POST or GET.
Returns deserialized JSON response.
"""
url = "%s%s" % ("http://disqus.com/api/", method)
if post:
# POST request
url += "/"
data = urlencode(data)
else:
# GET request
url += "?%s" % urlencode(data)
data = ""
res = json.load(urlopen(url, data))
if not res["succeeded"]:
raise CommandError(
"'%s' failed: %s\nData: %s" % (method, res["code"], data)
)
return res["message"]
module) moves
Import "six.moves.urllib_parse" could not be resolved from sourcePylancereportMissingModuleSource
installed the six module
to Python virtual environment
six can be imported without problems,
Occurs from six.moves MissingModuleSource
Why can't Import? six.moves
try changing system interpreter path of python in your IDE and set it to the virtual environment you use, in which you have installed the module.
an example in vscode
Related
I made a big mistake by creating an app inside my Django project called requests that happened a long time ago and the system has already been running for years. now I need to use the requests library and it is being imported like import requests as mentioned in the documentation ... and of course whenever I do this import it imports my app instead of the library .. so how to solve this?
You can try importing the requests/__init__.py file directly as shown in docs: https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly.
Example:
import sys
import importlib.util
module_name = 'requests'
# declare the full path to requests/__init__.py file below
module_path = '/path/to/virtualenv/site-packages/requests/__init__.py'
spec = importlib.util.spec_from_file_location(module_name, module_path)
requests = importlib.util.module_from_spec(spec)
sys.modules[module_name] = requests
spec.loader.exec_module(requests)
print(requests.post) # should not raise error
import urllib request
import requests
goog_url = "https://query1.finance.yahoo.com/v7/finance/download/GOOG?period1=1501517722&period2=1504196122&interval=1d&events=history&crumb=bU42Yaj88Bt"
def download_stock_data(csv_url):
response = ur.urlopen(csv_url)
csv = response.read()
csv_str = str(csv)
lines = csv_str.split("\\n")
dest_url = r'goog.csv'
fx = open(dest_url, "w")
for line in lines:
fx.write(line + "\n")
fx.close()
download_stock_data(goog_url)
I'm trying to import a CSV file from the internet with this code. But I continue, despite my best efforts, to get a syntax error that says that it cannot find the request module of the urllib import.
File "/Users/Micmaster/PycharmProjects/pythonProject/firstProject.py", line 1
import urllib request
^
SyntaxError: invalid syntax
I've tried many different variations "from urllib import request", "import urllib.request", "import urllib", "import urllib2.request" and even changing versions of my interpreter on pycharm. Any help would be appreciated, thanks!
The urllib.request module is in Python 3 library;
For Python 2 you'd use urllib2.
I will write about python2.
Why are you trying import request python or object from the urllib package, when this package doesn't have it.
And on the next line your import requests. So use requests.
And I don't know why you need urllib, but change import urllib request to import urllib as ur.
import urllib
from urllib import request
goog_url = "https://query1.finance.yahoo.com/v7/finance/download/GOOG?period1=1501517722&period2=1504196122&interval=1d&events=history&crumb=bU42Yaj88Bt"
def download_stock_data(csv_url):
response = urllib.request.urlopen(csv_url)
csv = response.read()
csv_str = str(csv)
lines = csv_str.split("\\n")
dest_url = r'goog.csv'
fx = open(dest_url, "w")
for line in lines:
fx.write(line + "\n")
fx.close()
download_stock_data(goog_url)
Your code should look like this. But it is still showing HTTP Error 401: Unauthorized you had taken the wrong URL.
Is there a way to force an import to be absolute instead of relative?
I am "overriding" the Python standard library json module, so in my projects I always use the correct encoder and parameters:
project/foo/json.py: (mark this filename)
import json as pyjson
class ComplexEncoder(pyjson.JSONEncoder):
def default(self, obj):
if hasattr(obj, 'isoformat'):
return obj.isoformat()
else:
if type(obj) == file:
return "filestream"
raise TypeError, 'Object of type %s with value of %s is not JSON serializable' % (type(obj), repr(obj))
def dumps(data):
return pyjson.dumps(data, cls=ComplexEncoder, check_circular=False, separators=(',', ':'), ensure_ascii=False)
def loads(data):
return pyjson.loads(data)
When I import this file I get the dreaded AttributeError: 'module' object has no attribute 'JSONEncoder'.
A print(pyjson.__file__) confirms my suspicion that import json as pyjson imports json from the local package instead of from the Python standard library.
Is there a way to force the import to be absolute, so the local directory is ignored?
If your "json" module is inside a package, then this will fix it:
from __future__ import absolute_import
With that __future__ statement, imports will be absolute be default - i.e. it will only look for a top-level module or file called json.
If you need to import your local one, you can either do import foo.json, or you can explictly ask for a relative import using from . import json or from .json import dumps.
(I am assuming you're using Python 2).
You can delete the self path from the sys.path:
self_dir = os.path.dirname(os.path.realpath(__file__))
if self_dir in sys.path:
sys.path.remove(self_dir)
Then import the json as regular.
If you need to import anything from the local directory, just append the path to the sys.path and import what you need:
sys.path.append(self_dir)
from local_module import *
Grinder is new for me and I am trying to figure out how to get rid of this error:
my test.py script:
import string
import random
from java.lang import String
from java.net import URLEncoder
from net.grinder.plugin.http import HTTPRequest
from net.grinder.common import GrinderException
log = grinder.logger.info
stat = grinder.statistics.forLastTest
SERVER = "http://www.google.com"
URI = "/"
class TestRunner:
def __call__(self):
requestString = "%s%s" % (SERVER, URI)
request = HTTPRequest()
result = request.GET(requestString)
if string.find(result.getText(), "SUCCESS") < 1:
stat.setSuccess(0)
I run
java net.grinder.Console
java net.grinder.Grinder
in my localhost.
after starting the test, this message keeps popping up:
aborting process - Jython exception, <type 'exceptions.NameError'>: name 'grinder' is not defined [initialising test script]
net.grinder.scriptengine.jython.JythonScriptExecutionException: <type 'exceptions.NameError'>: name 'grinder' is not defined
log = grinder.logger.info
File "./test.py", line 8, in <module>
It looks like I have to include some Grinder module for this "grinder.logger.info" but I just have no clue of what I should import... ...
Any hint?
Thanks in advance
you imported items from grinder, not grinder itself, try
import grinder.logger.info
import grinder.statistics.forLastTest
it might also be net.grinder.logger.info and net.grinder.statistics.forLastTest if this is the case then your code will need changing to go with the change from grinder to net.grinder
You have not imported grinder.
from net.grinder.script.Grinder import grinder
and now try again.
I'm writing a simple script that:
Loads a big list of URLs
Get the content of each URL making concurrent HTTP requests using requests' async module
Parses the content of the page with lxml in order to check if a link is in the page
If the link is present on the page, saves some info about the page in a ZODB database
When I test the script with 4 or 5 URLs It works well, I only have the following message when the script ends:
Exception KeyError: KeyError(45989520,) in <module 'threading' from '/usr/lib/python2.7/threading.pyc'> ignored
But when I try to check about 24000 URLs it fails toward the end of the list (when there are about 400 URLs left to check) with the following error:
Traceback (most recent call last):
File "check.py", line 95, in <module>
File "/home/alex/code/.virtualenvs/linka/local/lib/python2.7/site-packages/requests/async.py", line 83, in map
File "/home/alex/code/.virtualenvs/linka/local/lib/python2.7/site-packages/gevent-1.0b2-py2.7-linux-x86_64.egg/gevent/greenlet.py", line 405, in joinall
ImportError: No module named queue
Exception KeyError: KeyError(45989520,) in <module 'threading' from '/usr/lib/python2.7/threading.pyc'> ignored
I tried both with the version of gevent available on pypi and downloading and installing the latest version (1.0b2) from gevent repository.
I cannot understand why this happened, and why it happened only when I check a bunch of URLs. Any suggestions?
Here is the entire script:
from requests import async, defaults
from lxml import html
from urlparse import urlsplit
from gevent import monkey
from BeautifulSoup import UnicodeDammit
from ZODB.FileStorage import FileStorage
from ZODB.DB import DB
import transaction
import persistent
import random
storage = FileStorage('Data.fs')
db = DB(storage)
connection = db.open()
root = connection.root()
monkey.patch_all()
defaults.defaults['base_headers']['User-Agent'] = "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko/20100101 Firefox/11.0"
defaults.defaults['max_retries'] = 10
def save_data(source, target, anchor):
root[source] = persistent.mapping.PersistentMapping(dict(target=target, anchor=anchor))
transaction.commit()
def decode_html(html_string):
converted = UnicodeDammit(html_string, isHTML=True)
if not converted.unicode:
raise UnicodeDecodeError(
"Failed to detect encoding, tried [%s]",
', '.join(converted.triedEncodings))
# print converted.originalEncoding
return converted.unicode
def find_link(html_doc, url):
decoded = decode_html(html_doc)
doc = html.document_fromstring(decoded.encode('utf-8'))
for element, attribute, link, pos in doc.iterlinks():
if attribute == "href" and link.startswith('http'):
netloc = urlsplit(link).netloc
if "example.org" in netloc:
return (url, link, element.text_content().strip())
else:
return False
def check(response):
if response.status_code == 200:
html_doc = response.content
result = find_link(html_doc, response.url)
if result:
source, target, anchor = result
# print "Source: %s" % source
# print "Target: %s" % target
# print "Anchor: %s" % anchor
# print
save_data(source, target, anchor)
global todo
todo = todo -1
print todo
def load_urls(fname):
with open(fname) as fh:
urls = set([url.strip() for url in fh.readlines()])
urls = list(urls)
random.shuffle(urls)
return urls
if __name__ == "__main__":
urls = load_urls('urls.txt')
rs = []
todo = len(urls)
print "Ready to analyze %s pages" % len(urls)
for url in urls:
rs.append(async.get(url, hooks=dict(response=check), timeout=10.0))
responses = async.map(rs, size=100)
print "DONE."
I'm not sure what's the source of your problem, but why do you have monkey.patch_all() not at the top of the file?
Could you try putting
from gevent import monkey; monkey.patch_all()
at the top of your main program and see if it fixes anything?
I am such a big n00b but anyway, I can try ... !
I guess you can try to change your import list by this one :
from requests import async, defaults
import requests
from lxml import html
from urlparse import urlsplit
from gevent import monkey
import gevent
from BeautifulSoup import UnicodeDammit
from ZODB.FileStorage import FileStorage
from ZODB.DB import DB
import transaction
import persistent
import random
Try this and tell me if it works .. I guess that can solve your problem :)
good day.
I think it's open python bug with number Issue1596321
http://bugs.python.org/issue1596321