My Scrapy works just fine on my local machine, Windows. Then I try to run it on my AWS Linux server, but I got this
Traceback (most recent call last):
File "run<spider_name>.py", line 12, in <module>
spider_name).split())
File "/usr/lib/python2.7/site-packages/scrapy/cmdline.py", line 142, in execute
_run_print_help(parser, _run_command, cmd, args, opts)
File "/usr/lib/python2.7/site-packages/scrapy/cmdline.py", line 88, in _run_print_help
func(*a, **kw)
File "/usr/lib/python2.7/site-packages/scrapy/cmdline.py", line 149, in _run_command
cmd.run(args, opts)
File "/usr/lib/python2.7/site-packages/scrapy/commands/crawl.py", line 57, in run
self.crawler_process.crawl(spname, **opts.spargs)
File "/usr/lib/python2.7/site-packages/scrapy/crawler.py", line 162, in crawl
crawler = self.create_crawler(crawler_or_spidercls)
File "/usr/lib/python2.7/site-packages/scrapy/crawler.py", line 190, in create_crawler
return self._create_crawler(crawler_or_spidercls)
File "/usr/lib/python2.7/site-packages/scrapy/crawler.py", line 194, in _create_crawler
spidercls = self.spider_loader.load(spidercls)
File "/usr/lib/python2.7/site-packages/scrapy/spiderloader.py", line 51, in load
raise KeyError("Spider not found: {}".format(spider_name))
KeyError: 'Spider not found: <spider_name>'
Why is that? How can I run it on my Linux Server?
Suddenly solved and I confused myself.
I solve it by updating all the requirements using pip install -r requirements.txt. I added Scrapy Splash to the requirement and I forgot to install it.
Related
Trying to install PyTorch using this command:
py -m pip install --cache-dir=D:/CodePackages/temp --target=D:\CodePackages torch==1.13.1 -f https://download.pytorch.org/whl/torch_stable.html
And it is giving me these errors:
ERROR: Exception:
Traceback (most recent call last):
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_vendor\urllib3\response.py", line 437, in _error_catcher
yield
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_vendor\urllib3\response.py", line 560, in read
data = self._fp_read(amt) if not fp_closed else b""
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_vendor\urllib3\response.py", line 526, in _fp_read
return self._fp.read(amt) if amt is not None else self._fp.read()
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_vendor\cachecontrol\filewrapper.py", line 94, in read
self.__buf.write(data)
File "C:\Users\GorillaPenguin\AppData\Local\Programs\Python\Python310\lib\tempfile.py", line 483, in func_wrapper
return func(*args, **kwargs)
OSError: [Errno 28] No space left on device
Traceback (most recent call last):
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\cli\base_command.py", line 160, in exc_logging_wrapper
status = run_func(*args)
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\cli\req_command.py", line 247, in wrapper
return func(self, options, args)
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\commands\install.py", line 400, in run
requirement_set = resolver.resolve(
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\resolution\resolvelib\resolver.py", line 92, in resolve
result = self._result = resolver.resolve(
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 481, in resolve
state = resolution.resolve(requirements, max_rounds=max_rounds)
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 348, in resolve
self._add_to_criteria(self.state.criteria, r, parent=None)
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 172, in _add_to_criteria
if not criterion.candidates:
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_vendor\resolvelib\structs.py", line 151, in __bool__
return bool(self._sequence)
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 155, in __bool__
return any(self)
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 143, in <genexpr>
return (c for c in iterator if id(c) not in self._incompatible_ids)
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 47, in _iter_built
candidate = func()
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\resolution\resolvelib\factory.py", line 206, in _make_candidate_from_link
self._link_candidate_cache[link] = LinkCandidate(
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 297, in __init__
super().__init__(
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 162, in __init__
self.dist = self._prepare()
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 231, in _prepare
dist = self._prepare_distribution()
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 308, in _prepare_distribution
return preparer.prepare_linked_requirement(self._ireq, parallel_builds=True)
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\operations\prepare.py", line 491, in prepare_linked_requirement
return self._prepare_linked_requirement(req, parallel_builds)
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\operations\prepare.py", line 536, in _prepare_linked_requirement
local_file = unpack_url(
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\operations\prepare.py", line 166, in unpack_url
file = get_http_url(
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\operations\prepare.py", line 107, in get_http_url
from_path, content_type = download(link, temp_dir.path)
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\network\download.py", line 147, in __call__
for chunk in chunks:
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\cli\progress_bars.py", line 53, in _rich_progress_bar
for chunk in iterable:
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_internal\network\utils.py", line 63, in response_chunks
for chunk in response.raw.stream(
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_vendor\urllib3\response.py", line 621, in stream
data = self.read(amt=amt, decode_content=decode_content)
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_vendor\urllib3\response.py", line 559, in read
with self._error_catcher():
File "C:\Users\GorillaPenguin\AppData\Local\Programs\Python\Python310\lib\contextlib.py", line 153, in __exit__
self.gen.throw(typ, value, traceback)
File "D:\Unity\ReinforcementLeaning_InfoProject\venv\lib\site-packages\pip\_vendor\urllib3\response.py", line 454, in _error_catcher
raise ProtocolError("Connection broken: %r" % e, e)
pip._vendor.urllib3.exceptions.ProtocolError: ("Connection broken: OSError(28, 'No space left on device')", OSError(28, 'No space left on device'))
`
I'm using Python 3.10.8, latest version of pip, Windows 10.
The directory has more than enough space and I've tried changing the cache directory with --cache-dir without success. Any help is appreciated :) thanks
The error message you are seeing is likely due to one of the following reasons:
The specified path D:/CodePackages/temp or D:\CodePackages does not exist or is not accessible. You should make sure that the path exists and you have the necessary permissions to write to it.
The pip version that you are using is outdated. You should try running py -m pip install --upgrade pip to update pip to the latest version.
There may be an issue with the URL you are using in the -f flag. Try using the URL https://download.pytorch.org/whl/cu111/torch-1.13.1%2Bcu111-cp37-cp37m-win_amd64.whl .
If you have other version of python or pytorch installed it might cause conflict, you should try to uninstall previous versions of PyTorch, python and install the correct version.
Sometimes firewall or antivirus software can block the installation process, you should try to add exception in your firewall or antivirus software.
You might also want to try installing PyTorch without the --target and --cache-dir flags, to see if that resolves the issue.
py -m pip install torch==1.13.1 -f https://download.pytorch.org/whl/cu111/torch-1.13.1%2Bcu111-cp37-cp37m-win_amd64.whl
WARNING conda.exceptions:print_unexpected_error_report(1216):
KeyError('pkgs_dirs') Traceback (most recent call last): File
"C:\ProgramData\Anaconda3\lib\site-packages\conda\exceptions.py", line
1082, in call
return func(*args, **kwargs) File "C:\ProgramData\Anaconda3\lib\site-packages\conda\cli\main.py", line
87, in _main
exit_code = do_call(args, p) File "C:\ProgramData\Anaconda3\lib\site-packages\conda\cli\conda_argparse.py",
line 84, in do_call
return getattr(module, func_name)(args, parser) File "C:\ProgramData\Anaconda3\lib\site-packages\conda\cli\main_create.py",
line 21, in execute
if is_conda_environment(context.target_prefix): File "C:\ProgramData\Anaconda3\lib\site-packages\conda\base\context.py",
line 591, in target_prefix
return determine_target_prefix(self) File "C:\ProgramData\Anaconda3\lib\site-packages\conda\base\context.py",
line 1738, in determine_target_prefix
return locate_prefix_by_name(prefix_name) File "C:\ProgramData\Anaconda3\lib\site-packages\conda\base\context.py",
line 1681, in locate_prefix_by_name
envs_dirs = context.envs_dirs File "C:\ProgramData\Anaconda3\lib\site-packages\conda\base\context.py",
line 517, in envs_dirs
return mockable_context_envs_dirs(self.root_writable, self.root_prefix, self._envs_dirs) File
"C:\ProgramData\Anaconda3\lib\site-packages\conda\base\context.py",
line 91, in mockable_context_envs_dirs
fixed_dirs += join(user_data_dir(APP_NAME, APP_NAME), 'envs'), File
"C:\ProgramData\Anaconda3\lib\site-packages\conda_vendor\appdirs.py",
line 67, in user_data_dir
path = os.path.join(_get_win_folder(const), appauthor, appname) File
"C:\ProgramData\Anaconda3\lib\site-packages\conda_vendor\appdirs.py",
line 284, in _get_win_folder_with_pywin32
from win32com.shell import shellcon, shell ImportError: DLL load failed while importing shell: The specified procedure could not be
found.
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"C:\ProgramData\Anaconda3\lib\site-packages\conda\exceptions.py", line
1214, in print_unexpected_error_report
message_builder.append(get_main_info_str(error_report['conda_info']))
File
"C:\ProgramData\Anaconda3\lib\site-packages\conda\cli\main_info.py",
line 237, in get_main_info_str
info_dict['_' + key] = ('\n' + 26 * ' ').join(info_dict[key]) KeyError: 'pkgs_dirs'
>>>>>>>>>>>>>>>>>>>>>> ERROR REPORT <<<<<<<<<<<<<<<<<<<<<<
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\site-packages\conda\exceptions.py", line
1082, in call
return func(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\conda\cli\main.py", line
87, in _main
exit_code = do_call(args, p)
File "C:\ProgramData\Anaconda3\lib\site-packages\conda\cli\conda_argparse.py",
line 84, in do_call
return getattr(module, func_name)(args, parser)
File "C:\ProgramData\Anaconda3\lib\site-packages\conda\cli\main_create.py",
line 21, in execute
if is_conda_environment(context.target_prefix):
File "C:\ProgramData\Anaconda3\lib\site-packages\conda\base\context.py",
line 591, in target_prefix
return determine_target_prefix(self)
File "C:\ProgramData\Anaconda3\lib\site-packages\conda\base\context.py",
line 1738, in determine_target_prefix
return locate_prefix_by_name(prefix_name)
File "C:\ProgramData\Anaconda3\lib\site-packages\conda\base\context.py",
line 1681, in locate_prefix_by_name
envs_dirs = context.envs_dirs
File "C:\ProgramData\Anaconda3\lib\site-packages\conda\base\context.py",
line 517, in envs_dirs
return mockable_context_envs_dirs(self.root_writable, self.root_prefix, self._envs_dirs)
File "C:\ProgramData\Anaconda3\lib\site-packages\conda\base\context.py",
line 91, in mockable_context_envs_dirs
fixed_dirs += join(user_data_dir(APP_NAME, APP_NAME), 'envs'),
File "C:\ProgramData\Anaconda3\lib\site-packages\conda_vendor\appdirs.py",
line 67, in user_data_dir
path = os.path.join(_get_win_folder(const), appauthor, appname)
File "C:\ProgramData\Anaconda3\lib\site-packages\conda_vendor\appdirs.py",
line 284, in _get_win_folder_with_pywin32
from win32com.shell import shellcon, shell
ImportError: DLL load failed while importing shell: The specified procedure could not be found.
$ C:\ProgramData\Anaconda3\Scripts\conda-script.py create --name tf python=3.9
environment variables: conda info could not be constructed.
KeyError('pkgs_dirs')
An unexpected error has occurred. Conda has prepared the above report.
If submitted, this report will be used by core maintainers to improve
future releases of conda. Would you like conda to send this report to
the core maintainers?
[y/N]: y Upload successful.
Thank you for helping to improve conda. Opt-in to always sending
reports (and not see this message again) by running
$ conda config --set report_errors true
I keep getting this error. I tried conda info and it gives this error as well. Tried conda config --show-sources and it doesn't return anything.
I googled the pertinent part of the error and found this GitHub issue, where apparently downgrading the version of pywin32 you have installed could help:
pip install --upgrade pywin32==228
pip install clipboard
ERROR: Exception:
Traceback (most recent call last):
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_internal\cli\base_command.py", line 167, in exc_logging_wrapper
status = run_func(*args)
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_internal\cli\req_command.py", line 205, in wrapper
return func(self, options, args)
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_internal\commands\install.py", line 287, in run
session = self.get_default_session(options)
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_internal\cli\req_command.py", line 75, in get_default_session
self._session = self.enter_context(self._build_session(options))
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_internal\cli\req_command.py", line 89, in _build_session
session = PipSession(
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_internal\network\session.py", line 282, in __init__
self.headers["User-Agent"] = user_agent()
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_internal\network\session.py", line 157, in user_agent
setuptools_dist = get_default_environment().get_distribution("setuptools")
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_internal\metadata\__init__.py", line 75, in get_default_environment
return select_backend().Environment.default()
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_internal\metadata\__init__.py", line 63, in select_backend
from . import pkg_resources
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_internal\metadata\pkg_resources.py", line 8, in <module>
from pip._vendor import pkg_resources
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_vendor\pkg_resources\__init__.py", line 3252, in <module>
def _initialize_master_working_set():
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_vendor\pkg_resources\__init__.py", line 3235, in _call_aside
f(*args, **kwargs)
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_vendor\pkg_resources\__init__.py", line 3277, in _initialize_master_working_set
tuple(
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_vendor\pkg_resources\__init__.py", line 3278, in <genexpr>
dist.activate(replace=False)
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_vendor\pkg_resources\__init__.py", line 2785, in activate
declare_namespace(pkg)
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_vendor\pkg_resources\__init__.py", line 2284, in declare_namespace
_handle_ns(packageName, path_item)
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pip\_vendor\pkg_resources\__init__.py", line 2217, in _handle_ns
loader.load_module(packageName)
File "C:\Users\amin_\AppData\Roaming\Python\Python310\site-packages\pygoogle\google.py", line 113
print """PyGoogle %(__version__)s
^^^^^^^^^^^^^^^^^^^^^^^^^^
SyntaxError: Missing parentheses in call to 'print'. Did you mean print(...)?
WARNING: There was an error checking the latest version of pip.
I suggest you can use pyperclip as it is the same a s clipboard and it is newer.
https://pypi.org/project/pyperclip/
import pyperclip
>>> pyperclip.copy('The text to be copied to the clipboard.')
>>> pyperclip.paste()
'The text to be copied to the clipboard.'
Currently only handles plaintext.
On Windows, no additional modules are needed.
On Mac, this module makes use of the pbcopy and pbpaste commands, which should come with the os.
On Linux, this module makes use of the xclip or xsel commands, which should come with the os. Otherwise run “sudo apt-get install xclip” or “sudo apt-get install xsel” (Note: xsel does not always seem to work.)
Otherwise on Linux, you will need the gtk or PyQt4 modules installed.
I hope this help. :)
I'm deploying a django app made with django channels using nginx, gunicorn and daphne. During deployment when I run python manage.py makemigrations, I get the following error. Could it be because the python version on my local machine was 3.7 and on remote ubuntu server is 3.5?
from cryptography import x509
/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/OpenSSL/crypto.py:12: CryptographyDeprecationWarning: Python 3.5 support will be dropped in the next release of cryptography. Please upgrade your Python.
from cryptography import x509
Watching for file changes with StatReloader
Exception in thread django-main-thread:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/django/utils/autoreload.py", line 54, in wrapper
fn(*args, **kwargs)
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/django/core/management/commands/runserver.py", line 109, in inner_run
autoreload.raise_last_exception()
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/django/utils/autoreload.py", line 77, in raise_last_exception
raise _exception[1]
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/django/core/management/__init__.py", line 337, in execute
autoreload.check_errors(django.setup)()
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/django/utils/autoreload.py", line 54, in wrapper
fn(*args, **kwargs)
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/django/__init__.py", line 24, in setup
apps.populate(settings.INSTALLED_APPS)
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/django/apps/registry.py", line 122, in populate
app_config.ready()
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/channels/apps.py", line 20, in ready
monkeypatch_django()
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/channels/hacks.py", line 10, in monkeypatch_django
from .management.commands.runserver import Command as RunserverCommand
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/channels/management/commands/runserver.py", line 11, in <module>
from channels.routing import get_default_application
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/channels/routing.py", line 10, in <module>
from channels.http import AsgiHandler
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/channels/http.py", line 152, in <module>
class AsgiHandler(base.BaseHandler):
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/channels/http.py", line 214, in AsgiHandler
#sync_to_async
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/asgiref/sync.py", line 375, in sync_to_async
return SyncToAsync(func, thread_sensitive=thread_sensitive)
File "/home/ubuntu/django/virtualenv2/lib/python3.5/site-packages/asgiref/sync.py", line 262, in __init__
self._is_coroutine = asyncio.coroutines._is_coroutine
AttributeError: module 'asyncio.coroutines' has no attribute '_is_coroutine'
The error is beacause of asgiref package that uses asyncio.coroutines._is_coroutine attribute since version 3.2.2.
You can install asgiref 3.2.1 and this problem doesn't happen.
pip install asgiref==3.2.1
I have a python 2.7.12 installed on ubuntu 10.04. I have updated pip from 8.1.2 to the last version (9.0.1) but now when I run pip I get this error:
Traceback (most recent call last):
File "/usr/local/bin/pip", line 11, in <module>
sys.exit(main())
File "/usr/local/lib/python2.7/site-packages/pip/__init__.py", line 233, in main
return command.main(cmd_args)
File "/usr/local/lib/python2.7/site-packages/pip/basecommand.py", line 251, in main
timeout=min(5, options.timeout)) as session:
File "/usr/local/lib/python2.7/site-packages/pip/basecommand.py", line 72, in _build_session
insecure_hosts=options.trusted_hosts,
File "/usr/local/lib/python2.7/site-packages/pip/download.py", line 329, in __init__
self.headers["User-Agent"] = user_agent()
File "/usr/local/lib/python2.7/site-packages/pip/download.py", line 93, in user_agent
from pip._vendor import distro
File "/usr/local/lib/python2.7/site-packages/pip/_vendor/distro.py", line 1050, in <module>
_distro = LinuxDistribution()
File "/usr/local/lib/python2.7/site-packages/pip/_vendor/distro.py", line 594, in __init__
if include_lsb else {}
File "/usr/local/lib/python2.7/site-packages/pip/_vendor/distro.py", line 933, in _get_lsb_release_info
raise subprocess.CalledProcessError(code, cmd, stdout)
subprocess.CalledProcessError: Command 'lsb_release -a' returned non-zero exit status 1
Any ideas to solve this issue?