I want to save my data to remote machine by using peewee. When i run my crawler i found following error,
File "/usr/local/lib/python2.7/site-packages/scrapy/commands/crawl.py", line 57, in run
self.crawler_process.crawl(spname, **opts.spargs)
File "/usr/local/lib/python2.7/site-packages/scrapy/crawler.py", line 163, in crawl
return self._crawl(crawler, *args, **kwargs)
File "/usr/local/lib/python2.7/site-packages/scrapy/crawler.py", line 167, in _crawl
d = crawler.crawl(*args, **kwargs)
File "/usr/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1445, in unwindGenerator
return _inlineCallbacks(None, gen, Deferred())
--- <exception caught here> ---
File "/usr/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1299, in _inlineCallbacks
result = g.send(result)
File "/usr/local/lib/python2.7/site-packages/scrapy/crawler.py", line 90, in crawl
six.reraise(*exc_info)
File "/usr/local/lib/python2.7/site-packages/scrapy/crawler.py", line 72, in crawl
self.engine = self._create_engine()
File "/usr/local/lib/python2.7/site-packages/scrapy/crawler.py", line 97, in _create_engine
return ExecutionEngine(self, lambda _: self.stop())
File "/usr/local/lib/python2.7/site-packages/scrapy/core/engine.py", line 70, in __init__
self.scraper = Scraper(crawler)
File "/usr/local/lib/python2.7/site-packages/scrapy/core/scraper.py", line 71, in __init__
self.itemproc = itemproc_cls.from_crawler(crawler)
File "/usr/local/lib/python2.7/site-packages/scrapy/middleware.py", line 58, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/usr/local/lib/python2.7/site-packages/scrapy/middleware.py", line 34, in from_settings
mwcls = load_object(clspath)
File "/usr/local/lib/python2.7/site-packages/scrapy/utils/misc.py", line 44, in load_object
mod = import_module(module)
File "/usr/local/lib/python2.7/importlib/__init__.py", line 37, in import_module
__import__(name)
File "/app/__main__.egg/annuaire_agence_bio/pipelines.py", line 8, in <module>
exceptions.ImportError: No module named peewee
Any suggestions is more than welcome.
You cannot install module of your own choice on Scrapinhub ... You can only install MySQLDB as per my knowledge, to do that.
Create a file named scrapinghub.yml in your project's main folder with following contents.
projects:
default: 111149
requirements:
file: requirements.txt
Where 111149 is my proejct ID on scrapinghub.
Create another file named requirements.txt in same directory.
and put your required modules along with the version number you are using in that file like so,
MySQL-python==1.2.5
PS: I was using MySQLDB module so I put that.
Related
When I am trying to run my application I got the following error
Traceback (most recent call last):
File "./app.py", line 32, in <module>
from pushservices.bootstrap import init_messaging_agents
File "/home/airnotifier/airnotifier/pushservices/bootstrap.py", line 2, in <module>
from .apns import *
File "/home/airnotifier/airnotifier/pushservices/apns.py", line 5, in <module>
from util import json_encode
File "/home/airnotifier/airnotifier/util.py", line 37, in <module>
import tornado
ModuleNotFoundError: No module named 'tornado'
I tried to run this command to install tornado
python3 -m pip install tornado
then I got the following message
Requirement already satisfied: tornado in /usr/local/lib/python3.8/dist-packages (6.0.3)
followed by the following error log
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/pip/_internal/utils/logging.py", line 177, in emit
self.console.print(renderable, overflow="ignore", crop=False, style=style)
File "/usr/local/lib/python3.8/dist-packages/pip/_vendor/rich/console.py", line 1752, in print
extend(render(renderable, render_options))
File "/usr/local/lib/python3.8/dist-packages/pip/_vendor/rich/console.py", line 1390, in render
for render_output in iter_render:
File "/usr/local/lib/python3.8/dist-packages/pip/_internal/utils/logging.py", line 134, in __rich_console__
for line in lines:
File "/usr/local/lib/python3.8/dist-packages/pip/_vendor/rich/segment.py", line 245, in split_lines
for segment in segments:
File "/usr/local/lib/python3.8/dist-packages/pip/_vendor/rich/console.py", line 1368, in render
renderable = rich_cast(renderable)
File "/usr/local/lib/python3.8/dist-packages/pip/_vendor/rich/protocol.py", line 36, in rich_cast
renderable = cast_method()
File "/usr/local/lib/python3.8/dist-packages/pip/_internal/self_outdated_check.py", line 130, in __rich__
pip_cmd = get_best_invocation_for_this_pip()
File "/usr/local/lib/python3.8/dist-packages/pip/_internal/utils/entrypoints.py", line 58, in get_best_invocation_for_this_pip
if found_executable and os.path.samefile(
File "/usr/lib/python3.8/genericpath.py", line 101, in samefile
s2 = os.stat(f2)
FileNotFoundError: [Errno 2] No such file or directory: '/usr/bin/pip3.8'
Call stack:
File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.8/dist-packages/pip/__main__.py", line 31, in <module>
sys.exit(_main())
File "/usr/local/lib/python3.8/dist-packages/pip/_internal/cli/main.py", line 70, in main
return command.main(cmd_args)
File "/usr/local/lib/python3.8/dist-packages/pip/_internal/cli/base_command.py", line 101, in main
return self._main(args)
File "/usr/local/lib/python3.8/dist-packages/pip/_internal/cli/base_command.py", line 223, in _main
self.handle_pip_version_check(options)
File "/usr/local/lib/python3.8/dist-packages/pip/_internal/cli/req_command.py", line 148, in handle_pip_version_check
pip_self_version_check(session, options)
File "/usr/local/lib/python3.8/dist-packages/pip/_internal/self_outdated_check.py", line 237, in pip_self_version_check
logger.info("[present-rich] %s", upgrade_prompt)
File "/usr/lib/python3.8/logging/__init__.py", line 1446, in info
self._log(INFO, msg, args, **kwargs)
File "/usr/lib/python3.8/logging/__init__.py", line 1589, in _log
self.handle(record)
File "/usr/lib/python3.8/logging/__init__.py", line 1599, in handle
self.callHandlers(record)
File "/usr/lib/python3.8/logging/__init__.py", line 1661, in callHandlers
hdlr.handle(record)
File "/usr/lib/python3.8/logging/__init__.py", line 954, in handle
self.emit(record)
File "/usr/local/lib/python3.8/dist-packages/pip/_internal/utils/logging.py", line 179, in emit
self.handleError(record)
Message: '[present-rich] %s'
Arguments: (UpgradePrompt(old='22.1.2', new='22.2.2'),)
can anyone please tell me what should I do and what is the problem ?
Here's a possible scenario that could cause this:
you execute pip install tornado as a user but you run the tornado project as another user.
Sorry for my terrible english, hope you can understand
The module winrm is installed with all its dependencies, but still the celeryd service is not able to run. Here is the error trace:
worker1#man11idm01.stman00.isyntax.net: DOWN /usr/lib64/python2.6/site-packages/cryptography/__init__.py:26: DeprecationWarning: Python 2.6 is no longer supported by the Python core team, please upgrade your Python. A future version of cryptography will drop support for Python 2.6 DeprecationWarning Traceback (most recent call last): File "/usr/lib64/python2.6/runpy.py", line 122, in _run_module_as_main
"__main__", fname, loader, pkg_name) File "/usr/lib64/python2.6/runpy.py", line 34, in _run_code
exec code in run_globals File "/usr/lib/python2.6/site-packages/celery/__main__.py", line 54, in <module>
main() File "/usr/lib/python2.6/site-packages/celery/__main__.py", line 30, in main
main() File "/usr/lib/python2.6/site-packages/celery/bin/celery.py", line 81, in main
cmd.execute_from_commandline(argv) File "/usr/lib/python2.6/site-packages/celery/bin/celery.py", line 769, in execute_from_commandline
super(CeleryCommand, self).execute_from_commandline(argv))) File "/usr/lib/python2.6/site-packages/celery/bin/base.py", line 311, in execute_from_commandline
return self.handle_argv(self.prog_name, argv[1:]) File "/usr/lib/python2.6/site-packages/celery/bin/celery.py", line 761, in handle_argv
return self.execute(command, argv) File "/usr/lib/python2.6/site-packages/celery/bin/celery.py", line 693, in execute
).run_from_argv(self.prog_name, argv[1:], command=argv[0]) File "/usr/lib/python2.6/site-packages/celery/bin/worker.py", line 179, in run_from_argv
return self(*args, **options) File "/usr/lib/python2.6/site-packages/celery/bin/base.py", line 274, in
__call__
ret = self.run(*args, **kwargs) File "/usr/lib/python2.6/site-packages/celery/bin/worker.py", line 212, in run
state_db=self.node_format(state_db, hostname), **kwargs File "/usr/lib/python2.6/site-packages/celery/worker/__init__.py", line 95, in __init__
self.app.loader.init_worker() File "/usr/lib/python2.6/site-packages/celery/loaders/base.py", line 128, in init_worker
self.import_default_modules() File "/usr/lib/python2.6/site-packages/celery/loaders/base.py", line 121, in import_default_modules
tuple(maybe_list(self.app.conf.CELERY_INCLUDE)) File "/usr/lib/python2.6/site-packages/celery/loaders/base.py", line 103, in import_task_module
return self.import_from_cwd(module) File "/usr/lib/python2.6/site-packages/celery/loaders/base.py", line 112, in import_from_cwd
package=package, File "/usr/lib/python2.6/site-packages/celery/utils/imports.py", line 101, in import_from_cwd
return imp(module, package=package) File "/usr/lib64/python2.6/contextlib.py", line 34, in __exit__
self.gen.throw(type, value, traceback) File "/usr/lib/python2.6/site-packages/celery/utils/imports.py", line 64, in cwd_in_path
yield cwd File "/usr/lib/python2.6/site-packages/celery/utils/imports.py", line 101, in import_from_cwd
return imp(module, package=package) File "/usr/lib/python2.6/site-packages/celery/loaders/base.py", line 106, in import_module
return importlib.import_module(module, package=package) File "/usr/lib/python2.6/site-packages/importlib/__init__.py", line 37, in import_module
__import__(name) File "/usr/lib/celery/phim_onsite/operate.py", line 14, in <module>
from phim_onsite.transmit import tx_data, tx_pcm_discovery, tx_deployment_status File "/usr/lib/celery/phim_onsite/transmit.py", line 19, in <module>
from scanline.trinity import scanline_endpoints, scanline_scanner File "/usr/lib/python2.6/site-packages/scanline/trinity.py", line 15, in <module>
from scanline.product.isp import ISPProductScanner File "/usr/lib/python2.6/site-packages/scanline/product/isp.py", line 17, in <module>
from scanline.host.isp import ISPHostScanner File "/usr/lib/python2.6/site-packages/scanline/host/isp.py", line 6, in <module>
from scanline.utilities.win_rm import extract_credentials File "/usr/lib/python2.6/site-packages/scanline/utilities/win_rm.py", line 2, in <module>
import winrm ImportError: No module named winrm
Restarting node worker1#man11idm01.stman00.isyntax.net: * Child terminated with errorcode 255
FAILED
+ exit 0
I've been following several tutorials and I'm obtaining SyntaxError continuously:
C:\Users\Luis\mercado>scrapy crawl mercado -t csv
Traceback (most recent call last):
File "C:\Users\Luis\Anaconda2\Scripts\scrapy-script.py", line 5, in
<module>
sys.exit(scrapy.cmdline.execute())
File "C:\Users\Luis\Anaconda2\lib\site-packages\scrapy\cmdline.py", line
149,
in execute
cmd.crawler_process = CrawlerProcess(settings)
File "C:\Users\Luis\Anaconda2\lib\site-packages\scrapy\crawler.py", line
249,
in __init__
super(CrawlerProcess, self).__init__(settings)
File "C:\Users\Luis\Anaconda2\lib\site-packages\scrapy\crawler.py", line
137,
in __init__
self.spider_loader = _get_spider_loader(settings)
File "C:\Users\Luis\Anaconda2\lib\site-packages\scrapy\crawler.py", line
336,
in _get_spider_loader
return loader_cls.from_settings(settings.frozencopy())
File "C:\Users\Luis\Anaconda2\lib\site-packages\scrapy\spiderloader.py",
line 61, in from_settings return cls(settings)
File "C:\Users\Luis\Anaconda2\lib\site-packages\scrapy\spiderloader.py",
line 25, in __init__
self._load_all_spiders()
File "C:\Users\Luis\Anaconda2\lib\site-packages\scrapy\spiderloader.py",
line 47, in _load_all_spiders
for module in walk_modules(name):
File "C:\Users\Luis\Anaconda2\lib\site-packages\scrapy\utils\misc.py",
line 71
, in walk_modules
submod = import_module(fullpath)
File "C:\Users\Luis\Anaconda2\lib\importlib\__init__.py", line 37, in
import_module
__import__(name)
File "C:\Users\Luis\mercado\mercado\spiders\spider.py", line 7
Class MercadoSpider(CrawlSpider):
SyntaxError: invalid syntax
Could you help me with that please?
In Python, class is always written in lower case. You have a syntax error in your Python.
Trying to run a simple example according to
http://celery.readthedocs.org/en/latest/getting-started/next-steps.html
I'm getting ImportError: No module named celery (probably due to from celery import Celery).
On python shell it works perfectly.
Running: running python <proj dir>/celery_manager.py worker does work...
What am I doing wrong?
The full dump:
celery multi v3.1.16 (Cipater)
> Starting nodes...
Traceback (most recent call last):
File "/usr/lib64/python2.6/runpy.py", line 122, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/usr/lib64/python2.6/runpy.py", line 34, in _run_code
exec code in run_globals
File "/usr/lib/python2.6/site-packages/celery/__main__.py", line 54, in <module>
main()
File "/usr/lib/python2.6/site-packages/celery/__main__.py", line 30, in main
main()
File "/usr/lib/python2.6/site-packages/celery/bin/celery.py", line 81, in main
cmd.execute_from_commandline(argv)
File "/usr/lib/python2.6/site-packages/celery/bin/celery.py", line 769, in execute_from_commandline
super(CeleryCommand, self).execute_from_commandline(argv)))
File "/usr/lib/python2.6/site-packages/celery/bin/base.py", line 305, in execute_from_commandline
argv = self.setup_app_from_commandline(argv)
File "/usr/lib/python2.6/site-packages/celery/bin/base.py", line 465, in setup_app_from_commandline
self.app = self.find_app(app)
File "/usr/lib/python2.6/site-packages/celery/bin/base.py", line 485, in find_app
return find_app(app, symbol_by_name=self.symbol_by_name)
File "/usr/lib/python2.6/site-packages/celery/app/utils.py", line 248, in find_app
symbol_by_name=symbol_by_name, imp=imp,
File "/usr/lib/python2.6/site-packages/celery/app/utils.py", line 232, in find_app
sym = imp(app)
File "/usr/lib/python2.6/site-packages/celery/utils/imports.py", line 101, in import_from_cwd
return imp(module, package=package)
File "/usr/lib64/python2.6/contextlib.py", line 34, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/lib/python2.6/site-packages/celery/utils/imports.py", line 64, in cwd_in_path
yield cwd
File "/usr/lib/python2.6/site-packages/celery/utils/imports.py", line 101, in import_from_cwd
return imp(module, package=package)
File "/usr/lib/python2.6/site-packages/importlib/__init__.py", line 37, in import_module
__import__(name)
ImportError: No module named celery
I installed dajaxice into a django project. When i run the web using manage.py runserver, all went well. Now I try to deploy it with apache/wsgi. When I run manage.py collectstatic, error occur:
ImportError: cannot import name Bag
where Bag is one model within my app.
I tried
manage.py shell
then import Bag, i works well.
And I tried comment out DajaxiceFinder from settings.py, collectstatic finished normally.
Could any one help me out?
Thanks
The full stack of ImportError:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/django/core/management/base.py", line 222, in run_from_argv
self.execute(*args, **options.__dict__)
File "/usr/local/lib/python2.7/dist-packages/django/core/management/base.py", line 255, in execute
output = self.handle(*args, **options)
File "/usr/local/lib/python2.7/dist-packages/django/core/management/base.py", line 385, in handle
return self.handle_noargs(**options)
File "/usr/local/lib/python2.7/dist-packages/django/contrib/staticfiles/management/commands/collectstatic.py", line 164, in handle_noargs
collected = self.collect()
File "/usr/local/lib/python2.7/dist-packages/django/contrib/staticfiles/management/commands/collectstatic.py", line 114, in collect
handler(path, prefixed_path, storage)
File "/usr/local/lib/python2.7/dist-packages/django/contrib/staticfiles/management/commands/collectstatic.py", line 285, in copy_file
if not self.delete_file(path, prefixed_path, source_storage):
File "/usr/local/lib/python2.7/dist-packages/django/contrib/staticfiles/management/commands/collectstatic.py", line 224, in delete_file
source_last_modified = source_storage.modified_time(path)
File "/usr/local/lib/python2.7/dist-packages/django/core/files/storage.py", line 277, in modified_time
return datetime.fromtimestamp(os.path.getmtime(self.path(name)))
File "/usr/local/lib/python2.7/dist-packages/dajaxice/finders.py", line 55, in path
path = self.get_or_create_file(name)
File "/usr/local/lib/python2.7/dist-packages/dajaxice/finders.py", line 21, in get_or_create_file
data = getattr(self, self.files[path])()
File "/usr/local/lib/python2.7/dist-packages/dajaxice/finders.py", line 68, in dajaxice_core_js
dajaxice_autodiscover()
File "/usr/local/lib/python2.7/dist-packages/dajaxice/core/Dajaxice.py", line 135, in dajaxice_autodiscover
import_module("%s.ajax" % app)
File "/usr/local/lib/python2.7/dist-packages/django/utils/importlib.py", line 35, in import_module
__import__(name)
File "/home/emini/project/emini/trunk/emini/repairment/ajax.py", line 4, in <module>
from emini.repairment.models import Bag
File "/home/emini/project/emini/trunk/emini/repairment/models.py", line 9, in <module>
class Bag(models.Model):
File "/home/emini/project/emini/trunk/emini/repairment/models.py", line 90, in Bag
shop_id = models.IntegerField(default=0, db_index=True, choices=DataConfig.get_valid_conf_as_choices_by_group(DataConfig.GROUP_NAME_SHOP), verbose_name="店铺")
File "/home/emini/project/emini/trunk/emini/sysconfig/models.py", line 62, in get_valid_conf_as_choices_by_group
data_configs = cls.objects.filter(group_name=group_name, enabled=True).order_by('-id')
File "/usr/local/lib/python2.7/dist-packages/django/db/models/manager.py", line 155, in filter
return self.get_query_set().filter(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/django/db/models/query.py", line 669, in filter
return self._filter_or_exclude(False, *args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/django/db/models/query.py", line 687, in _filter_or_exclude
clone.query.add_q(Q(*args, **kwargs))
File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/query.py", line 1271, in add_q
can_reuse=used_aliases, force_having=force_having)
File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/query.py", line 1139, in add_filter
process_extras=process_extras)
File "/usr/local/lib/python2.7/dist-packages/django/db/models/sql/query.py", line 1325, in setup_joins
field, model, direct, m2m = opts.get_field_by_name(name)
File "/usr/local/lib/python2.7/dist-packages/django/db/models/options.py", line 351, in get_field_by_name
cache = self.init_name_map()
File "/usr/local/lib/python2.7/dist-packages/django/db/models/options.py", line 380, in init_name_map
for f, model in self.get_all_related_m2m_objects_with_model():
File "/usr/local/lib/python2.7/dist-packages/django/db/models/options.py", line 469, in get_all_related_m2m_objects_with_model
cache = self._fill_related_many_to_many_cache()
File "/usr/local/lib/python2.7/dist-packages/django/db/models/options.py", line 483, in _fill_related_many_to_many_cache
for klass in get_models(only_installed=False):
File "/usr/local/lib/python2.7/dist-packages/django/db/models/loading.py", line 197, in get_models
self._populate()
File "/usr/local/lib/python2.7/dist-packages/django/db/models/loading.py", line 75, in _populate
self.load_app(app_name)
File "/usr/local/lib/python2.7/dist-packages/django/db/models/loading.py", line 96, in load_app
models = import_module('.models', app_name)
File "/usr/local/lib/python2.7/dist-packages/django/utils/importlib.py", line 35, in import_module
__import__(name)
File "/home/emini/project/emini/trunk/emini/express/models.py", line 6, in <module>
from emini.repairment.models import Bag
ImportError: cannot import name Bag
Enviroment:
Django==1.5.4
django-dajaxice==0.5.5
I try to delete the import Bag from express/models.py, then collectstatic successed.
Structure of my apps:
emini
express
models.py
repairment
models.py
in emini.express.models:
from emini.repairment.models import Bag
class Package(models.Model):
foo = models.CharField(max_length=20)
class BagPackageLink(models.Model):
bag = models.ForeignKey(Bag)
package = models.ForeignKey(Package)
class Meta:
unique_together = (("bag", "package"),)
in emini.repairment.models
class Bag(models.Model):
foo = models.CharField(max_length=20)
Am i messing up with 'overlap import'?
But why the runserver and shell can import Bag model?