I want to add columns to a table in my database dynamically since I don't want to have to specify all columns when I set up the table in the SQLalchemy class
In order to solve this I am using alembic to add columns to the table but I am having problems.
In my python script I have defined a class as below.
DATABASE_URL_dev = "postgresql+psycopg2://user:password#localhost:5432/testdb"
engine = create_engine(DATABASE_URL_dev)
Base = declarative_base(engine)
class my_class1(Base):
__tablename__ = "test1"
id = Column(Integer, primary_key=True)
list = Column(String)
def loadsession():
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
return session
session = loadsession()
Now I want to add a new column ("new_column") to the table test1 in the database.
Using Alembic I then followed the instructions at https://alembic.sqlalchemy.org/en/latest/tutorial.html and did the following:
Installed alembic by running pip install alembic
Created an environment using alembic init alembic
Edited the .ini file by inputting "postgresql+psycopg2://user:password#localhost:5432/testdb" at sqlalchemy.url
Created a migration script by running alembic revision -m "add a column "
I then edited the script by adding the following "op.add_column('test1', sa.Column('new_column', sa.String))" in upgrade()
Running first migration with "alembic upgrade head"
This gave me the following error:
Traceback (most recent call last):
File "c:\users\user\appdata\local\programs\python\python38-32\lib\runpy.py", line 193, in _run_module_as_main
return _run_code(code, main_globals, None,
File "c:\users\user\appdata\local\programs\python\python38-32\lib\runpy.py", line 86, in _run_code
exec(code, run_globals)
File "C:\Users\user\AppData\Local\Programs\Python\Python38-32\Scripts\alembic.exe\__main__.py", line 9, in <module>
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\config.py", line 577, in main
CommandLine(prog=prog).main(argv=argv)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\config.py", line 571, in main
self.run_cmd(cfg, options)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\config.py", line 548, in run_cmd
fn(
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\command.py", line 298, in upgrade
script.run_env()
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\script\base.py", line 489, in run_env
util.load_python_file(self.dir, "env.py")
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\util\pyfiles.py", line 98, in load_python_file
module = load_module_py(module_id, path)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\alembic\util\compat.py", line 184, in load_module_py
spec.loader.exec_module(module)
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "y\env.py", line 77, in <module>
run_migrations_online()
File "y\env.py", line 65, in run_migrations_online
with connectable.connect() as connection:
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 2218, in connect
return self._connection_cls(self, **kwargs)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 103, in __init__
else engine.raw_connection()
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 2317, in raw_connection
return self._wrap_pool_connect(
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 2288, in _wrap_pool_connect
Connection._handle_dbapi_exception_noconnection(
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 1554, in _handle_dbapi_exception_noconnection
util.raise_(
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 2285, in _wrap_pool_connect
return fn()
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 303, in unique_connection
return _ConnectionFairy._checkout(self)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 773, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\impl.py", line 238, in _do_get
return self._create_connection()
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 657, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.raise_(
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\pool\base.py", line 652, in __connect
connection = pool._invoke_creator(self)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "c:\users\user\appdata\local\programs\python\python38-32\lib\site-packages\sqlalchemy\engine\default.py", line 488, in connect
return self.dbapi.connect(*cargs, **cparams)
File "c:\users\c\appdata\local\programs\python\python38-32\lib\site-packages\psycopg2\__init__.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError)
(Background on this error at: http://sqlalche.me/e/e3q8)
Questions
Is this the right way to proceed in order to add columns dynamically or should it be done in another way?
How do I resolve the sqlalchemy operationalerror?
Thanks
Try this in step 5
op.add_column('test1', sa.Column('new_column', sa.String()))
Related
Currently developing for python 3.9.6 with sqlalchemy version 1.4.23 and whenever I use sqlalchemy to connect to a database, I get an error with this stack trace:
Traceback (most recent call last):
File "/mnt/c/Importer/common/db.py", line 106, in db_cur
with engine.connect() as con:
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3166, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 96, in __init__
else engine.raw_connection()
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3245, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3212, in _wrap_pool_connect
return fn()
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 307, in connect
return _ConnectionFairy._checkout(self)
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 767, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 425, in checkout
rec = pool._do_get()
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 146, in _do_get
self._dec_overflow()
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__
compat.raise_(
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 143, in _do_get
return self._create_connection()
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 253, in _create_connection
return _ConnectionRecord(self)
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 368, in __init__
self.__connect()
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 622, in __connect
pool.dispatch.connect.for_modify(
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/event/attr.py", line 329, in _exec_w_sync_on_first_run
self(*args, **kw)
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/event/attr.py", line 343, in __call__
fn(*args, **kw)
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 1691, in go
return once_fn(*arg, **kw)
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 674, in first_connect
dialect.initialize(c)
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/dialects/mysql/base.py", line 2961, in initialize
default.DefaultDialect.initialize(self, connection)
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 407, in initialize
self.default_isolation_level = self.get_default_isolation_level(
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 473, in get_default_isolation_level
return self.get_isolation_level(dbapi_conn)
File "/home/jimmy/.local/share/virtualenvs/Importer-0-6a4YAh/lib/python3.9/site-packages/sqlalchemy/dialects/mysql/base.py", line 2720, in get_isolation_level
val = row[0]
KeyError: 0
I can fix the issue by manually changing the file from
val = row[0]
to
val = row['##transaction_isolation']
but I don't want to have to manually change dependency files after installing them. I have tried manually setting the transaction isolation when creating the engine, but that doesn't prevent get_isolation_level() from being called so the error persists. Is sqlalchemy not compatible with python 3.9.6? Does this have anything to do with the mysql server version?
Problem was with the DictCursor being used in the connection. Changing _fetch_type for SELECT ##transaction_isolation to be a tuple instead of a dictionary solved the issue.
When I run this basic code to access my AWS redshift data warehouse, I get this error:
Traceback (most recent call last):
File "<ipython-input-40-dcebeb0708d9>", line 25, in <module>
engine = sqlalchemy.create_engine(URL)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\__init__.py", line 479, in create_engine
return strategy.create(*args, **kwargs)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\strategies.py", line 56, in create
plugins = u._instantiate_plugins(kwargs)
TypeError: _instantiate_plugins() missing 1 required positional argument: 'kwargs'
This is the code I'm running. I originally planned on using psycopg2 to access the database but psycopg2 returns the sql query as a list with funky formatting; I'd have to do too much work to extract the data into a dataframe. I'm hoping sqlalchemy produces and output that's easier to work with.
import sqlalchemy
from sqlalchemy import Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base
import pandas as pd
from sqlalchemy.engine.url import URL
config = dict(
drivername='driver',
username= username_u,
password= password_u,
host='rs-data.prod-lde.bsp.gsa.gov',
port='5439',
database='edw',
query={'encoding': 'utf-8'}
)
url = URL(**config)
engine = sqlalchemy.create_engine(url) # Having "URL" instead of "url" caused the original problem
Thanks in advance.
>>>>>Update<<<<<:
I fixed my silly mistake and the new code is this:
import sqlalchemy
from sqlalchemy import Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base
import pandas as pd
from sqlalchemy.engine.url import URL
config = dict(
drivername='postgresql',
username= username_u,
password= password_u,
host='rs-data.prod-lde.bsp.gsa.gov',
port='5439',
database='edw',
query={'encoding': 'utf-8'}
)
url = URL(**config)
engine = sqlalchemy.create_engine(url)
data_frame = pd.read_sql('select date_signed from edw.fpds.fpds_atom limit 5;', engine)
Now I'm getting this error:
ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "encoding"
(Background on this error at: http://sqlalche.me/e/f405)
I think the problem lies in the config dict values. I first had it as drivername='driver' and that caused this error:
Traceback (most recent call last):
File "<ipython-input-48-dcb4b19c1bd3>", line 25, in <module>
engine = sqlalchemy.create_engine(url)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\__init__.py", line 479, in create_engine
return strategy.create(*args, **kwargs)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\strategies.py", line 61, in create
entrypoint = u._get_entrypoint()
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\url.py", line 172, in _get_entrypoint
cls = registry.load(name)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\util\langhelpers.py", line 240, in load
"Can't load plugin: %s:%s" % (self.group, name)
NoSuchModuleError: Can't load plugin: sqlalchemy.dialects:driver
so I changed it to drivername='postgresql' and that produced this error:
Traceback (most recent call last):
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 2276, in _wrap_pool_connect
return fn()
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\base.py", line 773, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\impl.py", line 139, in _do_get
self._dec_overflow()
File "C:\Anaconda3\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
raise value
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\impl.py", line 136, in _do_get
return self._create_connection()
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\base.py", line 652, in __connect
connection = pool._invoke_creator(self)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\default.py", line 489, in connect
return self.dbapi.connect(*cargs, **cparams)
File "C:\Anaconda3\lib\site-packages\psycopg2\__init__.py", line 126, in connect
dsn = _ext.make_dsn(dsn, **kwargs)
File "C:\Anaconda3\lib\site-packages\psycopg2\extensions.py", line 175, in make_dsn
parse_dsn(dsn)
ProgrammingError: invalid dsn: invalid connection option "encoding"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<ipython-input-53-3b57714b0ad8>", line 28, in <module>
data_frame = pd.read_sql('select date_signed from edw.fpds.fpds_atom limit 5;', engine)
File "C:\Anaconda3\lib\site-packages\pandas\io\sql.py", line 438, in read_sql
chunksize=chunksize,
File "C:\Anaconda3\lib\site-packages\pandas\io\sql.py", line 1218, in read_query
result = self.execute(*args)
File "C:\Anaconda3\lib\site-packages\pandas\io\sql.py", line 1087, in execute
return self.connectable.execute(*args, **kwargs)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 2181, in execute
connection = self._contextual_connect(close_with_result=True)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 2242, in _contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 2280, in _wrap_pool_connect
e, dialect, self
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1547, in _handle_dbapi_exception_noconnection
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 2276, in _wrap_pool_connect
return fn()
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\base.py", line 363, in connect
return _ConnectionFairy._checkout(self)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\base.py", line 773, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\base.py", line 492, in checkout
rec = pool._do_get()
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\impl.py", line 139, in _do_get
self._dec_overflow()
File "C:\Anaconda3\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
raise value
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\impl.py", line 136, in _do_get
return self._create_connection()
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\base.py", line 308, in _create_connection
return _ConnectionRecord(self)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\base.py", line 437, in __init__
self.__connect(first_connect_check=True)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\pool\base.py", line 652, in __connect
connection = pool._invoke_creator(self)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\strategies.py", line 114, in connect
return dialect.connect(*cargs, **cparams)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\default.py", line 489, in connect
return self.dbapi.connect(*cargs, **cparams)
File "C:\Anaconda3\lib\site-packages\psycopg2\__init__.py", line 126, in connect
dsn = _ext.make_dsn(dsn, **kwargs)
File "C:\Anaconda3\lib\site-packages\psycopg2\extensions.py", line 175, in make_dsn
parse_dsn(dsn)
ProgrammingError: (psycopg2.ProgrammingError) invalid dsn: invalid connection option "encoding"
(Background on this error at: http://sqlalche.me/e/f405)
The key question I guess is what should I be using for the drivername parameter?
I am running a Django and attempting to connect to a MySQL server at my university.
First of all:
In order to connect the database, I normally SSH into a remote server (with authentication) of theirs, and then run MySQL and enter my MySQL authentications.
This works without an issue, in Terminal as well as DataGrip.
Now, using PyCharm, I have installed mysqlclient in the intepreter and changed my
settings.py:
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': '[DB_NAME]',
'USER': '[DB_USERNAME]',
'PASSWORD': '[DB_PASSWORD]',
'HOST': '[REMOTE SERVER ADDRESS HOSTING MYSQL]',
'PORT': '3306',
}
}
Also using PyCharm, I have even opened up a data source in the Data Sources Tool Window and connected through SSH tunnel and MySQL authentication to the database. This is successful, and it is possible to make queries.
However, I try to run:
manage.py check
and I receive:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site- packages/django/db/backends/base/base.py", line 213, in ensure_connection
self.connect()
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/base/base.py", line 189, in connect
self.connection = self.get_new_connection(conn_params)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/mysql/base.py", line 274, in get_new_connection
conn = Database.connect(**conn_params)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/MySQLdb/__init__.py", line 86, in Connect
return Connection(*args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/MySQLdb/connections.py", line 204, in __init__
super(Connection, self).__init__(*args, **kwargs2)
_mysql_exceptions.OperationalError: (2003, "Can't connect to MySQL server on '[REMOTE SERVER ADDRESS HOSTING MYSQL]' (60)")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Applications/PyCharm.app/Contents/helpers/pycharm/django_manage.py", line 43, in <module>
run_module(manage_file, None, '__main__', True)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/runpy.py", line 205, in run_module
return _run_module_code(code, init_globals, run_name, mod_spec)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/runpy.py", line 96, in _run_module_code
mod_name, mod_spec, pkg_name, script_name)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/Users/skillapop/Desktop/streamcity_project/manage.py", line 22, in <module>
execute_from_command_line(sys.argv)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/core/management/__init__.py", line 363, in execute_from_command_line
utility.execute()
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/core/management/__init__.py", line 355, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/core/management/base.py", line 283, in run_from_argv
self.execute(*args, **cmd_options)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/core/management/base.py", line 330, in execute
output = self.handle(*args, **options)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/core/management/commands/check.py", line 68, in handle
fail_level=getattr(checks, options['fail_level']),
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/core/management/base.py", line 359, in check
include_deployment_checks=include_deployment_checks,
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/core/management/base.py", line 346, in _run_checks
return checks.run_checks(**kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/core/checks/registry.py", line 81, in run_checks
new_errors = check(app_configs=app_configs)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/core/checks/model_checks.py", line 30, in check_all_models
errors.extend(model.check(**kwargs))
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/models/base.py", line 1283, in check
errors.extend(cls._check_fields(**kwargs))
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/models/base.py", line 1358, in _check_fields
errors.extend(field.check(**kwargs))
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/models/fields/__init__.py", line 913, in check
errors = super(AutoField, self).check(**kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/models/fields/__init__.py", line 219, in check
errors.extend(self._check_backend_specific_checks(**kwargs))
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/models/fields/__init__.py", line 322, in _check_backend_specific_checks
return connections[db].validation.check_field(self, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/mysql/validation.py", line 49, in check_field
field_type = field.db_type(self.connection)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/models/fields/__init__.py", line 644, in db_type
return connection.data_types[self.get_internal_type()] % data
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/utils/functional.py", line 35, in __get__
res = instance.__dict__[self.name] = self.func(instance)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/mysql/base.py", line 174, in data_types
if self.features.supports_microsecond_precision:
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/utils/functional.py", line 35, in __get__
res = instance.__dict__[self.name] = self.func(instance)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/mysql/features.py", line 53, in supports_microsecond_precision
return self.connection.mysql_version >= (5, 6, 4) and Database.version_info >= (1, 2, 5)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/utils/functional.py", line 35, in __get__
res = instance.__dict__[self.name] = self.func(instance)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/mysql/base.py", line 385, in mysql_version
with self.temporary_connection() as cursor:
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/contextlib.py", line 81, in __enter__
return next(self.gen)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/base/base.py", line 591, in temporary_connection
cursor = self.cursor()
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/base/base.py", line 254, in cursor
return self._cursor()
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/base/base.py", line 229, in _cursor
self.ensure_connection()
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/base/base.py", line 213, in ensure_connection
self.connect()
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/utils.py", line 94, in __exit__
six.reraise(dj_exc_type, dj_exc_value, traceback)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/utils/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/base/base.py", line 213, in ensure_connection
self.connect()
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/base/base.py", line 189, in connect
self.connection = self.get_new_connection(conn_params)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/django/db/backends/mysql/base.py", line 274, in get_new_connection
conn = Database.connect(**conn_params)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/MySQLdb/__init__.py", line 86, in Connect
return Connection(*args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/MySQLdb/connections.py", line 204, in __init__
super(Connection, self).__init__(*args, **kwargs2)
django.db.utils.OperationalError: (2003, "Can't connect to MySQL server on '[REMOTE SERVER ADDRESS HOSTING MYSQL]' (60)")
It is also important to note that I HAVE connected a Django project to this exact same server and database before in a test run. Now, that test project and the new project does NOT connect.
Any help would be appreciated!
Background
I'm trying to use a PostgreSQL back-end instead of Sqlite in this Flask + RESTplus server example.
I faced an issue with the PasswordType db column type. In order to make it work, I had to change the following code in app/modules/users/models.py
password = db.Column(
column_types.PasswordType(
max_length=128,
schemes=('bcrypt', )
),
nullable=False
)
to
password = db.Column(db.String(length=128), nullable=False)
which is really bad as passwords will be stored in clear text... and I need your help!
After changing the line 13 to 15 in tasks/app/db_templates/flask/script.py.mako
to
from alembic import op
import sqlalchemy as sa
import sqlalchemy_utils
${imports if imports else ""}
I get the following error message, apparently related to passlib:
2016-08-31 23:18:52,751 [INFO] [alembic.runtime.migration] Context impl PostgresqlImpl.
2016-08-31 23:18:52,752 [INFO] [alembic.runtime.migration] Will assume transactional DDL.
2016-08-31 23:18:52,759 [INFO] [alembic.runtime.migration] Running upgrade -> 99b329343a41, empty message
Traceback (most recent call last):
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/type_api.py", line 359, in dialect_impl
return dialect._type_memos[self]['impl']
File "/usr/local/lib/python3.5/weakref.py", line 365, in __getitem__
return self.data[ref(key)]
KeyError: <weakref at 0x7fde70d524a8; to 'PasswordType' at 0x7fde70a840b8>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/bin/invoke", line 11, in <module>
sys.exit(program.run())
File "/usr/local/lib/python3.5/site-packages/invoke/program.py", line 270, in run
self.execute()
File "/usr/local/lib/python3.5/site-packages/invoke/program.py", line 381, in execute
executor.execute(*self.tasks)
File "/usr/local/lib/python3.5/site-packages/invoke/executor.py", line 113, in execute
result = call.task(*args, **call.kwargs)
File "/usr/local/lib/python3.5/site-packages/invoke/tasks.py", line 111, in __call__
result = self.body(*args, **kwargs)
File "/usr/src/app/tasks/app/run.py", line 35, in run
context.invoke_execute(context, 'app.db.upgrade')
File "/usr/src/app/tasks/__init__.py", line 72, in invoke_execute
results = Executor(namespace, config=context.config).execute((command_name, kwargs))
File "/usr/local/lib/python3.5/site-packages/invoke/executor.py", line 113, in execute
result = call.task(*args, **call.kwargs)
File "/usr/local/lib/python3.5/site-packages/invoke/tasks.py", line 111, in __call__
result = self.body(*args, **kwargs)
File "/usr/src/app/tasks/app/db.py", line 177, in upgrade
command.upgrade(config, revision, sql=sql, tag=tag)
File "/usr/local/lib/python3.5/site-packages/alembic/command.py", line 174, in upgrade
script.run_env()
File "/usr/local/lib/python3.5/site-packages/alembic/script/base.py", line 407, in run_env
util.load_python_file(self.dir, 'env.py')
File "/usr/local/lib/python3.5/site-packages/alembic/util/pyfiles.py", line 93, in load_python_file
module = load_module_py(module_id, path)
File "/usr/local/lib/python3.5/site-packages/alembic/util/compat.py", line 68, in load_module_py
module_id, path).load_module(module_id)
File "<frozen importlib._bootstrap_external>", line 385, in _check_name_wrapper
File "<frozen importlib._bootstrap_external>", line 806, in load_module
File "<frozen importlib._bootstrap_external>", line 665, in load_module
File "<frozen importlib._bootstrap>", line 268, in _load_module_shim
File "<frozen importlib._bootstrap>", line 693, in _load
File "<frozen importlib._bootstrap>", line 673, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 662, in exec_module
File "<frozen importlib._bootstrap>", line 222, in _call_with_frames_removed
File "migrations/env.py", line 93, in <module>
run_migrations_online()
File "migrations/env.py", line 86, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/usr/local/lib/python3.5/site-packages/alembic/runtime/environment.py", line 797, in run_migrations
self.get_context().run_migrations(**kw)
File "/usr/local/lib/python3.5/site-packages/alembic/runtime/migration.py", line 312, in run_migrations
step.migration_fn(**kw)
File "/usr/src/app/migrations/versions/99b329343a41_.py", line 47, in upgrade
sa.UniqueConstraint('username')
File "<string>", line 8, in create_table
File "<string>", line 3, in create_table
File "/usr/local/lib/python3.5/site-packages/alembic/operations/ops.py", line 1098, in create_table
return operations.invoke(op)
File "/usr/local/lib/python3.5/site-packages/alembic/operations/base.py", line 318, in invoke
return fn(self, operation)
File "/usr/local/lib/python3.5/site-packages/alembic/operations/toimpl.py", line 101, in create_table
operations.impl.create_table(table)
File "/usr/local/lib/python3.5/site-packages/alembic/ddl/impl.py", line 194, in create_table
self._exec(schema.CreateTable(table))
File "/usr/local/lib/python3.5/site-packages/alembic/ddl/impl.py", line 118, in _exec
return conn.execute(construct, *multiparams, **params)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/engine/base.py", line 914, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/ddl.py", line 68, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/engine/base.py", line 962, in _execute_ddl
compiled = ddl.compile(dialect=dialect)
File "<string>", line 1, in <lambda>
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/elements.py", line 494, in compile
return self._compiler(dialect, bind=bind, **kw)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/ddl.py", line 26, in _compiler
return dialect.ddl_compiler(dialect, self, **kw)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/compiler.py", line 190, in __init__
self.string = self.process(self.statement, **compile_kwargs)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/compiler.py", line 213, in process
return obj._compiler_dispatch(self, **kwargs)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/compiler.py", line 2157, in visit_create_table
and not first_pk)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/compiler.py", line 213, in process
return obj._compiler_dispatch(self, **kwargs)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/compiler.py", line 2188, in visit_create_column
first_pk=first_pk
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/dialects/postgresql/base.py", line 1580, in get_column_specification
impl_type = column.type.dialect_impl(self.dialect)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/type_api.py", line 361, in dialect_impl
return self._dialect_info(dialect)['impl']
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/type_api.py", line 403, in _dialect_info
impl = self._gen_dialect_impl(dialect)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy/sql/type_api.py", line 763, in _gen_dialect_impl
typedesc = self.load_dialect_impl(dialect).dialect_impl(dialect)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy_utils/types/password.py", line 194, in load_dialect_impl
impl = postgresql.BYTEA(self.length)
File "/usr/local/lib/python3.5/site-packages/sqlalchemy_utils/types/password.py", line 168, in length
self._max_length = self.calculate_max_length()
File "/usr/local/lib/python3.5/site-packages/sqlalchemy_utils/types/password.py", line 176, in calculate_max_length
for name in self.context.schemes():
File "/usr/local/lib/python3.5/site-packages/passlib/context.py", line 2714, in __getattribute__
self._lazy_init()
File "/usr/local/lib/python3.5/site-packages/passlib/context.py", line 2708, in _lazy_init
super(LazyCryptContext, self).__init__(**kwds)
File "/usr/local/lib/python3.5/site-packages/passlib/context.py", line 1707, in __init__
self.load(kwds)
File "/usr/local/lib/python3.5/site-packages/passlib/context.py", line 1896, in load
config = _CryptConfig(source)
File "/usr/local/lib/python3.5/site-packages/passlib/context.py", line 1019, in __init__
self._init_options(source)
File "/usr/local/lib/python3.5/site-packages/passlib/context.py", line 1097, in _init_options
key, value = norm_context_option(key, value)
File "/usr/local/lib/python3.5/site-packages/passlib/context.py", line 1162, in _norm_context_option
raise KeyError("unknown CryptContext keyword: %r" % (key,))
KeyError: "unknown CryptContext keyword: 'length'"
Any idea? Thanks in advance for your help!
I see that you are having a new migration 99b329343a41_.py (it doesn't exist in my Flask-RESTplus-example-server). Please, review your new migration and remove everything related to PasswordType. It is a bug in SQLAlchemy-Utils, which doesn't play nice with Alembic: https://github.com/kvesteri/sqlalchemy-utils/issues/106
UPDATE: I have submitted a PR to solve this issue: https://github.com/kvesteri/sqlalchemy-utils/pull/254
I have been looking for this error for a while but nothing seems to solve my problem, the thing is..
I was using sqlite3 with flask, but it seems like sqlite3 does not support migrations, so I decided to change to Mysql. I have installed LAMPP, and I created a symbolik link from:
/opt/lampp/var/mysql/mysql.sock
to:
/var/run/mysqld/mysql.sock
but still I get this error (it is a little long but the trouble relapse on the last line):
Traceback (most recent call last):
File "db_create.py", line 5, in <module>
db.create_all()
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/flask_sqlalchemy/__init__.py", line 895, in create_all
self._execute_for_all_tables(app, bind, 'create_all')
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/flask_sqlalchemy/__init__.py", line 887, in _execute_for_all_tables
op(bind=self.get_engine(app, bind), **extra)
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/sql/schema.py", line 3634, in create_all
tables=tables)
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1851, in _run_visitor
with self._optional_conn_ctx_manager(connection) as conn:
File "/usr/lib/python2.7/contextlib.py", line 17, in __enter__
return self.gen.next()
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1844, in _optional_conn_ctx_manager
with self.contextual_connect() as conn:
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 2035, in contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 2074, in _wrap_pool_connect
e, dialect, self)
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1405, in _handle_dbapi_exception_noconnection
exc_info
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 2070, in _wrap_pool_connect
return fn()
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 376, in connect
return _ConnectionFairy._checkout(self)
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 708, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 480, in checkout
rec = pool._do_get()
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 1055, in _do_get
self._dec_overflow()
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 1052, in _do_get
return self._create_connection()
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 323, in _create_connection
return _ConnectionRecord(self)
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 449, in __init__
self.connection = self.__connect()
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 602, in __connect
connection = self.__pool._invoke_creator(self)
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/strategies.py", line 97, in connect
return dialect.connect(*cargs, **cparams)
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 385, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/MySQLdb/__init__.py", line 81, in Connect
return Connection(*args, **kwargs)
File "/home/gerardo/Documentos/python_web_dev/flask-intro/venv/local/lib/python2.7/site-packages/MySQLdb/connections.py", line 193, in __init__
super(Connection, self).__init__(*args, **kwargs2)
sqlalchemy.exc.OperationalError: (_mysql_exceptions.OperationalError) (2002, "Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2)")
On my research, I found that I have to change the line:
socket = /var/run/mysqld/mysqld.sock
placed on /etc/mysql/my.cnf to:
socket = /opt/lampp/var/mysql/mysql.sock
(The path were "mysql.sock" is located), but still it does not solve my problem. Thank you!
In the database configuration. Replace localhost with 127.0.0.1:
environ.get('APP_DATABASE_HOST', 'localhost')
to
environ.get('APP_DATABASE_HOST', '127.0.0.1')