Apache Airflow broken migration prevents from running standalone mode - python

I've installed Apache Airflow locally, using Pip, and run in standalone mode like instructed in the Docs.
The problem
After the restart, trying to run again airflow standalone, gives me this error:
standalone | Starting Airflow Standalone
standalone | Checking database is initialized
INFO [alembic.runtime.migration] Context impl SQLiteImpl.
INFO [alembic.runtime.migration] Will assume non-transactional DDL.
INFO [alembic.runtime.migration] Running upgrade 127d2bf2dfa7 -> cc1e65623dc7, add max tries column to task instance
Traceback (most recent call last):
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: duplicate column name: max_tries
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/root/miniconda3/envs/localspark/bin/airflow", line 8, in <module>
sys.exit(main())
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/__main__.py", line 40, in main
args.func(args)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/cli/cli_parser.py", line 48, in command
return func(*args, **kwargs)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/cli/commands/standalone_command.py", line 48, in entrypoint
StandaloneCommand().run()
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/cli/commands/standalone_command.py", line 64, in run
self.initialize_database()
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/cli/commands/standalone_command.py", line 170, in initialize_database
db.initdb()
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/utils/session.py", line 70, in wrapper
return func(*args, session=session, **kwargs)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/utils/db.py", line 591, in initdb
upgradedb(session=session)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/utils/session.py", line 67, in wrapper
return func(*args, **kwargs)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/utils/db.py", line 824, in upgradedb
command.upgrade(config, 'heads')
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/command.py", line 298, in upgrade
script.run_env()
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/script/base.py", line 489, in run_env
util.load_python_file(self.dir, "env.py")
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/util/pyfiles.py", line 98, in load_python_file
module = load_module_py(module_id, path)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/util/compat.py", line 184, in load_module_py
spec.loader.exec_module(module)
File "<frozen importlib._bootstrap_external>", line 843, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/migrations/env.py", line 107, in <module>
run_migrations_online()
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/migrations/env.py", line 101, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/runtime/environment.py", line 846, in run_migrations
self.get_context().run_migrations(**kw)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/runtime/migration.py", line 518, in run_migrations
step.migration_fn(**kw)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py", line 60, in upgrade
op.add_column('task_instance', sa.Column('max_tries', sa.Integer, server_default="-1"))
File "<string>", line 8, in add_column
File "<string>", line 3, in add_column
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/operations/ops.py", line 1927, in add_column
return operations.invoke(op)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/operations/base.py", line 374, in invoke
return fn(self, operation)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/operations/toimpl.py", line 132, in add_column
operations.impl.add_column(table_name, column, schema=schema, **kw)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/ddl/impl.py", line 237, in add_column
self._exec(base.AddColumn(table_name, column, schema=schema))
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/ddl/impl.py", line 140, in _exec
return conn.execute(construct, *multiparams, **params)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
return meth(self, multiparams, params)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/sqlalchemy/sql/ddl.py", line 72, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1068, in _execute_ddl
ret = self._execute_context(
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1316, in _execute_context
self._handle_dbapi_exception(
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1510, in _handle_dbapi_exception
util.raise_(
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 182, in raise_
raise exception
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) duplicate column name: max_tries
[SQL: ALTER TABLE task_instance ADD COLUMN max_tries INTEGER DEFAULT '-1']
(Background on this error at: http://sqlalche.me/e/13/e3q8)
When trying to reset database using airflow db reset, it returns this:
airflow db reset
DB: sqlite:////root/airflow/airflow.db
This will drop existing tables if they exist. Proceed? (y/n)y
[2021-10-18 15:17:43,572] {db.py:831} INFO - Dropping tables that exist
[2021-10-18 15:17:43,704] {migration.py:154} INFO - Context impl SQLiteImpl.
[2021-10-18 15:17:43,705] {migration.py:157} INFO - Will assume non-transactional DDL.
[2021-10-18 15:17:43,800] {db.py:823} INFO - Creating tables
INFO [alembic.runtime.migration] Context impl SQLiteImpl.
INFO [alembic.runtime.migration] Will assume non-transactional DDL.
INFO [alembic.runtime.migration] Running upgrade -> e3a246e0dc1, current schema
INFO [alembic.runtime.migration] Running upgrade e3a246e0dc1 -> 1507a7289a2f, create is_encrypted
/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/ddl/sqlite.py:40 UserWarning: Skipping unsupported ALTER for creation of implicit constraintPlease refer to the batch mode feature which allows for SQLite migrations using a copy-and-move strategy.
INFO [alembic.runtime.migration] Running upgrade 1507a7289a2f -> 13eb55f81627, maintain history for compatibility with earlier migrations
INFO [alembic.runtime.migration] Running upgrade 13eb55f81627 -> 338e90f54d61, More logging into task_instance
INFO [alembic.runtime.migration] Running upgrade 338e90f54d61 -> 52d714495f0, job_id indices
INFO [alembic.runtime.migration] Running upgrade 52d714495f0 -> 502898887f84, Adding extra to Log
INFO [alembic.runtime.migration] Running upgrade 502898887f84 -> 1b38cef5b76e, add dagrun
INFO [alembic.runtime.migration] Running upgrade 1b38cef5b76e -> 2e541a1dcfed, task_duration
INFO [alembic.runtime.migration] Running upgrade 2e541a1dcfed -> 40e67319e3a9, dagrun_config
INFO [alembic.runtime.migration] Running upgrade 40e67319e3a9 -> 561833c1c74b, add password column to user
INFO [alembic.runtime.migration] Running upgrade 561833c1c74b -> 4446e08588, dagrun start end
INFO [alembic.runtime.migration] Running upgrade 4446e08588 -> bbc73705a13e, Add notification_sent column to sla_miss
INFO [alembic.runtime.migration] Running upgrade bbc73705a13e -> bba5a7cfc896, Add a column to track the encryption state of the 'Extra' field in connection
INFO [alembic.runtime.migration] Running upgrade bba5a7cfc896 -> 1968acfc09e3, add is_encrypted column to variable table
INFO [alembic.runtime.migration] Running upgrade 1968acfc09e3 -> 2e82aab8ef20, rename user table
INFO [alembic.runtime.migration] Running upgrade 2e82aab8ef20 -> 211e584da130, add TI state index
INFO [alembic.runtime.migration] Running upgrade 211e584da130 -> 64de9cddf6c9, add task fails journal table
INFO [alembic.runtime.migration] Running upgrade 64de9cddf6c9 -> f2ca10b85618, add dag_stats table
INFO [alembic.runtime.migration] Running upgrade f2ca10b85618 -> 4addfa1236f1, Add fractional seconds to mysql tables
INFO [alembic.runtime.migration] Running upgrade 4addfa1236f1 -> 8504051e801b, xcom dag task indices
INFO [alembic.runtime.migration] Running upgrade 8504051e801b -> 5e7d17757c7a, add pid field to TaskInstance
INFO [alembic.runtime.migration] Running upgrade 5e7d17757c7a -> 127d2bf2dfa7, Add dag_id/state index on dag_run table
INFO [alembic.runtime.migration] Running upgrade 127d2bf2dfa7 -> cc1e65623dc7, add max tries column to task instance
Traceback (most recent call last):
File "/root/miniconda3/envs/localspark/bin/airflow", line 8, in <module>
sys.exit(main())
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/__main__.py", line 40, in main
args.func(args)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/cli/cli_parser.py", line 48, in command
return func(*args, **kwargs)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/cli/commands/db_command.py", line 39, in resetdb
db.resetdb()
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/utils/session.py", line 70, in wrapper
return func(*args, session=session, **kwargs)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/utils/db.py", line 839, in resetdb
initdb(session=session)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/utils/session.py", line 67, in wrapper
return func(*args, **kwargs)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/utils/db.py", line 591, in initdb
upgradedb(session=session)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/utils/session.py", line 67, in wrapper
return func(*args, **kwargs)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/utils/db.py", line 824, in upgradedb
command.upgrade(config, 'heads')
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/command.py", line 298, in upgrade
script.run_env()
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/script/base.py", line 489, in run_env
util.load_python_file(self.dir, "env.py")
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/util/pyfiles.py", line 98, in load_python_file
module = load_module_py(module_id, path)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/util/compat.py", line 184, in load_module_py
spec.loader.exec_module(module)
File "<frozen importlib._bootstrap_external>", line 843, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/migrations/env.py", line 107, in <module>
run_migrations_online()
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/migrations/env.py", line 101, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/runtime/environment.py", line 846, in run_migrations
self.get_context().run_migrations(**kw)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/alembic/runtime/migration.py", line 540, in run_migrations
raise util.CommandError(
alembic.util.exc.CommandError: Migration "upgrade 127d2bf2dfa7 -> cc1e65623dc7, add max tries column to task instance" has left an uncommitted transaction opened; transactional_ddl is False so Alembic is not committing transactions
Command airflow db check-migrations returns this:
[2021-10-18 15:19:12,201] {migration.py:154} INFO - Context impl SQLiteImpl.
[2021-10-18 15:19:12,201] {migration.py:157} INFO - Will assume non-transactional DDL.
Traceback (most recent call last):
File "/root/miniconda3/envs/localspark/bin/airflow", line 8, in <module>
sys.exit(main())
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/__main__.py", line 40, in main
args.func(args)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/cli/cli_parser.py", line 48, in command
return func(*args, **kwargs)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/cli/commands/db_command.py", line 54, in check_migrations
db.check_migrations(timeout=args.migration_wait_timeout)
File "/root/miniconda3/envs/localspark/lib/python3.8/site-packages/airflow/utils/db.py", line 643, in check_migrations
raise TimeoutError(
TimeoutError: There are still unapplied migrations after 0 seconds. Migration Head(s) in DB: {'127d2bf2dfa7'} | Migration Head(s) in Source Code: {'7b2661a43ba3'}
Fixes tried
Deleting the Airflow DB (even the whole Airflow home directory) doesn't help. Uninstalling and reinstalling Airflow either
The question
Is there a way to either fix these broken migrations or remove the Airflow in a way that would also get rid of this problem?

Related

"ValueError: Unable to configure handler 'processor'" error when running "airflow standalone"

I am new to airflow. I created a virtual environment and followed the steps in https://airflow.apache.org/docs/apache-airflow/stable/start.html. In the end I gave "airflow standalone" and got "ValueError: Unable to configure handler 'processor'"
(venv) hgovea155#INSML-CPXX7WW AFDocProj % airflow standalone
Unable to load the config, contains a configuration error.
Traceback (most recent call last):
File "/usr/local/opt/python#3.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/logging/config.py", line 563, in configure
handler = self.configure_handler(handlers[name])
File "/usr/local/opt/python#3.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/logging/config.py", line 736, in configure_handler
result = factory(**kwargs)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/utils/log/file_processor_handler.py", line 49, in __init__
Path(self._get_log_directory()).mkdir(parents=True, exist_ok=True)
File "/usr/local/opt/python#3.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/pathlib.py", line 1273, in mkdir
self._accessor.mkdir(self, mode)
PermissionError: [Errno 13] Permission denied: '/Users/hgovea155/airflow/logs/scheduler/2023-01-02'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/bin/airflow", line 5, in <module>
from airflow.__main__ import main
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/__init__.py", line 46, in <module>
settings.initialize()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/settings.py", line 569, in initialize
LOGGING_CLASS_PATH = configure_logging()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/logging_config.py", line 74, in configure_logging
raise e
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/logging_config.py", line 69, in configure_logging
dictConfig(logging_config)
File "/usr/local/opt/python#3.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/logging/config.py", line 800, in dictConfig
dictConfigClass(config).configure()
File "/usr/local/opt/python#3.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/logging/config.py", line 571, in configure
'%r' % name) from e
ValueError: Unable to configure handler 'processor'
I then gave export AIRFLOW_HOME=. after which I gave "airflow standalone" again and I received "airflow.exceptions.AirflowConfigException: Cannot use relative path: sqlite:///./airflow.db to connect to sqlite. Please use absolute path such as sqlite:////tmp/airflow.db."
standalone | Database ready
Traceback (most recent call last):
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/bin/airflow", line 8, in <module>
sys.exit(main())
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/__main__.py", line 39, in main
args.func(args)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/cli_parser.py", line 52, in command
return func(*args, **kwargs)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/commands/standalone_command.py", line 48, in entrypoint
StandaloneCommand().run()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/commands/standalone_command.py", line 64, in run
self.initialize_database()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/commands/standalone_command.py", line 183, in initialize_database
appbuilder = cached_app().appbuilder
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/www/app.py", line 167, in cached_app
app = create_app(config=config, testing=testing)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/www/app.py", line 90, in create_app
f'Cannot use relative path: `{conf.get("database", "SQL_ALCHEMY_CONN")}` to connect to sqlite. '
airflow.exceptions.AirflowConfigException: Cannot use relative path: `sqlite:///./airflow.db` to connect to sqlite. Please use absolute path such as `sqlite:////tmp/airflow.db`.
I then tried the fix provided by #kulasangar
I navigated to user -> airflow and changed the "logs" folder to read write and execute for all users. The "ValueError: Unable to configure handler 'processor'" error didn't occur but I received another error.
(venv) hgovea155#INSML-CPXX7WW AFDocProj % airflow standalone
standalone | Starting Airflow Standalone
standalone | Checking database is initialized
Traceback (most recent call last):
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1901, in _execute_context
cursor, statement, parameters, context
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: attempt to write a readonly database
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/bin/airflow", line 8, in
sys.exit(main())
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/main.py", line 39, in main
args.func(args)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/cli_parser.py", line 52, in command
return func(*args, **kwargs)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/commands/standalone_command.py", line 48, in entrypoint
StandaloneCommand().run()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/commands/standalone_command.py", line 64, in run
self.initialize_database()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/commands/standalone_command.py", line 175, in initialize_database
db.initdb()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/utils/session.py", line 75, in wrapper
return func(*args, session=session, **kwargs)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/utils/db.py", line 697, in initdb
_create_db_from_orm(session=session)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/utils/db.py", line 682, in _create_db_from_orm
_create_flask_session_tbl()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/utils/db.py", line 677, in _create_flask_session_tbl
db.create_all()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/flask_sqlalchemy/init.py", line 1094, in create_all
self._execute_for_all_tables(app, bind, 'create_all')
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/flask_sqlalchemy/init.py", line 1086, in _execute_for_all_tables
op(bind=self.get_engine(app, bind), **extra)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 4931, in create_all
ddl.SchemaGenerator, self, checkfirst=checkfirst, tables=tables
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 3228, in _run_ddl_visitor
conn._run_ddl_visitor(visitorcallable, element, **kwargs)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2211, in _run_ddl_visitor
visitorcallable(self.dialect, self, **kwargs).traverse_single(element)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/sql/visitors.py", line 524, in traverse_single
return meth(obj, **kw)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 855, in visit_metadata
_is_metadata_operation=True,
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/sql/visitors.py", line 524, in traverse_single
return meth(obj, **kw)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 900, in visit_table
include_foreign_key_constraints, # noqa
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1380, in execute
return meth(self, multiparams, params, _EMPTY_EXECUTION_OPTS)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 81, in _execute_on_connection
self, multiparams, params, execution_options
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1478, in _execute_ddl
compiled,
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1944, in execute_context
e, statement, parameters, cursor, context
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2125, in handle_dbapi_exception
sqlalchemy_exception, with_traceback=exc_info[2], from=e
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 211, in raise
raise exception
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1901, in _execute_context
cursor, statement, parameters, context
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) attempt to write a readonly database
[SQL:
CREATE TABLE session (
id INTEGER NOT NULL,
session_id VARCHAR(255),
data BLOB,
expiry DATETIME,
PRIMARY KEY (id),
UNIQUE (session_id)
)
]
(Background on this error at: https://sqlalche.me/e/14/e3q8)
Is there a way to fix this? I believe this problem is because of some mistake in the initial setup or configuration. Can a more proper fix be found rather than a temporary one.
It seems like the user doesn't have the privilege to write logs under your airflow distribution folder.
Could you try granting write permission to the folder, so that airflow instance can write logs
sudo chmod -R 777 /home/user/airflow_logs
Please change the logs directory accordingly.

Error while upgrading superset to 2.0.0 psycopg2.errors.UndefinedColumn

After updating superset to 2.0.0 I cannot access sqllab nor view databases (Oracle DB).
I'm running superset on openshift with custom image built from apache/superset:2.0.0
Installed dependencies:
cx_Oracle certifi flask-oidc==1.4.0 flask_openid gevent psycopg2 psycopg2-binary==2.9.1 redis==3.5.3 itsdangerous==2.0.1
I tried running superset db upgrade with following result:
$ superset db upgrade
Loaded your LOCAL configuration at [/app/pythonpath/superset_config.py]
logging was configured successfully
2022-11-21 14:21:41,835:INFO:superset.utils.logging_configurator:logging was configured successfully
2022-11-21 14:21:41,841:INFO:root:Configured event logger of type <class 'superset.utils.log.DBEventLogger'>
INFO [alembic.runtime.migration] Context impl PostgresqlImpl.
INFO [alembic.runtime.migration] Will assume transactional DDL.
and superset init throws error:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
psycopg2.errors.UndefinedColumn: column dbs.allow_multi_schema_metadata_fetch does not exist
LINE 1: ..., dbs.force_ctas_schema AS dbs_force_ctas_schema, dbs.allow_...
^
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/bin/superset", line 33, in \<module\>
sys.exit(load_entry_point('apache-superset', 'console_scripts', 'superset')())
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1128, in __call__
return self.main(\*args, \*\*kwargs)
File "/usr/local/lib/python3.8/site-packages/flask/cli.py", line 601, in main
return super().main(\*args, \*\*kwargs)
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1053, in main
rv = self.invoke(ctx)
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1659, in invoke
return \_process_result(sub_ctx.command.invoke(sub_ctx))
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1395, in invoke
return ctx.invoke(self.callback, \*\*ctx.params)
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 754, in invoke
return \__callback(\*args, \*\*kwargs)
File "/usr/local/lib/python3.8/site-packages/click/decorators.py", line 26, in new_func
return f(get_current_context(), \*args, \*\*kwargs)
File "/usr/local/lib/python3.8/site-packages/flask/cli.py", line 445, in decorator
return \__ctx.invoke(f, \*args, \*\*kwargs)
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 754, in invoke
return \__callback(\*args, \*\*kwargs)
File "/usr/local/lib/python3.8/site-packages/click/decorators.py", line 26, in new_func
return f(get_current_context(), \*args, \*\*kwargs)
File "/usr/local/lib/python3.8/site-packages/flask/cli.py", line 445, in decorator
return \__ctx.invoke(f, \*args, \*\*kwargs)
File "/usr/local/lib/python3.8/site-packages/click/core.py", line 754, in invoke
return _callback(\*args, \*\*kwargs)
File "/app/superset/cli/main.py", line 62, in init
security_manager.sync_role_definitions()
File "/app/superset/security/manager.py", line 731, in sync_role_definitions
self.create_missing_perms()
File "/app/superset/security/manager.py", line 679, in create_missing_perms
merge_pv("datasource_access", datasource.get_perm())
File "/app/superset/connectors/sqla/models.py", line 826, in get_perm
if self.database is None:
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/attributes.py", line 294, in __get__
return self.impl.get(instance_state(instance), dict_)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/attributes.py", line 730, in get
value = self.callable(state, passive)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/strategies.py", line 759, in \_load_for_state
return self.\_emit_lazyload(
File "\<string\>", line 1, in \<lambda\>
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/strategies.py", line 847, in \_emit_lazyload
q(session)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/ext/baked.py", line 615, in \_load_on_pk_identity
result = list(bq.for_session(self.session).params(\*\*params))
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/ext/baked.py", line 444, in __iter__
return q.\_execute_and_instances(context)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3560, in \_execute_and_instances
result = conn.execute(querycontext.statement, self.\_params)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 298, in \_execute_on_connection
return connection.\_execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1124, in \_execute_clauseelement
ret = self.\_execute_context(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1316, in \_execute_context
self._handle_dbapi_exception(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1510, in handle_dbapi_exception
util.raise_(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 182, in raise
raise exception
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.ProgrammingError: (psycopg2.errors.UndefinedColumn) column dbs.allow_multi_schema_metadata_fetch does not exist
LINE 1: ..., dbs.force_ctas_schema AS dbs_force_ctas_schema, dbs.allow_...
^
\[SQL: SELECT dbs.uuid AS dbs_uuid, dbs.created_on AS dbs_created_on, dbs.changed_on AS dbs_changed_on, dbs.id AS dbs_id, dbs.verbose_name AS dbs_verbose_name, dbs.database_name AS dbs_database_name, dbs.sqlalchemy_uri AS dbs_sqlalchemy_uri, dbs.password AS dbs_password, dbs.cache_timeout AS dbs_cache_timeout, dbs.select_as_create_table_as AS dbs_select_as_create_table_as, dbs.expose_in_sqllab AS dbs_expose_in_sqllab, dbs.configuration_method AS dbs_configuration_method, dbs.allow_run_async AS dbs_allow_run_async, dbs.allow_file_upload AS dbs_allow_file_upload, dbs.allow_ctas AS dbs_allow_ctas, dbs.allow_cvas AS dbs_allow_cvas, dbs.allow_dml AS dbs_allow_dml, dbs.force_ctas_schema AS dbs_force_ctas_schema, dbs.allow_multi_schema_metadata_fetch AS dbs_allow_multi_schema_metadata_fetch, dbs.extra AS dbs_extra, dbs.encrypted_extra AS dbs_encrypted_extra, dbs.impersonate_user AS dbs_impersonate_user, dbs.server_cert AS dbs_server_cert, dbs.is_managed_externally AS dbs_is_managed_externally, dbs.external_url AS dbs_external_url, dbs.created_by_fk AS dbs_created_by_fk, dbs.changed_by_fk AS dbs_changed_by_fk
FROM dbs
WHERE dbs.id = %(param_1)s\]
\[parameters: {'param_1': 1}\]
(Background on this error at: http://sqlalche.me/e/13/f405)
Anyone found solution for this error?
Tried:
superset db upgrade
superset init
updating dependencies
restarting superset
Expect:
successfully migrate to 2.0.0 version

How to setup apache superset local flask backed correctly?

I always get the following error while setting up the local backend for apache-superset following this article https://superset.apache.org/docs/Contributing/local-backend/
How to reproduce the bug
clone this repo https://github.com/apache/superset
cd superset
# Create a virtual environment and activate it (recommended)
python3 -m venv venv # setup a python3 virtualenv
source venv/bin/activate
# Install external dependencies
pip install -r requirements/testing.txt
# Install Superset in editable (development) mode
pip install -e .
# Initialize the database
superset db upgrade
# Create an admin user in your metadata database (use `admin` as username to be able to load the examples)
superset fab create-admin
# Create default roles and permissions
superset init
Expected results
Apache superset must start successfully but upon running superset init the following error occurs frequently
Actual results
superset init command exits with exit code 1 and throwing following output.
clearly it can be seen the error is sqlite3.OperationalError: no such column: dbs.allow_file_upload
(venv) (base) mohnish#MMMDVMOHNISHSINT superset % superset init
--------------------------------------------------------------------------------
WARNING
--------------------------------------------------------------------------------
A Default SECRET_KEY was detected, please use superset_config.py to override it.
Use a strong complex alphanumeric string and use a tool to help you generate
a sufficiently random sequence, ex: openssl rand -base64 42
--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
logging was configured successfully
2022-02-09 12:27:38,482:INFO:superset.utils.logging_configurator:logging was configured successfully
2022-02-09 12:27:38,492:INFO:root:Configured event logger of type <class 'superset.utils.log.DBEventLogger'>
/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/flask_caching/__init__.py:201: UserWarning: Flask-Caching: CACHE_TYPE is set to null, caching is effectively disabled.
warnings.warn(
Syncing role definition
2022-02-09 12:27:40,476:INFO:superset.security.manager:Syncing role definition
Syncing Admin perms
2022-02-09 12:27:40,483:INFO:superset.security.manager:Syncing Admin perms
Syncing Alpha perms
2022-02-09 12:27:40,548:INFO:superset.security.manager:Syncing Alpha perms
Syncing Gamma perms
2022-02-09 12:27:40,613:INFO:superset.security.manager:Syncing Gamma perms
Syncing granter perms
2022-02-09 12:27:40,679:INFO:superset.security.manager:Syncing granter perms
Syncing sql_lab perms
2022-02-09 12:27:40,742:INFO:superset.security.manager:Syncing sql_lab perms
Fetching a set of all perms to lookup which ones are missing
2022-02-09 12:27:40,805:INFO:superset.security.manager:Fetching a set of all perms to lookup which ones are missing
Creating missing datasource permissions.
2022-02-09 12:27:40,864:INFO:superset.security.manager:Creating missing datasource permissions.
Traceback (most recent call last):
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: no such column: dbs.allow_file_upload
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/mohnish/Desktop/superset/venv/bin/superset", line 33, in <module>
sys.exit(load_entry_point('apache-superset', 'console_scripts', 'superset')())
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/flask/cli.py", line 586, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/decorators.py", line 21, in new_func
return f(get_current_context(), *args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/flask/cli.py", line 426, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/decorators.py", line 21, in new_func
return f(get_current_context(), *args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/flask/cli.py", line 426, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/Users/mohnish/Desktop/superset/superset/cli/main.py", line 61, in init
security_manager.sync_role_definitions()
File "/Users/mohnish/Desktop/superset/superset/security/manager.py", line 726, in sync_role_definitions
self.create_missing_perms()
File "/Users/mohnish/Desktop/superset/superset/security/manager.py", line 668, in create_missing_perms
merge_pv("datasource_access", datasource.get_perm())
File "/Users/mohnish/Desktop/superset/superset/connectors/sqla/models.py", line 634, in get_perm
return f"[{self.database}].[{self.table_name}](id:{self.id})"
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/orm/attributes.py", line 294, in __get__
return self.impl.get(instance_state(instance), dict_)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/orm/attributes.py", line 730, in get
value = self.callable_(state, passive)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/orm/strategies.py", line 759, in _load_for_state
return self._emit_lazyload(
File "<string>", line 1, in <lambda>
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/orm/strategies.py", line 847, in _emit_lazyload
q(session)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/ext/baked.py", line 615, in _load_on_pk_identity
result = list(bq.for_session(self.session).params(**params))
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/ext/baked.py", line 444, in __iter__
return q._execute_and_instances(context)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3560, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
return meth(self, multiparams, params)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1124, in _execute_clauseelement
ret = self._execute_context(
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1316, in _execute_context
self._handle_dbapi_exception(
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1510, in _handle_dbapi_exception
util.raise_(
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 182, in raise_
raise exception
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such column: dbs.allow_file_upload
[SQL: SELECT dbs.uuid AS dbs_uuid, dbs.created_on AS dbs_created_on, dbs.changed_on AS dbs_changed_on, dbs.id AS dbs_id, dbs.verbose_name AS dbs_verbose_name, dbs.database_name AS dbs_database_name, dbs.sqlalchemy_uri AS dbs_sqlalchemy_uri, dbs.password AS dbs_password, dbs.cache_timeout AS dbs_cache_timeout, dbs.select_as_create_table_as AS dbs_select_as_create_table_as, dbs.expose_in_sqllab AS dbs_expose_in_sqllab, dbs.configuration_method AS dbs_configuration_method, dbs.allow_run_async AS dbs_allow_run_async, dbs.allow_file_upload AS dbs_allow_file_upload, dbs.allow_ctas AS dbs_allow_ctas, dbs.allow_cvas AS dbs_allow_cvas, dbs.allow_dml AS dbs_allow_dml, dbs.force_ctas_schema AS dbs_force_ctas_schema, dbs.allow_multi_schema_metadata_fetch AS dbs_allow_multi_schema_metadata_fetch, dbs.extra AS dbs_extra, dbs.encrypted_extra AS dbs_encrypted_extra, dbs.impersonate_user AS dbs_impersonate_user, dbs.server_cert AS dbs_server_cert, dbs.is_managed_externally AS dbs_is_managed_externally, dbs.external_url AS dbs_external_url, dbs.created_by_fk AS dbs_created_by_fk, dbs.changed_by_fk AS dbs_changed_by_fk
FROM dbs
WHERE dbs.id = ?]
[parameters: (1,)]
(Background on this error at: http://sqlalche.me/e/13/e3q8)
(venv) (base) mohnish#MMMDVMOHNISHSINT superset % FLASK_ENV=development superset run -p 8088 --with-threads --reload --debugger
Environment
browser type and version: chrome latest macos
superset version: cloning directly from master branch of apache superset gihub repo
python version: 3.9
node.js version: 16.13.2
Just delete ~.superset/superset.db and run again it works fine!

flask alembic: avoid check DB revision ID

I am deploying a Flask-based app based on this amazing tutorial. I deploy my DB through:
flask db init
flask db migrate
flask db upgrade
sometimes it checks for older version of the DB inside migrations directory and I get this error (full log below, here just what I think are the most important parts):
# ...
alembic.util.exc.CommandError: Directory migrations already exists
# ...
alembic.script.revision.ResolutionError: No such revision or branch 'f05567e712eb'
# ...
alembic.util.exc.CommandError: Can't locate revision identified by 'f05567e712eb'
# ...
I know how to fix this in several ways (as example recreate DB or drop the revision ID from the table etc.). But is there any way to prevent this "revision-ID check" alembic behavior?
Here the full error log:
Traceback (most recent call last):
File "/home/diagnosticator/venv/bin/flask", line 8, in <module>
sys.exit(main())
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask/cli.py", line 894, in main
cli.main(args=args, prog_name=name)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask/cli.py", line 557, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 697, in main
rv = self.invoke(ctx)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 895, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/decorators.py", line 17, in new_func
return f(get_current_context(), *args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask/cli.py", line 412, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask_migrate/cli.py", line 31, in init
_init(directory, multidb)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask_migrate/__init__.py", line 118, in init
command.init(config, directory, 'flask')
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/command.py", line 42, in init
raise util.CommandError("Directory %s already exists" % directory)
alembic.util.exc.CommandError: Directory migrations already exists
[2022-01-04 18:30:59,314] INFO in __init__: Diagnosticator-local startup
INFO [alembic.runtime.migration] Context impl SQLiteImpl.
INFO [alembic.runtime.migration] Will assume non-transactional DDL.
20
Traceback (most recent call last):
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 143, in _catch_revision_errors
yield
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 206, in get_revisions
return self.revision_map.get_revisions(id_)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 299, in get_revisions
return sum([self.get_revisions(id_elem) for id_elem in id_], ())
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 299, in <listcomp>
return sum([self.get_revisions(id_elem) for id_elem in id_], ())
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 302, in get_revisions
return tuple(
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 303, in <genexpr>
self._revision_for_ident(rev_id, branch_label)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 360, in _revision_for_ident
raise ResolutionError(
alembic.script.revision.ResolutionError: No such revision or branch 'f05567e712eb'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/diagnosticator/venv/bin/flask", line 8, in <module>
sys.exit(main())
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask/cli.py", line 894, in main
cli.main(args=args, prog_name=name)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask/cli.py", line 557, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 697, in main
rv = self.invoke(ctx)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 895, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/decorators.py", line 17, in new_func
return f(get_current_context(), *args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask/cli.py", line 412, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask_migrate/cli.py", line 89, in migrate
_migrate(directory, message, sql, head, splice, branch_label, version_path,
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask_migrate/__init__.py", line 195, in migrate
command.revision(config, message, autogenerate=True, sql=sql,
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/command.py", line 176, in revision
script_directory.run_env()
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 425, in run_env
util.load_python_file(self.dir, 'env.py')
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/util/pyfiles.py", line 81, in load_python_file
module = load_module_py(module_id, path)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/util/compat.py", line 83, in load_module_py
spec.loader.exec_module(module)
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "migrations/env.py", line 87, in <module>
run_migrations_online()
File "migrations/env.py", line 80, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/runtime/environment.py", line 836, in run_migrations
self.get_context().run_migrations(**kw)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/runtime/migration.py", line 321, in run_migrations
for step in self._migrations_fn(heads, self):
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/command.py", line 156, in retrieve_migrations
revision_context.run_autogenerate(rev, context)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/autogenerate/api.py", line 415, in run_autogenerate
self._run_environment(rev, migration_context, True)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/autogenerate/api.py", line 425, in _run_environment
if set(self.script_directory.get_revisions(rev)) != \
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 206, in get_revisions
return self.revision_map.get_revisions(id_)
File "/usr/local/lib/python3.8/contextlib.py", line 131, in __exit__
self.gen.throw(type, value, traceback)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 174, in _catch_revision_errors
compat.raise_from_cause(util.CommandError(resolution))
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/util/compat.py", line 194, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=exc_value)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/util/compat.py", line 187, in reraise
raise value.with_traceback(tb)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 143, in _catch_revision_errors
yield
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 206, in get_revisions
return self.revision_map.get_revisions(id_)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 299, in get_revisions
return sum([self.get_revisions(id_elem) for id_elem in id_], ())
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 299, in <listcomp>
return sum([self.get_revisions(id_elem) for id_elem in id_], ())
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 302, in get_revisions
return tuple(
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 303, in <genexpr>
self._revision_for_ident(rev_id, branch_label)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 360, in _revision_for_ident
raise ResolutionError(
alembic.util.exc.CommandError: Can't locate revision identified by 'f05567e712eb'
Luckily, it's not possible. Such behaviour could make a lot harm - when we command alembic to do some operation it creates list of migrations from upper to lower in some topological order. One of them is given at command (e.g. head in alembic upgrade head), second is took from that table. Then it can safely apply migrations one by one.
Even if we come with some hack, there is no good way to answer what should be done in such case. And whatever we do, it would mean that some migration was applied but will never be downgraded.
Saying that, if you really want act some logical way instead of throwing error, it's a workaround I invented. It replaces current version of alembic table with current head. Use it at your own risk.
# It's alembic env.py file
from sqlalchemy.sql.elements import literal_column
from alembic import context
from alembic.util import CommandError
from alembic.script import ScriptDirectory
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
# Until now everything is just defaults generated by alembic
try:
context.run_migrations()
except CommandError: # in case run_migrations failed
c = context.get_context()
c.impl._exec(c._version.delete()) # delete current version from alembic table
script = ScriptDirectory.from_config(config)
head = script.revision_map.get_current_head() # get last migration in topological order
c.impl._exec(
c._version.insert().values(
version_num=literal_column("'%s'" % head),
)
) # insert it into alembic table
context.run_migrations() # run again as usual
this is what might be happening
You create a migration and run "flask db migrate -m 'msg'". This will look through your model and create a migrations file. If you open the migrations file, among other things you will see "revision=REV1" and "down_revision=None". The very first migration will have "down_revision = None" as there is nothing to downgrade do.
You then deploy the migration by "flask db upgrade". At this point, the migration (only migration you have created in step.1) gets applied. If you were to go into the db and do "select * from alembic_version", you will see the same version that was there in the revision field..
You now make some changes to the model and create another migration. The migration file will now have "revision=REV2", "down_revision=REV1"
You do "flask db upgrade" and if all goes well alembic_version will show "REV2"
Now - if for some reason you decide that you had made a mistake in the model and delete the migrations/version/REV2-migration-created-in-step-3 and say make some DB fixes and create another migration (REV3).
If you try to do "flask db upgrade" the alembic version (REV2) does not match any of the files you have in migrations/versions and you will see "Can't locate the revision identified by REV2".
If you create a migration and apply it, the way to un-apply it is to do "flask db downgrade" and then delete the migrations file - if you have to.
If you happen to delete the migrations file after doing flask db upgrade (and its not in source control for you to rever the deletion), then you have to manually deal with the situation that will require you to undo the db changes and also "update alembic version set version_num=version" where version is the previous version to which you manually reverted to.
While at this, I'd like to point out another common pitfall..
When multiple developers are on a project and there is parallel development going on in 2 branches. Let's say main branch was on REV10 and 2 developers pull a private-branch. They both have "revision=REV10, down_revision=REV9".. At the end of their development let's say
DEVELOPER1: has created 2 migrations thus having "revision=REV12, down_revision=REV11" and "revision=REV11, down_revision=REV10" in the 2 migration files he has checked-in to his private-branch [now - we can debate why not just one revision but I am taking an example here]
DEVELOPER2: has created 1 migration thus having "revision=REV11, down_revision=REV10" in his private-branch
Say Developer1 merges to main ..
Now when developer2 merges main to his private branch (before sending out a pull) - the fun (or the trouble) begins for him.
This will require developer2 to manually edit his migrations file(s) change the down_revisions so the migrations chain is in-tact at the end of combining the migration files.

alembic won't execute raw sql that sets delimiter

I'm trying to create a trigger in my database by executing a raw SQL string in alembic's upgrade function. However I get a sqlalchemy.exc.ProgrammingError when running upgrade:
Running this in my migration script:
def upgrade():
connection = op.get_bind()
connection.execute('DELIMITER #')
Produces this traceback:
app/migrate.py:10: ExtDeprecationWarning: Importing flask.ext.script is deprecated, use flask_script instead.
from flask.ext.script import Manager
app/migrate.py:11: ExtDeprecationWarning: Importing flask.ext.migrate is deprecated, use flask_migrate instead.
from flask.ext.migrate import Migrate, MigrateCommand
/home/my_project/app/database/models.py:2: ExtDeprecationWarning: Importing flask.ext.whooshalchemy is deprecated, use flask_whooshalchemy instead.
import flask.ext.whooshalchemy
/home/my_project/venv/local/lib/python2.7/site-packages/flask_whooshalchemy.py:18: ExtDeprecationWarning: Importing flask.ext.sqlalchemy is deprecated, use flask_sqlalchemy instead.
import flask.ext.sqlalchemy as flask_sqlalchemy
/home/my_project/venv/local/lib/python2.7/site-packages/sqlalchemy/dialects/mysql/base.py:1518: Warning: '##tx_isolation' is deprecated and will be removed in a future release. Please use '##transaction_isolation' instead
cursor.execute('SELECT ##tx_isolation')
INFO [alembic.runtime.migration] Context impl MySQLImpl.
INFO [alembic.runtime.migration] Will assume non-transactional DDL.
INFO [alembic.runtime.migration] Running upgrade 60f73629ea28 -> 48dd7624f294, add trigger for adding to history table
Traceback (most recent call last):
File "app/migrate.py", line 25, in <module>
manager.run()
File "/home/my_project/venv/local/lib/python2.7/site-packages/flask_script/__init__.py", line 412, in run
result = self.handle(sys.argv[0], sys.argv[1:])
File "/home/my_project/venv/local/lib/python2.7/site-packages/flask_script/__init__.py", line 383, in handle
res = handle(*args, **config)
File "/home/my_project/venv/local/lib/python2.7/site-packages/flask_script/commands.py", line 216, in __call__
return self.run(*args, **kwargs)
File "/home/my_project/venv/local/lib/python2.7/site-packages/flask_migrate/__init__.py", line 244, in upgrade
command.upgrade(config, revision, sql=sql, tag=tag)
File "/home/my_project/venv/local/lib/python2.7/site-packages/alembic/command.py", line 254, in upgrade
script.run_env()
File "/home/my_project/venv/local/lib/python2.7/site-packages/alembic/script/base.py", line 416, in run_env
util.load_python_file(self.dir, 'env.py')
File "/home/my_project/venv/local/lib/python2.7/site-packages/alembic/util/pyfiles.py", line 93, in load_python_file
module = load_module_py(module_id, path)
File "/home/my_project/venv/local/lib/python2.7/site-packages/alembic/util/compat.py", line 75, in load_module_py
mod = imp.load_source(module_id, path, fp)
File "migrations/env.py", line 87, in <module>
run_migrations_online()
File "migrations/env.py", line 80, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/home/my_project/venv/local/lib/python2.7/site-packages/alembic/runtime/environment.py", line 817, in run_migrations
self.get_context().run_migrations(**kw)
File "/home/my_project/venv/local/lib/python2.7/site-packages/alembic/runtime/migration.py", line 323, in run_migrations
step.migration_fn(**kw)
File "/home/my_project/migrations/versions/48dd7624f294_add_trigger_for_adding_to_history_table.py", line 22, in upgrade
connection.execute('DELIMITER #')
File "/home/my_project/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 939, in execute
return self._execute_text(object, multiparams, params)
File "/home/my_project/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1097, in _execute_text
statement, parameters
File "/home/my_project/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
context)
File "/home/my_project/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1393, in _handle_dbapi_exception
exc_info
File "/home/my_project/venv/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 202, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/my_project/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
context)
File "/home/my_project/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 469, in do_execute
cursor.execute(statement, parameters)
File "/home/my_project/venv/local/lib/python2.7/site-packages/MySQLdb/cursors.py", line 205, in execute
self.errorhandler(self, exc, value)
File "/home/my_project/venv/local/lib/python2.7/site-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
sqlalchemy.exc.ProgrammingError: (_mysql_exceptions.ProgrammingError) (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'DELIMITER #' at line 1") [SQL: 'DELIMITER #']
Can DELIMETER not be used in connection.execute for some reason or am I doing something wrong?
Relevant versions:
pip:
- alembic==0.9.1
- Flask-SQLAlchemy==2.1
- MySQL-python==1.2.5
- SQLAlchemy==1.1.4
- SQLAlchemy-Utils==0.32.12
MySQL: 5.7.21
OS: ubuntu 16

Categories

Resources