I am deploying a Flask-based app based on this amazing tutorial. I deploy my DB through:
flask db init
flask db migrate
flask db upgrade
sometimes it checks for older version of the DB inside migrations directory and I get this error (full log below, here just what I think are the most important parts):
# ...
alembic.util.exc.CommandError: Directory migrations already exists
# ...
alembic.script.revision.ResolutionError: No such revision or branch 'f05567e712eb'
# ...
alembic.util.exc.CommandError: Can't locate revision identified by 'f05567e712eb'
# ...
I know how to fix this in several ways (as example recreate DB or drop the revision ID from the table etc.). But is there any way to prevent this "revision-ID check" alembic behavior?
Here the full error log:
Traceback (most recent call last):
File "/home/diagnosticator/venv/bin/flask", line 8, in <module>
sys.exit(main())
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask/cli.py", line 894, in main
cli.main(args=args, prog_name=name)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask/cli.py", line 557, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 697, in main
rv = self.invoke(ctx)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 895, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/decorators.py", line 17, in new_func
return f(get_current_context(), *args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask/cli.py", line 412, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask_migrate/cli.py", line 31, in init
_init(directory, multidb)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask_migrate/__init__.py", line 118, in init
command.init(config, directory, 'flask')
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/command.py", line 42, in init
raise util.CommandError("Directory %s already exists" % directory)
alembic.util.exc.CommandError: Directory migrations already exists
[2022-01-04 18:30:59,314] INFO in __init__: Diagnosticator-local startup
INFO [alembic.runtime.migration] Context impl SQLiteImpl.
INFO [alembic.runtime.migration] Will assume non-transactional DDL.
20
Traceback (most recent call last):
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 143, in _catch_revision_errors
yield
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 206, in get_revisions
return self.revision_map.get_revisions(id_)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 299, in get_revisions
return sum([self.get_revisions(id_elem) for id_elem in id_], ())
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 299, in <listcomp>
return sum([self.get_revisions(id_elem) for id_elem in id_], ())
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 302, in get_revisions
return tuple(
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 303, in <genexpr>
self._revision_for_ident(rev_id, branch_label)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 360, in _revision_for_ident
raise ResolutionError(
alembic.script.revision.ResolutionError: No such revision or branch 'f05567e712eb'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/diagnosticator/venv/bin/flask", line 8, in <module>
sys.exit(main())
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask/cli.py", line 894, in main
cli.main(args=args, prog_name=name)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask/cli.py", line 557, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 697, in main
rv = self.invoke(ctx)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 895, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/decorators.py", line 17, in new_func
return f(get_current_context(), *args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask/cli.py", line 412, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask_migrate/cli.py", line 89, in migrate
_migrate(directory, message, sql, head, splice, branch_label, version_path,
File "/home/diagnosticator/venv/lib/python3.8/site-packages/flask_migrate/__init__.py", line 195, in migrate
command.revision(config, message, autogenerate=True, sql=sql,
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/command.py", line 176, in revision
script_directory.run_env()
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 425, in run_env
util.load_python_file(self.dir, 'env.py')
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/util/pyfiles.py", line 81, in load_python_file
module = load_module_py(module_id, path)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/util/compat.py", line 83, in load_module_py
spec.loader.exec_module(module)
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "migrations/env.py", line 87, in <module>
run_migrations_online()
File "migrations/env.py", line 80, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/runtime/environment.py", line 836, in run_migrations
self.get_context().run_migrations(**kw)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/runtime/migration.py", line 321, in run_migrations
for step in self._migrations_fn(heads, self):
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/command.py", line 156, in retrieve_migrations
revision_context.run_autogenerate(rev, context)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/autogenerate/api.py", line 415, in run_autogenerate
self._run_environment(rev, migration_context, True)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/autogenerate/api.py", line 425, in _run_environment
if set(self.script_directory.get_revisions(rev)) != \
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 206, in get_revisions
return self.revision_map.get_revisions(id_)
File "/usr/local/lib/python3.8/contextlib.py", line 131, in __exit__
self.gen.throw(type, value, traceback)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 174, in _catch_revision_errors
compat.raise_from_cause(util.CommandError(resolution))
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/util/compat.py", line 194, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=exc_value)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/util/compat.py", line 187, in reraise
raise value.with_traceback(tb)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 143, in _catch_revision_errors
yield
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/base.py", line 206, in get_revisions
return self.revision_map.get_revisions(id_)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 299, in get_revisions
return sum([self.get_revisions(id_elem) for id_elem in id_], ())
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 299, in <listcomp>
return sum([self.get_revisions(id_elem) for id_elem in id_], ())
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 302, in get_revisions
return tuple(
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 303, in <genexpr>
self._revision_for_ident(rev_id, branch_label)
File "/home/diagnosticator/venv/lib/python3.8/site-packages/alembic/script/revision.py", line 360, in _revision_for_ident
raise ResolutionError(
alembic.util.exc.CommandError: Can't locate revision identified by 'f05567e712eb'
Luckily, it's not possible. Such behaviour could make a lot harm - when we command alembic to do some operation it creates list of migrations from upper to lower in some topological order. One of them is given at command (e.g. head in alembic upgrade head), second is took from that table. Then it can safely apply migrations one by one.
Even if we come with some hack, there is no good way to answer what should be done in such case. And whatever we do, it would mean that some migration was applied but will never be downgraded.
Saying that, if you really want act some logical way instead of throwing error, it's a workaround I invented. It replaces current version of alembic table with current head. Use it at your own risk.
# It's alembic env.py file
from sqlalchemy.sql.elements import literal_column
from alembic import context
from alembic.util import CommandError
from alembic.script import ScriptDirectory
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
# Until now everything is just defaults generated by alembic
try:
context.run_migrations()
except CommandError: # in case run_migrations failed
c = context.get_context()
c.impl._exec(c._version.delete()) # delete current version from alembic table
script = ScriptDirectory.from_config(config)
head = script.revision_map.get_current_head() # get last migration in topological order
c.impl._exec(
c._version.insert().values(
version_num=literal_column("'%s'" % head),
)
) # insert it into alembic table
context.run_migrations() # run again as usual
this is what might be happening
You create a migration and run "flask db migrate -m 'msg'". This will look through your model and create a migrations file. If you open the migrations file, among other things you will see "revision=REV1" and "down_revision=None". The very first migration will have "down_revision = None" as there is nothing to downgrade do.
You then deploy the migration by "flask db upgrade". At this point, the migration (only migration you have created in step.1) gets applied. If you were to go into the db and do "select * from alembic_version", you will see the same version that was there in the revision field..
You now make some changes to the model and create another migration. The migration file will now have "revision=REV2", "down_revision=REV1"
You do "flask db upgrade" and if all goes well alembic_version will show "REV2"
Now - if for some reason you decide that you had made a mistake in the model and delete the migrations/version/REV2-migration-created-in-step-3 and say make some DB fixes and create another migration (REV3).
If you try to do "flask db upgrade" the alembic version (REV2) does not match any of the files you have in migrations/versions and you will see "Can't locate the revision identified by REV2".
If you create a migration and apply it, the way to un-apply it is to do "flask db downgrade" and then delete the migrations file - if you have to.
If you happen to delete the migrations file after doing flask db upgrade (and its not in source control for you to rever the deletion), then you have to manually deal with the situation that will require you to undo the db changes and also "update alembic version set version_num=version" where version is the previous version to which you manually reverted to.
While at this, I'd like to point out another common pitfall..
When multiple developers are on a project and there is parallel development going on in 2 branches. Let's say main branch was on REV10 and 2 developers pull a private-branch. They both have "revision=REV10, down_revision=REV9".. At the end of their development let's say
DEVELOPER1: has created 2 migrations thus having "revision=REV12, down_revision=REV11" and "revision=REV11, down_revision=REV10" in the 2 migration files he has checked-in to his private-branch [now - we can debate why not just one revision but I am taking an example here]
DEVELOPER2: has created 1 migration thus having "revision=REV11, down_revision=REV10" in his private-branch
Say Developer1 merges to main ..
Now when developer2 merges main to his private branch (before sending out a pull) - the fun (or the trouble) begins for him.
This will require developer2 to manually edit his migrations file(s) change the down_revisions so the migrations chain is in-tact at the end of combining the migration files.
Related
I am new to airflow. I created a virtual environment and followed the steps in https://airflow.apache.org/docs/apache-airflow/stable/start.html. In the end I gave "airflow standalone" and got "ValueError: Unable to configure handler 'processor'"
(venv) hgovea155#INSML-CPXX7WW AFDocProj % airflow standalone
Unable to load the config, contains a configuration error.
Traceback (most recent call last):
File "/usr/local/opt/python#3.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/logging/config.py", line 563, in configure
handler = self.configure_handler(handlers[name])
File "/usr/local/opt/python#3.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/logging/config.py", line 736, in configure_handler
result = factory(**kwargs)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/utils/log/file_processor_handler.py", line 49, in __init__
Path(self._get_log_directory()).mkdir(parents=True, exist_ok=True)
File "/usr/local/opt/python#3.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/pathlib.py", line 1273, in mkdir
self._accessor.mkdir(self, mode)
PermissionError: [Errno 13] Permission denied: '/Users/hgovea155/airflow/logs/scheduler/2023-01-02'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/bin/airflow", line 5, in <module>
from airflow.__main__ import main
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/__init__.py", line 46, in <module>
settings.initialize()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/settings.py", line 569, in initialize
LOGGING_CLASS_PATH = configure_logging()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/logging_config.py", line 74, in configure_logging
raise e
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/logging_config.py", line 69, in configure_logging
dictConfig(logging_config)
File "/usr/local/opt/python#3.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/logging/config.py", line 800, in dictConfig
dictConfigClass(config).configure()
File "/usr/local/opt/python#3.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/logging/config.py", line 571, in configure
'%r' % name) from e
ValueError: Unable to configure handler 'processor'
I then gave export AIRFLOW_HOME=. after which I gave "airflow standalone" again and I received "airflow.exceptions.AirflowConfigException: Cannot use relative path: sqlite:///./airflow.db to connect to sqlite. Please use absolute path such as sqlite:////tmp/airflow.db."
standalone | Database ready
Traceback (most recent call last):
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/bin/airflow", line 8, in <module>
sys.exit(main())
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/__main__.py", line 39, in main
args.func(args)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/cli_parser.py", line 52, in command
return func(*args, **kwargs)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/commands/standalone_command.py", line 48, in entrypoint
StandaloneCommand().run()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/commands/standalone_command.py", line 64, in run
self.initialize_database()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/commands/standalone_command.py", line 183, in initialize_database
appbuilder = cached_app().appbuilder
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/www/app.py", line 167, in cached_app
app = create_app(config=config, testing=testing)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/www/app.py", line 90, in create_app
f'Cannot use relative path: `{conf.get("database", "SQL_ALCHEMY_CONN")}` to connect to sqlite. '
airflow.exceptions.AirflowConfigException: Cannot use relative path: `sqlite:///./airflow.db` to connect to sqlite. Please use absolute path such as `sqlite:////tmp/airflow.db`.
I then tried the fix provided by #kulasangar
I navigated to user -> airflow and changed the "logs" folder to read write and execute for all users. The "ValueError: Unable to configure handler 'processor'" error didn't occur but I received another error.
(venv) hgovea155#INSML-CPXX7WW AFDocProj % airflow standalone
standalone | Starting Airflow Standalone
standalone | Checking database is initialized
Traceback (most recent call last):
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1901, in _execute_context
cursor, statement, parameters, context
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: attempt to write a readonly database
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/bin/airflow", line 8, in
sys.exit(main())
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/main.py", line 39, in main
args.func(args)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/cli_parser.py", line 52, in command
return func(*args, **kwargs)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/commands/standalone_command.py", line 48, in entrypoint
StandaloneCommand().run()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/commands/standalone_command.py", line 64, in run
self.initialize_database()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/cli/commands/standalone_command.py", line 175, in initialize_database
db.initdb()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/utils/session.py", line 75, in wrapper
return func(*args, session=session, **kwargs)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/utils/db.py", line 697, in initdb
_create_db_from_orm(session=session)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/utils/db.py", line 682, in _create_db_from_orm
_create_flask_session_tbl()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/airflow/utils/db.py", line 677, in _create_flask_session_tbl
db.create_all()
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/flask_sqlalchemy/init.py", line 1094, in create_all
self._execute_for_all_tables(app, bind, 'create_all')
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/flask_sqlalchemy/init.py", line 1086, in _execute_for_all_tables
op(bind=self.get_engine(app, bind), **extra)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 4931, in create_all
ddl.SchemaGenerator, self, checkfirst=checkfirst, tables=tables
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 3228, in _run_ddl_visitor
conn._run_ddl_visitor(visitorcallable, element, **kwargs)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2211, in _run_ddl_visitor
visitorcallable(self.dialect, self, **kwargs).traverse_single(element)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/sql/visitors.py", line 524, in traverse_single
return meth(obj, **kw)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 855, in visit_metadata
_is_metadata_operation=True,
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/sql/visitors.py", line 524, in traverse_single
return meth(obj, **kw)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 900, in visit_table
include_foreign_key_constraints, # noqa
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1380, in execute
return meth(self, multiparams, params, _EMPTY_EXECUTION_OPTS)
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 81, in _execute_on_connection
self, multiparams, params, execution_options
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1478, in _execute_ddl
compiled,
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1944, in execute_context
e, statement, parameters, cursor, context
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 2125, in handle_dbapi_exception
sqlalchemy_exception, with_traceback=exc_info[2], from=e
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 211, in raise
raise exception
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1901, in _execute_context
cursor, statement, parameters, context
File "/Users/hgovea155/PycharmProjects/AFDocProj/venv/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) attempt to write a readonly database
[SQL:
CREATE TABLE session (
id INTEGER NOT NULL,
session_id VARCHAR(255),
data BLOB,
expiry DATETIME,
PRIMARY KEY (id),
UNIQUE (session_id)
)
]
(Background on this error at: https://sqlalche.me/e/14/e3q8)
Is there a way to fix this? I believe this problem is because of some mistake in the initial setup or configuration. Can a more proper fix be found rather than a temporary one.
It seems like the user doesn't have the privilege to write logs under your airflow distribution folder.
Could you try granting write permission to the folder, so that airflow instance can write logs
sudo chmod -R 777 /home/user/airflow_logs
Please change the logs directory accordingly.
I always get the following error while setting up the local backend for apache-superset following this article https://superset.apache.org/docs/Contributing/local-backend/
How to reproduce the bug
clone this repo https://github.com/apache/superset
cd superset
# Create a virtual environment and activate it (recommended)
python3 -m venv venv # setup a python3 virtualenv
source venv/bin/activate
# Install external dependencies
pip install -r requirements/testing.txt
# Install Superset in editable (development) mode
pip install -e .
# Initialize the database
superset db upgrade
# Create an admin user in your metadata database (use `admin` as username to be able to load the examples)
superset fab create-admin
# Create default roles and permissions
superset init
Expected results
Apache superset must start successfully but upon running superset init the following error occurs frequently
Actual results
superset init command exits with exit code 1 and throwing following output.
clearly it can be seen the error is sqlite3.OperationalError: no such column: dbs.allow_file_upload
(venv) (base) mohnish#MMMDVMOHNISHSINT superset % superset init
--------------------------------------------------------------------------------
WARNING
--------------------------------------------------------------------------------
A Default SECRET_KEY was detected, please use superset_config.py to override it.
Use a strong complex alphanumeric string and use a tool to help you generate
a sufficiently random sequence, ex: openssl rand -base64 42
--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
logging was configured successfully
2022-02-09 12:27:38,482:INFO:superset.utils.logging_configurator:logging was configured successfully
2022-02-09 12:27:38,492:INFO:root:Configured event logger of type <class 'superset.utils.log.DBEventLogger'>
/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/flask_caching/__init__.py:201: UserWarning: Flask-Caching: CACHE_TYPE is set to null, caching is effectively disabled.
warnings.warn(
Syncing role definition
2022-02-09 12:27:40,476:INFO:superset.security.manager:Syncing role definition
Syncing Admin perms
2022-02-09 12:27:40,483:INFO:superset.security.manager:Syncing Admin perms
Syncing Alpha perms
2022-02-09 12:27:40,548:INFO:superset.security.manager:Syncing Alpha perms
Syncing Gamma perms
2022-02-09 12:27:40,613:INFO:superset.security.manager:Syncing Gamma perms
Syncing granter perms
2022-02-09 12:27:40,679:INFO:superset.security.manager:Syncing granter perms
Syncing sql_lab perms
2022-02-09 12:27:40,742:INFO:superset.security.manager:Syncing sql_lab perms
Fetching a set of all perms to lookup which ones are missing
2022-02-09 12:27:40,805:INFO:superset.security.manager:Fetching a set of all perms to lookup which ones are missing
Creating missing datasource permissions.
2022-02-09 12:27:40,864:INFO:superset.security.manager:Creating missing datasource permissions.
Traceback (most recent call last):
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: no such column: dbs.allow_file_upload
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/mohnish/Desktop/superset/venv/bin/superset", line 33, in <module>
sys.exit(load_entry_point('apache-superset', 'console_scripts', 'superset')())
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/flask/cli.py", line 586, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/decorators.py", line 21, in new_func
return f(get_current_context(), *args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/flask/cli.py", line 426, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/decorators.py", line 21, in new_func
return f(get_current_context(), *args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/flask/cli.py", line 426, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/Users/mohnish/Desktop/superset/superset/cli/main.py", line 61, in init
security_manager.sync_role_definitions()
File "/Users/mohnish/Desktop/superset/superset/security/manager.py", line 726, in sync_role_definitions
self.create_missing_perms()
File "/Users/mohnish/Desktop/superset/superset/security/manager.py", line 668, in create_missing_perms
merge_pv("datasource_access", datasource.get_perm())
File "/Users/mohnish/Desktop/superset/superset/connectors/sqla/models.py", line 634, in get_perm
return f"[{self.database}].[{self.table_name}](id:{self.id})"
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/orm/attributes.py", line 294, in __get__
return self.impl.get(instance_state(instance), dict_)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/orm/attributes.py", line 730, in get
value = self.callable_(state, passive)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/orm/strategies.py", line 759, in _load_for_state
return self._emit_lazyload(
File "<string>", line 1, in <lambda>
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/orm/strategies.py", line 847, in _emit_lazyload
q(session)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/ext/baked.py", line 615, in _load_on_pk_identity
result = list(bq.for_session(self.session).params(**params))
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/ext/baked.py", line 444, in __iter__
return q._execute_and_instances(context)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 3560, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
return meth(self, multiparams, params)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1124, in _execute_clauseelement
ret = self._execute_context(
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1316, in _execute_context
self._handle_dbapi_exception(
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1510, in _handle_dbapi_exception
util.raise_(
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 182, in raise_
raise exception
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/Users/mohnish/Desktop/superset/venv/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such column: dbs.allow_file_upload
[SQL: SELECT dbs.uuid AS dbs_uuid, dbs.created_on AS dbs_created_on, dbs.changed_on AS dbs_changed_on, dbs.id AS dbs_id, dbs.verbose_name AS dbs_verbose_name, dbs.database_name AS dbs_database_name, dbs.sqlalchemy_uri AS dbs_sqlalchemy_uri, dbs.password AS dbs_password, dbs.cache_timeout AS dbs_cache_timeout, dbs.select_as_create_table_as AS dbs_select_as_create_table_as, dbs.expose_in_sqllab AS dbs_expose_in_sqllab, dbs.configuration_method AS dbs_configuration_method, dbs.allow_run_async AS dbs_allow_run_async, dbs.allow_file_upload AS dbs_allow_file_upload, dbs.allow_ctas AS dbs_allow_ctas, dbs.allow_cvas AS dbs_allow_cvas, dbs.allow_dml AS dbs_allow_dml, dbs.force_ctas_schema AS dbs_force_ctas_schema, dbs.allow_multi_schema_metadata_fetch AS dbs_allow_multi_schema_metadata_fetch, dbs.extra AS dbs_extra, dbs.encrypted_extra AS dbs_encrypted_extra, dbs.impersonate_user AS dbs_impersonate_user, dbs.server_cert AS dbs_server_cert, dbs.is_managed_externally AS dbs_is_managed_externally, dbs.external_url AS dbs_external_url, dbs.created_by_fk AS dbs_created_by_fk, dbs.changed_by_fk AS dbs_changed_by_fk
FROM dbs
WHERE dbs.id = ?]
[parameters: (1,)]
(Background on this error at: http://sqlalche.me/e/13/e3q8)
(venv) (base) mohnish#MMMDVMOHNISHSINT superset % FLASK_ENV=development superset run -p 8088 --with-threads --reload --debugger
Environment
browser type and version: chrome latest macos
superset version: cloning directly from master branch of apache superset gihub repo
python version: 3.9
node.js version: 16.13.2
Just delete ~.superset/superset.db and run again it works fine!
I am trying to reflect an existing MS Access database into a new model. To link the database to SQLAlchemy, I am using
engine = create_engine("access+pyodbc://#db-dns")
Since the database already exists, I am following the basic use scheme from this article: https://docs.sqlalchemy.org/en/14/orm/extensions/automap.html
# reflect the table
Base.prepare(engine, reflect=True)
# mapped classes are now created with names by default
# matching that of the table name
Personen = Base.classes.ref_personen #here, 'ref_personen' is a table from my database
session = Session(engine)
When I try to access the table using Personen.query.all() I get an AttributeError (error log see below).
EDIT: To raise the error, I dont even have to query anything. I assume that the error is already caused at this line: Personen = Base.classes.ref_personen
Often, the reason for this error is due to not hanving defined a primary key in the external database. However, this is not the case for me. In my MS Access DB rel_personen table, a primary key is defined.
Thanks, I appreciate any help on this topic!
(appenv4) C:\Users\Code\webapps4>flask run
* Environment: production
WARNING: This is a development server. Do not use it in a production deployment.
Use a production WSGI server instead.
* Debug mode: off
c:\users\code\webapps4\appenv4\lib\site-packages\sqlalchemy\engine\reflection.py:919: SAWarning: The Access ODBC driver does not support the ODBC "SQLPrimaryKeys" function. get_pk_constraint() is returning an empty list.
pk_cons = self.get_pk_constraint(
c:\users\code\webapps4\appenv4\lib\site-packages\sqlalchemy\engine\reflection.py:947: SAWarning: The Access ODBC driver does not support the ODBC "SQLForeignKeys" function. get_foreign_keys() is returning an empty list.
fkeys = self.get_foreign_keys(
Traceback (most recent call last):
File "c:\users\code\webapps4\appenv4\lib\site-packages\sqlalchemy\util\_collections.py", line 186, in __getattr__
return self._data[key]
KeyError: 'ref_personen'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\anaconda3\lib\runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "C:\Users\anaconda3\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "C:\Users\Code\webapps4\appenv4\Scripts\flask.exe\__main__.py", line 7, in <module>
File "c:\users\code\webapps4\appenv4\lib\site-packages\flask\cli.py", line 990, in main
cli.main(args=sys.argv[1:])
File "c:\users\code\webapps4\appenv4\lib\site-packages\flask\cli.py", line 596, in main
return super().main(*args, **kwargs)
File "c:\users\code\webapps4\appenv4\lib\site-packages\click\core.py", line 1062, in main
rv = self.invoke(ctx)
File "c:\users\code\webapps4\appenv4\lib\site-packages\click\core.py", line 1668, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "c:\users\code\webapps4\appenv4\lib\site-packages\click\core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "c:\users\code\webapps4\appenv4\lib\site-packages\click\core.py", line 763, in invoke
return __callback(*args, **kwargs)
File "c:\users\code\webapps4\appenv4\lib\site-packages\click\decorators.py", line 84, in new_func
return ctx.invoke(f, obj, *args, **kwargs)
File "c:\users\code\webapps4\appenv4\lib\site-packages\click\core.py", line 763, in invoke
return __callback(*args, **kwargs)
File "c:\users\code\webapps4\appenv4\lib\site-packages\flask\cli.py", line 845, in run_command
app = DispatchingApp(info.load_app, use_eager_loading=eager_loading)
File "c:\users\code\webapps4\appenv4\lib\site-packages\flask\cli.py", line 321, in __init__
self._load_unlocked()
File "c:\users\code\webapps4\appenv4\lib\site-packages\flask\cli.py", line 346, in _load_unlocked
self._app = rv = self.loader()
File "c:\users\code\webapps4\appenv4\lib\site-packages\flask\cli.py", line 406, in load_app
app = locate_app(self, import_name, None, raise_if_not_found=False)
File "c:\users\code\webapps4\appenv4\lib\site-packages\flask\cli.py", line 273, in locate_app
return find_best_app(script_info, module)
File "c:\users\code\webapps4\appenv4\lib\site-packages\flask\cli.py", line 68, in find_best_app
app = call_factory(script_info, app_factory)
File "c:\users\code\webapps4\appenv4\lib\site-packages\flask\cli.py", line 119, in call_factory
return app_factory(*args, **kwargs)
File "C:\Users\Code\webapps4\app\__init__.py", line 19, in create_app
from .zkl import zkl as zkl_bp
File "C:\Users\Code\webapps4\app\zkl\__init__.py", line 6, in <module>
from . import zkl_forms, zkl_views
File "C:\Users\Code\webapps4\app\zkl\zkl_views.py", line 3, in <module>
from .zkl_database import db_session, init_db
File "C:\Users\Code\webapps4\app\zkl\zkl_database.py", line 12, in <module>
Personen = Base.classes.ref_personen
File "c:\users\code\webapps4\appenv4\lib\site-packages\sqlalchemy\util\_collections.py", line 188, in __getattr__
raise AttributeError(key)
AttributeError: ref_personen
Update: sqlalchemy-access version 1.1.0 now supports reflection of Primary Key (PK) and Foreign Key (FK) constraints.
(previous answer)
As noted by one of the warning messages in your stack trace:
SAWarning: The Access ODBC driver does not support the ODBC "SQLPrimaryKeys" function. get_pk_constraint() is returning an empty list.
This is a known issue with the Access ODBC driver. It is also discussed here
https://github.com/gordthompson/sqlalchemy-access/issues/9
and here
pyodbc - read primary keys from MS Access (MDB) database
I am using Postgres DB with Flask migration. I have changed a table name and want to upgrade it to my local & then remote.
Init give:
flask db init
Python-dotenv could not parse statement starting at line 4
Python-dotenv could not parse statement starting at line 5
Python-dotenv could not parse statement starting at line 6
Error: Directory migrations already exists and is not empty
Migrate gives:
(env) (base) Daniyal#Daniyals-MacBook-Pro Website % flask db migrate
Python-dotenv could not parse statement starting at line 4
Python-dotenv could not parse statement starting at line 5
Python-dotenv could not parse statement starting at line 6
INFO [alembic.runtime.migration] Context impl PostgresqlImpl.
INFO [alembic.runtime.migration] Will assume transactional DDL.
/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/script/revision.py:152: UserWarning: Revision 6ee19758a13a referenced from 6ee19758a13a -> 2e8e5e007c0a (head), empty message is not present
util.warn(
Traceback (most recent call last):
File "/Users/Daniyal/Startup/Website/env/bin/flask", line 8, in <module>
sys.exit(main())
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/flask/cli.py", line 967, in main
cli.main(args=sys.argv[1:], prog_name="python -m flask" if as_module else None)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/flask/cli.py", line 586, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/click/core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/click/core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/click/decorators.py", line 21, in new_func
return f(get_current_context(), *args, **kwargs)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/flask/cli.py", line 426, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/flask_migrate/cli.py", line 89, in migrate
_migrate(directory, message, sql, head, splice, branch_label, version_path,
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/flask_migrate/__init__.py", line 96, in wrapped
f(*args, **kwargs)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/flask_migrate/__init__.py", line 210, in migrate
command.revision(config, message, autogenerate=True, sql=sql,
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/command.py", line 214, in revision
script_directory.run_env()
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/script/base.py", line 489, in run_env
util.load_python_file(self.dir, "env.py")
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/util/pyfiles.py", line 98, in load_python_file
module = load_module_py(module_id, path)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/util/compat.py", line 184, in load_module_py
spec.loader.exec_module(module)
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "migrations/env.py", line 96, in <module>
run_migrations_online()
File "migrations/env.py", line 90, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/runtime/environment.py", line 846, in run_migrations
self.get_context().run_migrations(**kw)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/runtime/migration.py", line 511, in run_migrations
for step in self._migrations_fn(heads, self):
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/command.py", line 190, in retrieve_migrations
revision_context.run_autogenerate(rev, context)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/autogenerate/api.py", line 442, in run_autogenerate
self._run_environment(rev, migration_context, True)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/autogenerate/api.py", line 454, in _run_environment
self.script_directory.get_revisions("heads")
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/script/base.py", line 227, in get_revisions
return self.revision_map.get_revisions(id_)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/script/revision.py", line 321, in get_revisions
resolved_id, branch_label = self._resolve_revision_number(id_)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/script/revision.py", line 501, in _resolve_revision_number
self._revision_map
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/util/langhelpers.py", line 230, in __get__
obj.__dict__[self.__name__] = result = self.fget(obj)
File "/Users/Daniyal/Startup/Website/env/lib/python3.8/site-packages/alembic/script/revision.py", line 156, in _revision_map
down_revision = map_[downrev]
KeyError: '6ee19758a13a'
I am not sure why upgrading such a trivial change to DB's schema has to be such a hassle.
I would really appreciate your help on how I can make such change.
I have a cron job that downloads data from MySQL every night at 3. I can test this connection and download and it works. Occasionally the download partially fails. (partial download) If I try and re-run the py script, it barks. Duplicate entry error for key 2.
I want to be able to run a script and erase only the entries for the previous night so I can rerun the script that populates the db. There are three other tables tied to this one. What is django going to do if is create a SQL script that deletes yesterdays records? Will it automatically delete necessary additions to the other tables, or should I do this in the script also?
Traceback (most recent call last):
File "manage.py", line 10, in <module>
execute_from_command_line(sys.argv)
File "/usr/local/lib/python2.6/site-packages/django/core/management/__init__.py", line 443, in execute_from_command_line
utility.execute()
File "/usr/local/lib/python2.6/site-packages/django/core/management/__init__.py", line 382, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/usr/local/lib/python2.6/site-packages/django/core/management/base.py", line 196, in run_from_argv
self.execute(*args, **options.__dict__)
File "/usr/local/lib/python2.6/site-packages/django/core/management/base.py", line 232, in execute
output = self.handle(*args, **options)
File "/usr/local/django/grease/greaseboard/management/commands/import_patients.py", line 27, in handle
mrn = row.MRN,
File "/usr/local/lib/python2.6/site-packages/django/db/models/manager.py", line 134, in get_or_create
return self.get_query_set().get_or_create(**kwargs)
File "/usr/local/lib/python2.6/site-packages/django/db/models/query.py", line 449, in get_or_create
obj.save(force_insert=True, using=self.db)
File "/usr/local/lib/python2.6/site-packages/django/db/models/base.py", line 463, in save
self.save_base(using=using, force_insert=force_insert, force_update=force_update)
File "/usr/local/lib/python2.6/site-packages/django/db/models/base.py", line 551, in save_base
result = manager._insert([self], fields=fields, return_id=update_pk, using=using, raw=raw)
File "/usr/local/lib/python2.6/site-packages/django/db/models/manager.py", line 203, in _insert
return insert_query(self.model, objs, fields, **kwargs)
File "/usr/local/lib/python2.6/site-packages/django/db/models/query.py", line 1576, in insert_query
return query.get_compiler(using=using).execute_sql(return_id)
File "/usr/local/lib/python2.6/site-packages/django/db/models/sql/compiler.py", line 910, in execute_sql
cursor.execute(sql, params)
File "/usr/local/lib/python2.6/site-packages/django/db/backends/util.py", line 40, in execute
return self.cursor.execute(sql, params)
File "/usr/local/lib/python2.6/site-packages/django/db/backends/mysql/base.py", line 114, in execute
return self.cursor.execute(query, args)
File "/usr/local/lib/python2.6/site-packages/MySQLdb/cursors.py", line 174, in execute
self.errorhandler(self, exc, value)
File "/usr/local/lib/python2.6/site-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
django.db.utils.IntegrityError: (1062, "Duplicate entry '000xxxxxxxx' for key 2")
No idea how large the work is, but for similar problem I used transactions alongside savepoints - https://docs.djangoproject.com/en/dev/topics/db/transactions/#savepoints
so given something like:
transaction.auto_commit(False)
try:
for raw_input in streaming_filelikeobject.readline():
product = do_work(raw_input)
MyTable(**product).save()
except SomeFileIOError:
transaction.rollback()
else:
transaction.commit()
Another idea would be to have a batch_id column and assign that on the start of each batch.
For very large data sets, you could use something like Memcache/Redis for managing inventory.
transaction.auto_commit(False)
try:
for raw_input in streaming_filelikeobject.readline():
product = do_work(raw_input)
if redis_conn.sadd("my_input_set", product['some_unique_id']):
MyTable(**product).save()
except SomeFileIOError:
transaction.rollback()
else:
transaction.commit()
.sadd() is a Redis command that returns true if an element doesn't exist already in a redis set.
Please note I am typing this stuff of the top of my head so method django transaction method signatures may not be authoritative.