I´m not able to connect in the SQL Server inside Airflow using docker-compose. I want to take data from SQL Server direct to Cloud Storage and then the data will be sent to Big Query.
How to solve this?
import json
from datetime import timedelta, datetime
from airflow import DAG
from airflow.models import Variable
from airflow.contrib.operators.bigquery_operator import BigQueryOperator
from airflow.contrib.operators.bigquery_check_operator import BigQueryCheckOperator
from airflow.contrib.operators.file_to_gcs import FileToGoogleCloudStorageOperator
from airflow.contrib.operators.gcs_to_bq import GoogleCloudStorageToBigQueryOperator
from airflow.contrib.operators.mysql_to_gcs import MySqlToGoogleCloudStorageOperator
default_args = {
'owner': 'Test Data',
'depends_on_past': True,
'start_date': datetime(2019, 5, 29),
'end_date': datetime(2019, 5, 30),
'email': ['email#clientx.com.br'],
'email_on_failure': True,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
# Set Schedule: Run pipeline once a day.
# Use cron to define exact time. Eg. 8:15am would be "15 08 * * *"
schedule_interval = "* * * * *"
# Define DAG: Set ID and assign default args and schedule interval
dag = DAG(
'bigquery_github_trends',
default_args=default_args,
schedule_interval=schedule_interval
)
extract = MySqlToGoogleCloudStorageOperator(
task_id='chama_extract',
mysql_conn_id='mysql_hml',
google_cloud_storage_conn_id='my_gcp_conn',
sql="""SELECT * FROM test""",
bucket='my_bucket',
filename='test/test{}.json',
schema_filename='schemas/test.json',
dag=dag)
load = GoogleCloudStorageToBigQueryOperator(
task_id='chama_load',
bigquery_conn_id='my_gcp_conn',
google_cloud_storage_conn_id='my_gcp_conn',
bucket='my_bucket',
destination_project_dataset_table="tst.teste123",
source_objects=['test/test0.json'],
schema_object='schemas/test.json',
source_format='NEWLINE_DELIMITED_JSON',
create_disposition='CREATE_IF_NEEDED',
write_disposition='WRITE_TRUNCATE',
dag=dag)
# Setting up Dependencies
load.set_upstream(extract)
Docker-compose.yml
version: '3'
services:
postgres:
image: postgres:9.6
environment:
- POSTGRES_USER=airflow
- POSTGRES_PASSWORD=airflow
- POSTGRES_DB=airflow
ports:
- "5432:5432"
webserver:
image: puckel/docker-airflow:1.10.1
build:
context: https://github.com/puckel/docker-airflow.git#1.10.1
dockerfile: Dockerfile
args:
AIRFLOW_DEPS: gcp_api,s3
restart: always
depends_on:
- postgres
environment:
- LOAD_EX=n
- EXECUTOR=Local
- FERNET_KEY=jsDPRErfv8Z_eVTnGfF8ywd19j4pyqE3NpdUBA_oRTo=
volumes:
- ./examples/intro-example/dags:/usr/local/airflow/dags
# Uncomment to include custom plugins
# - ./plugins:/usr/local/airflow/plugins
ports:
- "8080:8080"
command: webserver
healthcheck:
test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
interval: 30s
timeout: 30s
retries: 3
docker-compose-gcloud.yml
version: '3'
services:
postgres:
image: postgres:9.6
environment:
- POSTGRES_USER=airflow
- POSTGRES_PASSWORD=airflow
- POSTGRES_DB=airflow
ports:
- "5432:5432"
webserver:
image: puckel/docker-airflow:1.10.1
build:
context: https://github.com/puckel/docker-airflow.git#1.10.1
dockerfile: Dockerfile
args:
AIRFLOW_DEPS: gcp_api,s3
restart: always
depends_on:
- postgres
environment:
- LOAD_EX=n
- EXECUTOR=Local
- FERNET_KEY=jsDPRErfv8Z_eVTnGfF8ywd19j4pyqE3NpdUBA_oRTo=
volumes:
- ./examples/gcloud-example/dags:/usr/local/airflow/dags
# Uncomment to include custom plugins
# - ./plugins:/usr/local/airflow/plugins
ports:
- "8080:8080"
command: webserver
healthcheck:
test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
interval: 30s
timeout: 30s
retries: 3
And execute in docker the command:
docker-compose -f docker-compose-gcloud.yml up
--abort-on-container-exit
Error message in Airflow:
[2019-05-29 07:00:37,938] {{logging_mixin.py:95}} INFO - [2019-05-29 07:00:37,937] {{base_hook.py:83}} INFO - Using connection to: 10.0.0.1
[2019-05-29 07:00:58,974] {{models.py:1760}} ERROR - (2003, 'Can\'t connect to MySQL server on 10.0.0.1 (111 "Connection refused")')
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/airflow/models.py", line 1659, in _run_raw_task
result = task_copy.execute(context=context)
File "/usr/local/lib/python3.6/site-packages/airflow/contrib/operators/mysql_to_gcs.py", line 105, in execute
cursor = self._query_mysql()
File "/usr/local/lib/python3.6/site-packages/airflow/contrib/operators/mysql_to_gcs.py", line 127, in _query_mysql
conn = mysql.get_conn()
File "/usr/local/lib/python3.6/site-packages/airflow/hooks/mysql_hook.py", line 103, in get_conn
conn = MySQLdb.connect(**conn_config)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/init.py", line 84, in Connect
return Connection(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/connections.py", line 164, in init
super(Connection, self).init(*args, **kwargs2)
MySQLdb._exceptions.OperationalError: (2003, 'Can\'t connect to MySQL server on 10.0.0.1 (111 "Connection refused")')
[2019-05-29 07:00:58,988] {{models.py:1789}} INFO - All retries failed; marking task as FAILED
[2019-05-29 07:00:58,992] {{logging_mixin.py:95}} INFO - [2019-05-29 07:00:58,991] {{configuration.py:255}} WARNING - section/key [smtp/smtp_user] not found in config
[2019-05-29 07:00:58,998] {{models.py:1796}} ERROR - [Errno 99] Cannot assign requested address
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/airflow/models.py", line 1659, in _run_raw_task
result = task_copy.execute(context=context)
File "/usr/local/lib/python3.6/site-packages/airflow/contrib/operators/mysql_to_gcs.py", line 105, in execute
cursor = self._query_mysql()
File "/usr/local/lib/python3.6/site-packages/airflow/contrib/operators/mysql_to_gcs.py", line 127, in _query_mysql
conn = mysql.get_conn()
File "/usr/local/lib/python3.6/site-packages/airflow/hooks/mysql_hook.py", line 103, in get_conn
conn = MySQLdb.connect(**conn_config)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/init.py", line 84, in Connect
return Connection(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/connections.py", line 164, in init
super(Connection, self).init(*args, **kwargs2)
MySQLdb._exceptions.OperationalError: (2003, 'Can\'t connect to MySQL server on 10.0.0.1 (111 "Connection refused")')
From the error, the key part to me seems to be the "get_conn" piece. This indicates that when airflow is trying to establish the connection to the database it fails. This means either your connection is not specified (it looks like it might be) or that some part of it is incorrect.
You should check the password, server address, and port are all correct. These should be in either you airflow.cfg, as environment variables, or set in the webserver (Admin panel)
Related
I am getting errors post configuration
*** Reading local file: /opt/airflow/logs/dag_id=DocImageExec/run_id=manual__2023-02-09T16:06:23.630116+00:00/task_id=execute_docker_command/attempt=1.log
[2023-02-09, 16:06:26 UTC] {taskinstance.py:1083} INFO - Dependencies all met for <TaskInstance: DocImageExec.execute_docker_command manual__2023-02-09T16:06:23.630116+00:00 [queued]>
[2023-02-09, 16:06:26 UTC] {taskinstance.py:1083} INFO - Dependencies all met for <TaskInstance: DocImageExec.execute_docker_command manual__2023-02-09T16:06:23.630116+00:00 [queued]>
[2023-02-09, 16:06:26 UTC] {taskinstance.py:1279} INFO -
--------------------------------------------------------------------------------
[2023-02-09, 16:06:26 UTC] {taskinstance.py:1280} INFO - Starting attempt 1 of 2
[2023-02-09, 16:06:26 UTC] {taskinstance.py:1281} INFO -
--------------------------------------------------------------------------------
[2023-02-09, 16:06:26 UTC] {taskinstance.py:1300} INFO - Executing <Task(BashOperator): execute_docker_command> on 2023-02-09 16:06:23.630116+00:00
[2023-02-09, 16:06:26 UTC] {standard_task_runner.py:55} INFO - Started process 17492 to run task
[2023-02-09, 16:06:26 UTC] {standard_task_runner.py:82} INFO - Running: ['***', 'tasks', 'run', 'DocImageExec', 'execute_docker_command', 'manual__2023-02-09T16:06:23.630116+00:00', '--job-id', '109', '--raw', '--subdir', 'DAGS_FOLDER/docimage.py', '--cfg-path', '/tmp/tmptln30ewq']
[2023-02-09, 16:06:26 UTC] {standard_task_runner.py:83} INFO - Job 109: Subtask execute_docker_command
[2023-02-09, 16:06:26 UTC] {task_command.py:388} INFO - Running <TaskInstance: DocImageExec.execute_docker_command manual__2023-02-09T16:06:23.630116+00:00 [running]> on host 14b5d43a840e
[2023-02-09, 16:06:26 UTC] {taskinstance.py:1509} INFO - Exporting the following env vars:
AIRFLOW_CTX_DAG_EMAIL=austin.jackson#xxxxx.com
AIRFLOW_CTX_DAG_OWNER=***
AIRFLOW_CTX_DAG_ID=DocImageExec
AIRFLOW_CTX_TASK_ID=execute_docker_command
AIRFLOW_CTX_EXECUTION_DATE=2023-02-09T16:06:23.630116+00:00
AIRFLOW_CTX_TRY_NUMBER=1
AIRFLOW_CTX_DAG_RUN_ID=manual__2023-02-09T16:06:23.630116+00:00
[2023-02-09, 16:06:26 UTC] {subprocess.py:63} INFO - Tmp dir root location:
/tmp
[2023-02-09, 16:06:26 UTC] {subprocess.py:75} INFO - Running command: ['/bin/bash', '-c', 'docker run -d -p 5000:5000 image-docker']
[2023-02-09, 16:06:26 UTC] {subprocess.py:86} INFO - Output:
[2023-02-09, 16:06:26 UTC] {subprocess.py:93} INFO - docker: Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?.
[2023-02-09, 16:06:26 UTC] {subprocess.py:93} INFO - See 'docker run --help'.
[2023-02-09, 16:06:26 UTC] {subprocess.py:97} INFO - Command exited with return code 125
[2023-02-09, 16:06:27 UTC] {taskinstance.py:1768} ERROR - Task failed with exception
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/operators/bash.py", line 197, in execute
f"Bash command failed. The command returned a non-zero exit code {result.exit_code}."
airflow.exceptions.AirflowException: Bash command failed. The command returned a non-zero exit code 125.
[2023-02-09, 16:06:27 UTC] {taskinstance.py:1323} INFO - Marking task as UP_FOR_RETRY. dag_id=DocImageExec, task_id=execute_docker_command, execution_date=20230209T160623, start_date=20230209T160626, end_date=20230209T160627
[2023-02-09, 16:06:27 UTC] {warnings.py:110} WARNING - /home/***/.local/lib/python3.7/site-packages/***/utils/email.py:152: RemovedInAirflow3Warning: Fetching SMTP credentials from configuration variables will be deprecated in a future release. Please set credentials using a connection instead.
send_mime_email(e_from=mail_from, e_to=recipients, mime_msg=msg, conn_id=conn_id, dryrun=dryrun)
[2023-02-09, 16:06:27 UTC] {configuration.py:663} WARNING - section/key [smtp/smtp_user] not found in config
[2023-02-09, 16:06:27 UTC] {email.py:268} INFO - Email alerting: attempt 1
[2023-02-09, 16:06:27 UTC] {configuration.py:663} WARNING - section/key [smtp/smtp_user] not found in config
[2023-02-09, 16:06:27 UTC] {email.py:268} INFO - Email alerting: attempt 1
[2023-02-09, 16:06:27 UTC] {taskinstance.py:1831} ERROR - Failed to send email to: ['austin.jackson#xxxx.com']
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/models/taskinstance.py", line 1374, in _run_raw_task
self._execute_task_with_callbacks(context, test_mode)
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/models/taskinstance.py", line 1520, in _execute_task_with_callbacks
result = self._execute_task(context, task_orig)
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/models/taskinstance.py", line 1581, in _execute_task
result = execute_callable(context=context)
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/operators/bash.py", line 197, in execute
f"Bash command failed. The command returned a non-zero exit code {result.exit_code}."
airflow.exceptions.AirflowException: Bash command failed. The command returned a non-zero exit code 125.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/models/taskinstance.py", line 2231, in email_alert
send_email(task.email, subject, html_content)
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/utils/email.py", line 91, in send_email
**kwargs,
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/utils/email.py", line 152, in send_email_smtp
send_mime_email(e_from=mail_from, e_to=recipients, mime_msg=msg, conn_id=conn_id, dryrun=dryrun)
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/utils/email.py", line 270, in send_mime_email
smtp_conn = _get_smtp_connection(smtp_host, smtp_port, smtp_timeout, smtp_ssl)
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/utils/email.py", line 317, in _get_smtp_connection
else smtplib.SMTP(host=host, port=port, timeout=timeout)
File "/usr/local/lib/python3.7/smtplib.py", line 251, in __init__
(code, msg) = self.connect(host, port)
File "/usr/local/lib/python3.7/smtplib.py", line 336, in connect
self.sock = self._get_socket(host, port, self.timeout)
File "/usr/local/lib/python3.7/smtplib.py", line 307, in _get_socket
self.source_address)
File "/usr/local/lib/python3.7/socket.py", line 728, in create_connection
raise err
File "/usr/local/lib/python3.7/socket.py", line 716, in create_connection
sock.connect(sa)
OSError: [Errno 99] Cannot assign requested address
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/models/taskinstance.py", line 1829, in handle_failure
self.email_alert(error, task)
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/models/taskinstance.py", line 2233, in email_alert
send_email(task.email, subject, html_content_err)
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/utils/email.py", line 91, in send_email
**kwargs,
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/utils/email.py", line 152, in send_email_smtp
send_mime_email(e_from=mail_from, e_to=recipients, mime_msg=msg, conn_id=conn_id, dryrun=dryrun)
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/utils/email.py", line 270, in send_mime_email
smtp_conn = _get_smtp_connection(smtp_host, smtp_port, smtp_timeout, smtp_ssl)
File "/home/airflow/.local/lib/python3.7/site-packages/airflow/utils/email.py", line 317, in _get_smtp_connection
else smtplib.SMTP(host=host, port=port, timeout=timeout)
File "/usr/local/lib/python3.7/smtplib.py", line 251, in __init__
(code, msg) = self.connect(host, port)
File "/usr/local/lib/python3.7/smtplib.py", line 336, in connect
self.sock = self._get_socket(host, port, self.timeout)
File "/usr/local/lib/python3.7/smtplib.py", line 307, in _get_socket
self.source_address)
File "/usr/local/lib/python3.7/socket.py", line 728, in create_connection
raise err
File "/usr/local/lib/python3.7/socket.py", line 716, in create_connection
sock.connect(sa)
OSError: [Errno 99] Cannot assign requested address
[2023-02-09, 16:06:27 UTC] {standard_task_runner.py:105} ERROR - Failed to execute job 109 for task execute_docker_command (Bash command failed. The command returned a non-zero exit code 125.; 17492)
[2023-02-09, 16:06:27 UTC] {local_task_job.py:208} INFO - Task exited with return code 1
[2023-02-09, 16:06:27 UTC] {taskinstance.py:2578} INFO - 0 downstream tasks scheduled from follow-on schedule check
Script is like below
"""
Code that goes along with the Airflow located at:
http://airflow.readthedocs.org/en/latest/tutorial.html
"""
import airflow
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from datetime import datetime, timedelta
from airflow.operators.docker_operator import DockerOperator
from airflow.operators.python_operator import PythonOperator
from airflow.operators.email_operator import EmailOperator
default_args = {
"owner": "airflow",
"depends_on_past": False,
"start_date": datetime(2023, 2, 8),
"email": ["austin.jackson#xxxx.com"],
#"email_on_failure": False,
"email_on_failure": True,
"email_on_retry": True,
"retries": 1,
"retry_delay": timedelta(minutes=5),
# 'queue': 'bash_queue',
# 'pool': 'backfill',
# 'priority_weight': 10,
# 'end_date': datetime(2016, 1, 1),
}
dag = DAG("DocImageExec", default_args=default_args, schedule_interval=timedelta(1))
# t1, t2 and t3 are examples of tasks created by instantiating operators
t1 = BashOperator(task_id="execute_docker_command", bash_command="docker run -d -p 5000:5000 image-v1", dag=dag)
t1
Please help with the proper configuration required for the mail with the airflow to work, I need office 365 mail integration for Apache Airflow alerts.
Below is the code:
"""
Code that goes along with the Airflow located at:
http://airflow.readthedocs.org/en/latest/tutorial.html
"""
import airflow
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from datetime import datetime, timedelta
from airflow.operators.docker_operator import DockerOperator
from airflow.operators.python_operator import PythonOperator
from airflow.operators.email_operator import EmailOperator
default_args = {
"owner": "airflow",
"depends_on_past": False,
"start_date": datetime(2023, 2, 8),
"email": ["austin.jackson#xxxx.com"],
#"email_on_failure": False,
"email_on_failure": True,
"email_on_retry": True,
"retries": 1,
"retry_delay": timedelta(minutes=5),
# 'queue': 'bash_queue',
# 'pool': 'backfill',
# 'priority_weight': 10,
# 'end_date': datetime(2016, 1, 1),
}
dag = DAG("DocImageExec", default_args=default_args, schedule_interval=timedelta(1))
# t1 example of tasks created by instantiating operators
t1 = BashOperator(task_id="execute_docker_command", bash_command="docker run -d -p 6000:6000 image-docker", dag=dag)
t1
As please review the code output, so please help as the SMTP is giving the error and not able to send the post/email to the required email id
The error you mentioned is something different not related to SMTP, first figure that out. It says docker demon is not running? How are you running airflow standalone/K8s(standalone I guess).
Normally for setting up SMTP to send email for alerts its better to use EmailOperator
Enable IMAP for SMTP
Update airflow config file with smtp details like smtp_host, SSL, user, password and port.
Use EmailOperator in the dag
It should do the job.
I'm integrating keycloak authentication in my django app
After i am logged in into keycloak server i am facing error in the login callback
HTTPError at /oidc/callback/
401 Client Error: Unauthorized for url: http://keycloak:8080/realms/SquadStack/protocol/openid-connect/userinfo
here is my docker-compose.yml
services:
db:
image: postgres:13
environment:
- POSTGRES_DB=squadrun
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=123456
volumes:
- ~/.siq/pg_data:/var/lib/postgresql/data
ports:
- "5432:5432"
networks:
- local-keycloak
redis:
image: redis
expose:
- 6379
networks:
- local-keycloak
celery:
build:
context: .
dockerfile: Dockerfile
command: celery --app=squadrun worker -Q voice_make_ivr_india_queue,voice_send_email_india_queue,voice_send_email_india_upstox_queue,voice_send_sms_india_queue,voice_workflow_india_queue,celery_voice,ivr_queue,voice_analytics_and_metrics_queue,voice_fsm_india_queue,dnd_queue,voice_bulk_sync,voice_dnd_and_compliance_actions_queue,voice_notifications_queue,voice_make_ivr_india_queue,voice_send_email_india_queue,voice_send_email_india_upstox_queue,voice_send_sms_india_queue,voice_sync_ivr_details_india_queue,voice_workflow_india_queue,voice_sync_sms_details_india_queue,voice_send_sms_india_upstox_queue,voice_dnd_and_compliance_actions_upstox_queue,voice_imports_queue --concurrency=3 --without-heartbeat --without-gossip -n celery.%%h --loglevel=INFO
container_name: celery
working_dir: /home/docker/code
volumes:
- .:/home/docker/code
depends_on:
- db
- redis
networks:
- local-keycloak
web:
build:
context: .
dockerfile: Dockerfile
command: python -Wignore manage.py runserver 0.0.0.0:8001
container_name: django_web3
volumes:
- .:/home/docker/code
ports:
- "8001:8001"
depends_on:
- db
- redis
networks:
- local-keycloak
keycloak:
image: quay.io/keycloak/keycloak:20.0.2
command: start-dev
container_name: keycloak
ports:
- "8080:8080"
environment:
- KEYCLOAK_ADMIN=admin
- KEYCLOAK_ADMIN_PASSWORD=admin
networks:
- local-keycloak
networks:
local-keycloak:
driver: bridge
here is my setting.py
AUTHENTICATION_BACKENDS = [
'apps.voice.voice_auth.auth.KeycloakOIDCAuthenticationBackend',
'django.contrib.auth.backends.ModelBackend',
]
OIDC_RP_CLIENT_ID = "voice-dashboard"
OIDC_RP_CLIENT_SECRET = "rnX0eo0R43xnZficrZTkQQseyBip4V7t"
OIDC_RP_SIGN_ALGO = "RS256"
OIDC_OP_JWKS_ENDPOINT = "http://keycloak:8080/realms/SquadStack/protocol/openid-connect/certs"
OIDC_OP_AUTHORIZATION_ENDPOINT = "http://172.20.0.3:8080/realms/SquadStack/protocol/openid-connect/auth"
OIDC_OP_TOKEN_ENDPOINT = "http://keycloak:8080/realms/SquadStack/protocol/openid-connect/token"
OIDC_OP_USER_ENDPOINT = "http://keycloak:8080/realms/SquadStack/protocol/openid-connect/userinfo"
LOGIN_REDIRECT_URL = "/voice/dashboard/index/"
LOGOUT_REDIRECT_URL = "/voice/dashboard/index/"
here is auth.py
class KeycloakOIDCAuthenticationBackend(OIDCAuthenticationBackend):
def create_user(self, claims):
""" Overrides Authentication Backend so that Django users are
created with the keycloak preferred_username.
If nothing found matching the email, then try the username.
"""
user = super(KeycloakOIDCAuthenticationBackend, self).create_user(claims)
user.first_name = claims.get('given_name', '')
user.last_name = claims.get('family_name', '')
user.email = claims.get('email')
user.username = claims.get('preferred_username')
user.save()
return user
def filter_users_by_claims(self, claims):
""" Return all users matching the specified email.
If nothing found matching the email, then try the username
"""
email = claims.get('email')
preferred_username = claims.get('preferred_username')
if not email:
return self.UserModel.objects.none()
users = self.UserModel.objects.filter(email__iexact=email)
if len(users) < 1:
if not preferred_username:
return self.UserModel.objects.none()
users = self.UserModel.objects.filter(username__iexact=preferred_username)
return users
def update_user(self, user, claims):
user.first_name = claims.get('given_name', '')
user.last_name = claims.get('family_name', '')
user.email = claims.get('email')
user.username = claims.get('preferred_username')
user.save()
return user
django logs
django_web3 | django 02/Jan/2023:14:02:14,278955 +0000 [ERROR] django.request: thread=281473135460832 extra:status_code=500&request=<WSGIRequest:GET'/oidc/callback/?state=R1OaaRk4jfr5nNNI4gr5CSdfX6sXoEqg&session_state=ea277a99-7da4-4904-abb6-63fa9bb7fb75&code=4b3c49b6-11c9-4cef-b847-fe356dfe86c3.ea277a99-7da4-4904-abb6-63fa9bb7fb75.bad660f7-7969-42d3-8475-536e4955c554'> Internal Server Error: /oidc/callback/
django_web3 | Traceback (most recent call last):
django_web3 | File "/usr/local/lib/python3.5/site-packages/django/core/handlers/base.py", line 149, in get_response
django_web3 | response = self.process_exception_by_middleware(e, request)
django_web3 | File "/usr/local/lib/python3.5/site-packages/django/core/handlers/base.py", line 147, in get_response
django_web3 | response = wrapped_callback(request, *callback_args, **callback_kwargs)
django_web3 | File "/usr/local/lib/python3.5/site-packages/django/views/generic/base.py", line 68, in view
django_web3 | return self.dispatch(request, *args, **kwargs)
django_web3 | File "/usr/local/lib/python3.5/site-packages/django/views/generic/base.py", line 88, in dispatch
django_web3 | return handler(request, *args, **kwargs)
django_web3 | File "/usr/local/lib/python3.5/site-packages/mozilla_django_oidc/views.py", line 88, in get
django_web3 | self.user = auth.authenticate(**kwargs)
django_web3 | File "/usr/local/lib/python3.5/site-packages/django/contrib/auth/__init__.py", line 74, in authenticate
django_web3 | user = backend.authenticate(**credentials)
django_web3 | File "/usr/local/lib/python3.5/site-packages/mozilla_django_oidc/auth.py", line 242, in authenticate
django_web3 | return self.get_or_create_user(access_token, id_token, verified_id)
django_web3 | File "/usr/local/lib/python3.5/site-packages/mozilla_django_oidc/auth.py", line 259, in get_or_create_user
django_web3 | user_info = self.get_userinfo(access_token, id_token, verified_id)
django_web3 | File "/usr/local/lib/python3.5/site-packages/mozilla_django_oidc/auth.py", line 202, in get_userinfo
django_web3 | user_response.raise_for_status()
django_web3 | File "/usr/local/lib/python3.5/site-packages/requests/models.py", line 943, in raise_for_status
django_web3 | raise HTTPError(http_error_msg, response=self)
django_web3 | requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: http://keycloak:8080/realms/SquadStack/protocol/openid-connect/userinfo
django_web3 | django 02/Jan/2023:14:02:14,398185 +0000 [INFO] qinspect.middleware: thread=281473135460832 extra: [SQL] 0 queries (0 duplicates), 0 ms SQL time, 201 ms total request time
django_web3 | [02/Jan/2023 19:32:14] "GET /oidc/callback/?state=R1OaaRk4jfr5nNNI4gr5CSdfX6sXoEqg&session_state=ea277a99-7da4-4904-abb6-63fa9bb7fb75&code=4b3c49b6-11c9-4cef-b847-fe356dfe86c3.ea277a99-7da4-4904-abb6-63fa9bb7fb75.bad660f7-7969-42d3-8475-536e4955c554 HTTP/1.1" 500 269936
This issue is same as this one
https://github.com/mozilla/mozilla-django-oidc/issues/481
As per it's solution i think i have to change OIDC_OP_AUTHORIZATION_ENDPOINT to "http://keycloak:8080/realms/SquadStack/protocol/openid-connect/auth"
but i can't reach this site http://keycloak:8080
I can't reach keycloak server by using docker container name instead of localhost in hostname
i can't figure out whether the problem is with docker, mozilla-django-oidc or Keycloak and i am stuck here
I am new to docker and keycloak might have done some naive mistake
Any help will be appreciated!
I used kubernetes.docker.internal as domain name instead of localhost and keycloak which resolves my problem
My changed settings:
KEYCLOACK_BASE_URI = "http://kubernetes.docker.internal:8080"
KEYCLOACK_REALM_NAME = "SquadStack"
OIDC_AUTH_URI = KEYCLOACK_BASE_URI + "/realms/" + KEYCLOACK_REALM_NAME
OIDC_OP_JWKS_ENDPOINT = OIDC_AUTH_URI + "/protocol/openid-connect/certs"
OIDC_OP_AUTHORIZATION_ENDPOINT = OIDC_AUTH_URI + "/protocol/openid-connect/auth"
OIDC_OP_TOKEN_ENDPOINT = OIDC_AUTH_URI + "/protocol/openid-connect/token"
OIDC_OP_USER_ENDPOINT = OIDC_AUTH_URI + "/protocol/openid-connect/userinfo"
OIDC_OP_LOGOUT_ENDPOINT = OIDC_AUTH_URI + "/protocol/openid-connect/logout"
I followed the number 2 instruction from this manual to use S3Hook:
Note: I hide the credential info with THIS_IS_CREDENTIAL.
And here is the simple code to test:
from airflow.operators.python import task
#task
def load_to_s3():
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
s3_hook = S3Hook(aws_conn_id="my_s3")
s3_hook.load_string(
string_data="ABC",
key="year=2021/month=1/day=1/test.txt",
bucket_name="my_bucket_in_s3",
)
default_args = {
"depends_on_past": False,
"start_date": datetime(2021, 1, 1),
"schedule_interval": "#daily",
}
with DAG("my_test_dag2", default_args=default_args) as dag:
load_to_s3()
Errors occured:
[2021-05-27 09:58:26,896] {base_aws.py:362} INFO - Airflow Connection: aws_conn_id=my_s3
[2021-05-27 09:58:26,905] {base_aws.py:173} INFO - No credentials retrieved from Connection
[2021-05-27 09:58:26,905] {base_aws.py:76} INFO - Retrieving region_name from Connection.extra_config['region_name']
[2021-05-27 09:58:26,905] {base_aws.py:78} INFO - Creating session with aws_access_key_id=None region_name=ap-northeast-1
[2021-05-27 09:58:26,913] {base_aws.py:151} INFO - role_arn is arn:aws:iam::THIS_IS_CREDENTIAL:role/airflow-v1
[2021-05-27 09:58:26,913] {base_aws.py:97} INFO - assume_role_method=None
[2021-05-27 09:58:26,930] {base_aws.py:182} INFO - Doing sts_client.assume_role to role_arn=arn:aws:iam::THIS_IS_CREDENTIAL:role/airflow-v1 (role_session_name=Airflow_my_s3)
[2021-05-27 09:58:26,932] {credentials.py:519} WARNING - Refreshing temporary credentials failed during mandatory refresh period.
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.8/site-packages/botocore/credentials.py", line 516, in _protected_refresh
metadata = self._refresh_using()
File "/home/airflow/.local/lib/python3.8/site-packages/botocore/credentials.py", line 657, in fetch_credentials
return self._get_cached_credentials()
File "/home/airflow/.local/lib/python3.8/site-packages/botocore/credentials.py", line 667, in _get_cached_credentials
response = self._get_credentials()
File "/home/airflow/.local/lib/python3.8/site-packages/botocore/credentials.py", line 872, in _get_credentials
kwargs = self._assume_role_kwargs()
File "/home/airflow/.local/lib/python3.8/site-packages/botocore/credentials.py", line 882, in _assume_role_kwargs
identity_token = self._web_identity_token_loader()
File "/home/airflow/.local/lib/python3.8/site-packages/botocore/utils.py", line 2152, in __call__
with self._open(self._web_identity_token_path) as token_file:
PermissionError: [Errno 13] Permission denied: '/var/run/secrets/eks.amazonaws.com/serviceaccount/token'
...
It looks like airflow found out role_arn=arn:aws:iam::THIS_IS_CREDENTIAL:role/airflow-v1 but
can't understand why the Airflow tries to access /var/run/secrets/eks.amazonaws.com/serviceaccount/token. I'd like to solve it without something like chmod 755 /var/run/secrets/eks.amazonaws.com/serviceaccount/token (I have no root privilege currently)
, I think you don't need to specify the AWS Airflow role, so you can do it by adding the programmatic credentials, something like that :
👇
I have the following docker-compose.yml
version: '2.1'
services:
files:
image: busybox
volumes:
- ./file/input:/file/input
- ./file/output:/file/output
[..]
classificatore:
build: classificatore/.
volumes:
- ./classificatore:/src
volumes_from:
- files
ports:
- 8080:5000
command: ["python", "/src/main.py"]
depends_on:
rabbit:
condition: service_healthy
mpeg-pre-encoder:
build: mpeg-pre-encoder/.
volumes:
- ./mpeg-pre-encoder:/src
- ./gabac_encoder:/src/gabac_encoder
volumes_from:
- files
depends_on:
rabbit:
condition: service_healthy
rabbit:
image: rabbitmq
ports:
- 5672:5672
healthcheck:
test: ["CMD", "rabbitmqctl", "cluster_status"]
interval: 5s
timeout: 2s
retries: 20
and the following dir structure:
classificatore/
Dockerfile
[..]
aggregatore/
Dockerfile
[..]
mpeg-pre-encoder/
Dockerfile
[..]
docker-compose.yml
[..]
the mpeg-pre-encoder Dockerfile
FROM python:3
RUN mkdir /src
ADD requirements.txt /src/.
WORKDIR /src
RUN pip install -r requirements.txt
ADD . /src/.
CMD ["python", "main.py"]
When I run
docker-compose up -d --build
and check for running containers
docker ps -a
I get
b6d7bac3e9b1 encoder_mpeg-pre-encoder "python main.py" 13 seconds ago Exited (1) 9 seconds ago encoder_mpeg-pre-encoder_1
b1182f765113 encoder_classificatore "python /src/main.py" 32 minutes ago Up 32 minutes 0.0.0.0:8080->5000/tcp encoder_classificatore_1
e9d8387430f2 encoder_aggregatore "python /src/main.py" 32 minutes ago Up 32 minutes 0.0.0.0:8000->8000/tcp encoder_aggregatore_1
a69183f42ab6 grafana/grafana:5.1.0 "/run.sh" About an hour ago Up About an hour 0.0.0.0:3000->3000/tcp encoder_grafana_1
ade83e898a98 prom/prometheus "/bin/prometheus --c…" About an hour ago Up 33 minutes 0.0.0.0:9090->9090/tcp encoder_prometheus_1
ff9960a7e924 busybox "sh" About an hour ago Exited (0) 12 seconds ago encoder_files_1
1738eea0645d rabbitmq "docker-entrypoint.s…" 2 weeks ago Up 2 hours (healthy) 4369/tcp, 5671/tcp, 25672/tcp, 0.0.0.0:5672->5672/tcp encoder_rabbit_1
(
The encoder_mpeg-pre-encoder container fails during building process.
Any hints on why I get this error?
EDIT 1: $docker logs b6d7bac3e9b1
2019-06-10 09:25:58,560 In attesa di nuovi job....
2019-06-10 09:25:58,560 RECEIVED: b"{'file_id': '59f311da-f02f-496a-a899-b1afd1768d64', 'parameter_set': {'reads_len': 136, 'myRnameDict': {'chrM': 0}}, 'au': {'id': 0, 'parameter_set_id': 16, 'au_type': 3, 'sequence_id': 0, 'file_name': '/file/output/minisam/59f311da-f02f-496a-a899-b1afd1768d64/CLASS_M chrM 1:7762.sam', 'AU_start_position': 0, 'AU_end_position': 7897}, 'au_count': 1}"
2019-06-10 09:25:58,561 reading the mini sam file
2019-06-10 09:25:58,582 Tempo totale impiegato per la lettura: 0 sec
2019-06-10 09:25:58,582 Tempo totale impiegato per la lettura: 0 sec
Traceback (most recent call last):
File "main.py", line 68, in <module>
rabbit.wait_for_job(decoded_symbols_from_sam_file)
File "/src/my_util.py", line 55, in wait_for_job
self.channel_input.start_consuming()
File "/usr/local/lib/python3.7/site-packages/pika/adapters/blocking_connection.py", line 1822, in start_consuming
self.connection.process_data_events(time_limit=None)
File "/usr/local/lib/python3.7/site-packages/pika/adapters/blocking_connection.py", line 758, in process_data_events
self._dispatch_channel_events()
File "/usr/local/lib/python3.7/site-packages/pika/adapters/blocking_connection.py", line 521, in _dispatch_channel_events
impl_channel._get_cookie()._dispatch_events()
File "/usr/local/lib/python3.7/site-packages/pika/adapters/blocking_connection.py", line 1445, in _dispatch_events
evt.body)
File "/src/my_util.py", line 48, in callback
worker_function(eval(body))
File "<decorator-gen-1>", line 2, in decoded_symbols_from_sam_file
File "/usr/local/lib/python3.7/site-packages/prometheus_client/context_managers.py", line 66, in wrapped
return func(*args, **kwargs)
File "main.py", line 38, in decoded_symbols_from_sam_file
mpegg_fields = convert_sam_to_mpegg(sam_fields, job_data['parameter_set']['myRnameDict'])
File "/src/mpegg_fields.py", line 40, in convert_sam_to_mpegg
mpegg_line[MPEGG_FIELD_ECIGAR_STRING] = calculate_ECIGAR_STRING(sam_line)
File "/src/mpegg_fields.py", line 107, in calculate_ECIGAR_STRING
raise ('Impossible to calculate ecigar string')
TypeError: exceptions must derive from BaseException
EDIT 2: running mpeg-pre-encoder script outside Docker
$python main.py
2019-06-10 12:49:29,541 Address resolution failed: gaierror(-2, 'Name or service not known')
2019-06-10 12:49:29,542 getaddrinfo failed: gaierror(-2, 'Name or service not known').
2019-06-10 12:49:29,542 AMQP connection workflow failed: AMQPConnectionWorkflowFailed: 1 exceptions in all; last exception - gaierror(-2, 'Name or service not known'); first exception - None.
2019-06-10 12:49:29,543 AMQPConnectionWorkflow - reporting failure: AMQPConnectionWorkflowFailed: 1 exceptions in all; last exception - gaierror(-2, 'Name or service not known'); first exception - None
2019-06-10 12:49:29,543 Connection workflow failed: AMQPConnectionWorkflowFailed: 1 exceptions in all; last exception - gaierror(-2, 'Name or service not known'); first exception - None
2019-06-10 12:49:29,543 Error in _create_connection().
Traceback (most recent call last):
File "/home/martina/PycharmProjects/mpegg_std/venv/lib/python3.6/site-packages/pika/adapters/blocking_connection.py", line 451, in _create_connection
raise self._reap_last_connection_workflow_error(error)
File "/home/martina/PycharmProjects/mpegg_std/venv/lib/python3.6/site-packages/pika/adapters/utils/selector_ioloop_adapter.py", line 564, in _resolve
self._flags)
File "/usr/lib/python3.6/socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
Traceback (most recent call last):
File "main.py", line 18, in <module>
rabbit = Rabbit()
File "/home/martina/PycharmProjects/mpegg_std/venv/lib/python3.6/site-packages/singleton_decorator/decorator.py", line 14, in __call__
self._instance = self.__wrapped__(*args, **kwargs)
File "/home/martina/PycharmProjects/mpegg_std/applicazione/encoder/mpeg-pre-encoder/my_util.py", line 26, in __init__
self.initConnect()
File "/home/martina/PycharmProjects/mpegg_std/applicazione/encoder/mpeg-pre-encoder/my_util.py", line 19, in initConnect
self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=RABBIT_HOST))
File "/home/martina/PycharmProjects/mpegg_std/venv/lib/python3.6/site-packages/pika/adapters/blocking_connection.py", line 360, in __init__
self._impl = self._create_connection(parameters, _impl_class)
File "/home/martina/PycharmProjects/mpegg_std/venv/lib/python3.6/site-packages/pika/adapters/blocking_connection.py", line 451, in _create_connection
raise self._reap_last_connection_workflow_error(error)
File "/home/martina/PycharmProjects/mpegg_std/venv/lib/python3.6/site-packages/pika/adapters/utils/selector_ioloop_adapter.py", line 564, in _resolve
self._flags)
File "/usr/lib/python3.6/socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
EDIT 3:
I isolated the error coming from the exception, but I get a more serious error. In fact the mpeg-pre-encoder service is fired when the classificatore service sends data to a queue. I think something is broken in rabbitMQ, how can I check if I'm right?
Below the mpeg-pre-encoder service building log:
Building mpeg-pre-encoder
Step 1/7 : FROM python:3
---> a4cc999cf2aa
Step 2/7 : RUN mkdir /src
---> Using cache
---> 31d8a2dc751d
Step 3/7 : ADD requirements.txt /src/.
---> Using cache
---> 9c58880cc0b1
Step 4/7 : WORKDIR /src
---> Using cache
---> 910b4249cc31
Step 5/7 : RUN pip install -r requirements.txt
---> Using cache
---> a8dc0f729257
Step 6/7 : ADD . /src/.
---> Using cache
---> 591968ce0427
Step 7/7 : CMD ["python", "main.py"]
---> Using cache
---> b1500f95a936
Successfully built b1500f95a936
Successfully tagged encoder_mpeg-pre-encoder:latest
It seems building fine but as soon as I run docker ps -a
9e7c4723cd98 encoder_mpeg-pre-encoder "python main.py" 17 seconds ago Exited (1) 14 seconds ago encoder_mpeg-pre-encoder_1
with a different error
Traceback (most recent call last):
File "main.py", line 45, in <module>
rabbit.wait_for_job(decoded_symbols_from_sam_file)
File "/src/my_util.py", line 55, in wait_for_job
self.channel_input.start_consuming()
File "/usr/local/lib/python3.7/site-packages/pika/adapters/blocking_connection.py", line 1822, in start_consuming
self.connection.process_data_events(time_limit=None)
File "/usr/local/lib/python3.7/site-packages/pika/adapters/blocking_connection.py", line 758, in process_data_events
self._dispatch_channel_events()
File "/usr/local/lib/python3.7/site-packages/pika/adapters/blocking_connection.py", line 521, in _dispatch_channel_events
impl_channel._get_cookie()._dispatch_events()
File "/usr/local/lib/python3.7/site-packages/pika/adapters/blocking_connection.py", line 1445, in _dispatch_events
evt.body)
File "/src/my_util.py", line 48, in callback
worker_function(eval(body))
File "<decorator-gen-1>", line 2, in decoded_symbols_from_sam_file
File "/usr/local/lib/python3.7/site-packages/prometheus_client/context_managers.py", line 66, in wrapped
return func(*args, **kwargs)
File "main.py", line 26, in decoded_symbols_from_sam_file
COUNTER_INPUT_FILE_SIZE.inc(path.getsize(file_path))
File "/usr/local/lib/python3.7/genericpath.py", line 50, in getsize
return os.stat(filename).st_size
FileNotFoundError: [Errno 2] No such file or directory: '/file/output/minisam/9fdb727a-5093-4800-a1ba-c807dc48ca99/CLASS_P chrM 1:7757.sam'
You are trying to run python 2 code with python 3 interpreter. They changed how the exceptions are handled between the versions, so now you can’t just raise 'My exception'.
Change the base container to python 2 or rewrite the code, python 2 is nearing the end of life in 2020.
I am currently running Airflow on Kubernetes in Google Cloud GCP. I based my project off of docker-airflow. I am able to start the UI but when I try to create a connection for google cloud and submit the connection I get the following errors.
ValueError: Fernet key must be 32 url-safe base64-encoded bytes.
[2018-09-21 19:45:13,345] AirflowException: Could not create Fernet object: Fernet key must be 32 url-safe base64-encoded bytes.
The first issue the docs recommend is to make sure you have cryptography installed, which I do. I installed both types, the one that comes with airflow and the standard one from PyPi.
pip3 install apache-airflow[kubernetes,crypto] and also tried
pip install cryptography
I tried to run the commands for generating and storing env variables as explained in the documentation, found here. (and shown below)
1) Either generate a fernet key manually and add to airflow.cfg
2) Set the environment variable and restarting the server.
python -c "from cryptography.fernet import Fernet;
print(Fernet.generate_key().decode())"
Example Key:81HqDtbqAywKSOumSha3BhWNOdQ26slT6K0YaZeZyPs=
Using kubernetes I am unable to restart the server using the typical method of shutting down the process ID since its tied to the container. I also tried putting a generated key (above) in the configmaps.yaml file of the kubernetes cluster (equal to airflow.cfg when deployed).
I tried running the GCP connection through DAG, via the UI, and manually by using the airflow command line client. All three methods returned the same error. I am including a picture of the UI submission here along with the full stack-trace.
Question
Why might this be happening? Is the fernet key not being generated? Is it not being saved on the underlying volume maybe?*
Thanks for the help.
-RR
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/airflow/models.py", line 159, in get_fernet
_fernet = Fernet(configuration.conf.get('core', 'FERNET_KEY').encode('utf-8'))
File "/usr/local/lib/python3.6/site-packages/cryptography/fernet.py", line 37, in __init__
"Fernet key must be 32 url-safe base64-encoded bytes."
ValueError: Fernet key must be 32 url-safe base64-encoded bytes.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1982, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1614, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1517, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/local/lib/python3.6/site-packages/flask/_compat.py", line 33, in reraise
raise value
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1612, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1598, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/local/lib/python3.6/site-packages/flask_appbuilder/security/decorators.py", line 26, in wraps
return f(self, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/flask_appbuilder/views.py", line 524, in edit
widgets = self._edit(pk)
File "/usr/local/lib/python3.6/site-packages/flask_appbuilder/baseviews.py", line 965, in _edit
form.populate_obj(item)
File "/usr/local/lib/python3.6/site-packages/wtforms/form.py", line 96, in populate_obj
field.populate_obj(obj, name)
File "/usr/local/lib/python3.6/site-packages/wtforms/fields/core.py", line 330, in populate_obj
setattr(obj, name, self.data)
File "<string>", line 1, in __set__
File "/usr/local/lib/python3.6/site-packages/airflow/models.py", line 731, in set_extra
fernet = get_fernet()
File "/usr/local/lib/python3.6/site-packages/airflow/models.py", line 163, in get_fernet
raise AirflowException("Could not create Fernet object: {}".format(ve))
airflow.exceptions.AirflowException: Could not create Fernet object:
Fernet key must be 32 url-safe base64-encoded bytes.
This is the YAML for the underlying persisted volumes.
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: airflow-dags
namespace: data
spec:
accessModes:
- ReadOnlyMany
storageClassName: standard
resources:
requests:
storage: 8Gi
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: airflow-logs
namespace: data
spec:
accessModes:
- ReadOnlyMany
storageClassName: standard
resources:
requests:
storage: 8Gi
This is the airflow configuration YAML.
apiVersion: apps/v1
kind: Deployment
metadata:
name: airflow
namespace: data
labels:
name: airflow
spec:
replicas: 1
selector:
matchLabels:
name: airflow
template:
metadata:
labels:
name: airflow
spec:
serviceAccountName: spark-service-account
automountServiceAccountToken: true
initContainers:
- name: "init"
image: <image_name>
imagePullPolicy: Always
volumeMounts:
- name: airflow-configmap
mountPath: /root/airflow/airflow.cfg
subPath: airflow.cfg
- name: airflow-dags
mountPath: /root/airflow/dags
# - name: test-volume
# mountPath: /root/test_volume
env:
- name: SQL_ALCHEMY_CONN
valueFrom:
secretKeyRef:
name: airflow-secrets
key: sql_alchemy_conn
command:
- "bash"
args:
- "-cx"
- "airflow initdb || true && airflow create_user -u airflow -l airflow -f jon -e airflow#apache.org -r Admin -p airflow || true"
containers:
- name: webserver
image: <image_name>
imagePullPolicy: IfNotPresent
ports:
- name: webserver
containerPort: 8080
env:
- name: <namespace_name>
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: SQL_ALCHEMY_CONN
valueFrom:
secretKeyRef:
name: airflow-secrets
key: sql_alchemy_conn
command: ["/bin/sh", "-c"]
args: ["airflow webserver"]
volumeMounts:
- name: airflow-configmap
mountPath: /root/airflow/airflow.cfg
subPath: airflow.cfg
- name: airflow-dags
mountPath: /root/airflow/dags
- name: airflow-logs
mountPath: /root/airflow/logs
# readinessProbe:
# initialDelaySeconds: 5
# timeoutSeconds: 5
# periodSeconds: 5
# httpGet:
# path: /login
# port: 8080
# livenessProbe:
# initialDelaySeconds: 5
# timeoutSeconds: 5
# failureThreshold: 5
# httpGet:
# path: /login
# port: 8080
- name: scheduler
image: image-name
imagePullPolicy: IfNotPresent
env:
- name: namespace_name
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: SQL_ALCHEMY_CONN
valueFrom:
secretKeyRef:
name: airflow-secrets
key: sql_alchemy_conn
command: ["/bin/sh", "-c"]
args: ["cp ./dags/* /root/airflow/dags/; airflow scheduler"]
volumeMounts:
- name: airflow-configmap
mountPath: /root/airflow/airflow.cfg
subPath: airflow.cfg
- name: airflow-dags
mountPath: /root/airflow/dags
- name: airflow-logs
mountPath: /root/airflow/logs
volumes:
- name: airflow-configmap
configMap:
name: airflow-configmap
- name: airflow-dags
persistentVolumeClaim:
claimName: airflow-dags
- name: airflow-logs
persistentVolumeClaim:
claimName: airflow-logs
---
apiVersion: v1
kind: Service
metadata:
name: airflow
namespace: data
spec:
type: NodePort
ports:
- port: 8080
nodePort: 30809
selector:
name: airflow
Restart the worker and webserver.
Your worker and webserver are operating on the old fernet key. You changed the key in your config, so all your newly stored or modified Connections will use the new key, but the webserver/worker are still operating on the old key. They will never match and continue to give this error, till they're restarted.