Scheduling Python script in Airflow
To schedule a Python script or Python function in Airflow, we use PythonOperator.
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
def my_python_function():
# your code goes here
print('Hello')
def my_python_function_with_context(**context):
# For more detail about "context" object,
# please refer to https://blog.duyet.net/2019/08/airflow-context.html
ds = context['ds']
print(f'Dag run at {ds}')
dag = DAG('dag_id')
PythonOperator(dag=dag,
task_id='my_python_function',
python_callable=my_python_function)
PythonOperator(dag=dag,
task_id='my_python_function_with_context',
provide_context=True,
python_callable=my_python_function_with_context)
Passing in arguments
Use the op_args and op_kwargs arguments to pass additional arguments to the Python callable.
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
def my_python_function(ds, lucky_number, **kwargs):
print(f'Dag run at {ds}')
print(f'Random number is {lucky_number}')
dag = DAG('dag_id')
run_this = PythonOperator(dag=dag,
task_id='my_python_function',
provide_context=True,
python_callable=my_python_function,
op_kwargs={
'lucky_number': 99
})