diff --git a/BREEZE.rst b/BREEZE.rst index 555a09b140114..90663657a85ad 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1830,10 +1830,15 @@ This is the current syntax for `./breeze <./breeze>`_: 9.6 10 11 12 13 --mysql-version MYSQL_VERSION - Mysql version used. One of: + MySql version used. One of: 5.7 8 + --mssql-version MSSQL_VERSION + MSSql version used. One of: + + 2017-latest 2019-latest + -v, --verbose Show verbose information about executed docker, kind, kubectl, helm commands. Useful for debugging - when you run breeze with --verbose flags you will be able to see the commands @@ -2299,10 +2304,15 @@ This is the current syntax for `./breeze <./breeze>`_: 9.6 10 11 12 13 --mysql-version MYSQL_VERSION - Mysql version used. One of: + MySql version used. One of: 5.7 8 + --mssql-version MSSQL_VERSION + MSSql version used. One of: + + 2017-latest 2019-latest + **************************************************************************************************** Enable production image diff --git a/airflow/executors/celery_executor.py b/airflow/executors/celery_executor.py index b2c50163af5b2..e3fc398a2c071 100644 --- a/airflow/executors/celery_executor.py +++ b/airflow/executors/celery_executor.py @@ -46,7 +46,7 @@ from airflow.configuration import conf from airflow.exceptions import AirflowException, AirflowTaskTimeout from airflow.executors.base_executor import BaseExecutor, CommandType, EventBufferValueType -from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance, TaskInstanceKey +from airflow.models.taskinstance import TaskInstance, TaskInstanceKey from airflow.stats import Stats from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.net import get_hostname @@ -154,15 +154,15 @@ def __init__(self, exception: Exception, exception_traceback: str): # Task instance that is sent over Celery queues -# TaskInstanceKey, SimpleTaskInstance, Command, queue_name, CallableTask -TaskInstanceInCelery = Tuple[TaskInstanceKey, SimpleTaskInstance, CommandType, Optional[str], Task] +# TaskInstanceKey, Command, queue_name, CallableTask +TaskInstanceInCelery = Tuple[TaskInstanceKey, CommandType, Optional[str], Task] def send_task_to_executor( task_tuple: TaskInstanceInCelery, ) -> Tuple[TaskInstanceKey, CommandType, Union[AsyncResult, ExceptionWithTraceback]]: """Sends task to executor.""" - key, _, command, queue, task_to_run = task_tuple + key, command, queue, task_to_run = task_tuple try: with timeout(seconds=OPERATION_TIMEOUT): result = task_to_run.apply_async(args=[command], queue=queue) @@ -250,8 +250,8 @@ def trigger_tasks(self, open_slots: int) -> None: task_tuples_to_send: List[TaskInstanceInCelery] = [] for _ in range(min(open_slots, len(self.queued_tasks))): - key, (command, _, queue, simple_ti) = sorted_queue.pop(0) - task_tuple = (key, simple_ti, command, queue, execute_command) + key, (command, _, queue, _) = sorted_queue.pop(0) + task_tuple = (key, command, queue, execute_command) task_tuples_to_send.append(task_tuple) if key not in self.task_publish_retries: self.task_publish_retries[key] = 1 @@ -260,7 +260,7 @@ def trigger_tasks(self, open_slots: int) -> None: self._process_tasks(task_tuples_to_send) def _process_tasks(self, task_tuples_to_send: List[TaskInstanceInCelery]) -> None: - first_task = next(t[4] for t in task_tuples_to_send) + first_task = next(t[3] for t in task_tuples_to_send) # Celery state queries will stuck if we do not use one same backend # for all tasks. diff --git a/airflow/providers/airbyte/CHANGELOG.rst b/airflow/providers/airbyte/CHANGELOG.rst index 3863acdc0b0ed..9a9371c0fb15f 100644 --- a/airflow/providers/airbyte/CHANGELOG.rst +++ b/airflow/providers/airbyte/CHANGELOG.rst @@ -19,6 +19,29 @@ Changelog --------- +2.1.0 +..... + +Breaking changes +~~~~~~~~~~~~~~~~ + + +Features +~~~~~~~~ + + +Bug Fixes +~~~~~~~~~ + +* ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``AirbyteHook - Consider incomplete status (#16965)`` + * ``Prepare documentation for July release of providers. (#17015)`` + * ``Updating Airbyte example DAG to use XComArgs (#16867)`` + * ``Removes pylint from our toolchain (#16682)`` + 2.0.0 ..... diff --git a/airflow/providers/airbyte/provider.yaml b/airflow/providers/airbyte/provider.yaml index 87bb1d123e652..3ea81858e04d7 100644 --- a/airflow/providers/airbyte/provider.yaml +++ b/airflow/providers/airbyte/provider.yaml @@ -22,6 +22,7 @@ description: | `Airbyte `__ versions: + - 2.1.0 - 2.0.0 - 1.0.0 diff --git a/airflow/providers/amazon/CHANGELOG.rst b/airflow/providers/amazon/CHANGELOG.rst index eac711dc1a16f..788c5c4d0aad1 100644 --- a/airflow/providers/amazon/CHANGELOG.rst +++ b/airflow/providers/amazon/CHANGELOG.rst @@ -29,6 +29,8 @@ Features * ``Added select_query to the templated fields in RedshiftToS3Operator (#16767)`` * ``AWS Hook - allow IDP HTTP retry (#12639) (#16612)`` * ``Update Boto3 API calls in ECSOperator (#16050)`` +* ``Adding custom Salesforce connection type + SalesforceToS3Operator updates (#17162)`` +* ``Adding SalesforceToS3Operator to Amazon Provider (#17094)`` Bug Fixes ~~~~~~~~~ @@ -36,12 +38,19 @@ Bug Fixes * ``AWS DataSync default polling adjusted from 5s to 30s (#11011)`` * ``Fix wrong template_fields_renderers for AWS operators (#16820)`` * ``AWS DataSync cancel task on exception (#11011) (#16589)`` +* ``Fixed template_fields_renderers for Amazon provider (#17087)`` +* ``removing try-catch block (#17081)`` +* ``ECSOperator / pass context to self.xcom_pull as it was missing (when using reattach) (#17141)`` +* ``Made S3ToRedshiftOperator transaction safe (#17117)`` .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``Removes pylint from our toolchain (#16682)`` * ``Bump sphinxcontrib-spelling and minor improvements (#16675)`` * ``Prepare documentation for July release of providers. (#17015)`` + * ``Added docs & doc ref's for AWS transfer operators between SFTP & S3 (#16964)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` + * ``Updating Amazon-AWS example DAGs to use XComArgs (#16868)`` 2.0.0 ..... diff --git a/airflow/providers/amazon/aws/example_dags/example_dms_full_load_task.py b/airflow/providers/amazon/aws/example_dags/example_dms_full_load_task.py index 9833d1d92d1f8..681e8dcdd67f8 100644 --- a/airflow/providers/amazon/aws/example_dags/example_dms_full_load_task.py +++ b/airflow/providers/amazon/aws/example_dags/example_dms_full_load_task.py @@ -47,17 +47,16 @@ ] } -DEFAULT_ARGS = { - 'owner': 'airflow', - 'depends_on_past': False, - 'email': ['airflow@example.com'], - 'email_on_failure': False, - 'email_on_retry': False, -} with DAG( dag_id='dms_full_load_task_run_dag', - default_args=DEFAULT_ARGS, + default_args={ + 'owner': 'airflow', + 'depends_on_past': False, + 'email': ['airflow@example.com'], + 'email_on_failure': False, + 'email_on_retry': False, + }, dagrun_timeout=timedelta(hours=2), start_date=days_ago(2), schedule_interval='0 3 * * *', @@ -78,22 +77,27 @@ # [START howto_dms_start_task_operator] start_task = DmsStartTaskOperator( task_id='start_task', - replication_task_arn='{{ task_instance.xcom_pull(task_ids="create_task", key="return_value") }}', + replication_task_arn=create_task.output, ) # [END howto_dms_start_task_operator] # [START howto_dms_task_completed_sensor] wait_for_completion = DmsTaskCompletedSensor( task_id='wait_for_completion', - replication_task_arn='{{ task_instance.xcom_pull(task_ids="create_task", key="return_value") }}', + replication_task_arn=create_task.output, ) # [END howto_dms_task_completed_sensor] # [START howto_dms_delete_task_operator] delete_task = DmsDeleteTaskOperator( task_id='delete_task', - replication_task_arn='{{ task_instance.xcom_pull(task_ids="create_task", key="return_value") }}', + replication_task_arn=create_task.output, ) # [END howto_dms_delete_task_operator] - create_task >> start_task >> wait_for_completion >> delete_task + start_task >> wait_for_completion >> delete_task + + # Task dependencies created via `XComArgs`: + # create_task >> start_task + # create_task >> wait_for_completion + # create_task >> delete_task diff --git a/airflow/providers/amazon/aws/example_dags/example_ecs_fargate.py b/airflow/providers/amazon/aws/example_dags/example_ecs_fargate.py index 94cecba7f9269..63d3f74ad47bd 100644 --- a/airflow/providers/amazon/aws/example_dags/example_ecs_fargate.py +++ b/airflow/providers/amazon/aws/example_dags/example_ecs_fargate.py @@ -28,17 +28,15 @@ from airflow import DAG from airflow.providers.amazon.aws.operators.ecs import ECSOperator -DEFAULT_ARGS = { - "owner": "airflow", - "depends_on_past": False, - "email": ["airflow@example.com"], - "email_on_failure": False, - "email_on_retry": False, -} - dag = DAG( dag_id="ecs_fargate_dag", - default_args=DEFAULT_ARGS, + default_args={ + "owner": "airflow", + "depends_on_past": False, + "email": ["airflow@example.com"], + "email_on_failure": False, + "email_on_retry": False, + }, default_view="graph", schedule_interval=None, start_date=datetime.datetime(2020, 1, 1), diff --git a/airflow/providers/amazon/aws/example_dags/example_emr_job_flow_automatic_steps.py b/airflow/providers/amazon/aws/example_dags/example_emr_job_flow_automatic_steps.py index 3077944a0b7dc..707482b0c63d7 100644 --- a/airflow/providers/amazon/aws/example_dags/example_emr_job_flow_automatic_steps.py +++ b/airflow/providers/amazon/aws/example_dags/example_emr_job_flow_automatic_steps.py @@ -25,14 +25,6 @@ from airflow.providers.amazon.aws.sensors.emr_job_flow import EmrJobFlowSensor from airflow.utils.dates import days_ago -DEFAULT_ARGS = { - 'owner': 'airflow', - 'depends_on_past': False, - 'email': ['airflow@example.com'], - 'email_on_failure': False, - 'email_on_retry': False, -} - # [START howto_operator_emr_automatic_steps_config] SPARK_STEPS = [ { @@ -69,7 +61,13 @@ with DAG( dag_id='emr_job_flow_automatic_steps_dag', - default_args=DEFAULT_ARGS, + default_args={ + 'owner': 'airflow', + 'depends_on_past': False, + 'email': ['airflow@example.com'], + 'email_on_failure': False, + 'email_on_retry': False, + }, dagrun_timeout=timedelta(hours=2), start_date=days_ago(2), schedule_interval='0 3 * * *', @@ -86,9 +84,10 @@ job_sensor = EmrJobFlowSensor( task_id='check_job_flow', - job_flow_id="{{ task_instance.xcom_pull(task_ids='create_job_flow', key='return_value') }}", + job_flow_id=job_flow_creator.output, aws_conn_id='aws_default', ) - - job_flow_creator >> job_sensor # [END howto_operator_emr_automatic_steps_tasks] + + # Task dependency created via `XComArgs`: + # job_flow_creator >> job_sensor diff --git a/airflow/providers/amazon/aws/example_dags/example_emr_job_flow_manual_steps.py b/airflow/providers/amazon/aws/example_dags/example_emr_job_flow_manual_steps.py index 1eb857a6a2d29..e7af7fce29107 100644 --- a/airflow/providers/amazon/aws/example_dags/example_emr_job_flow_manual_steps.py +++ b/airflow/providers/amazon/aws/example_dags/example_emr_job_flow_manual_steps.py @@ -30,14 +30,6 @@ from airflow.providers.amazon.aws.sensors.emr_step import EmrStepSensor from airflow.utils.dates import days_ago -DEFAULT_ARGS = { - 'owner': 'airflow', - 'depends_on_past': False, - 'email': ['airflow@example.com'], - 'email_on_failure': False, - 'email_on_retry': False, -} - SPARK_STEPS = [ { 'Name': 'calculate_pi', @@ -71,7 +63,13 @@ with DAG( dag_id='emr_job_flow_manual_steps_dag', - default_args=DEFAULT_ARGS, + default_args={ + 'owner': 'airflow', + 'depends_on_past': False, + 'email': ['airflow@example.com'], + 'email_on_failure': False, + 'email_on_retry': False, + }, dagrun_timeout=timedelta(hours=2), start_date=days_ago(2), schedule_interval='0 3 * * *', @@ -88,23 +86,28 @@ step_adder = EmrAddStepsOperator( task_id='add_steps', - job_flow_id="{{ task_instance.xcom_pull(task_ids='create_job_flow', key='return_value') }}", + job_flow_id=cluster_creator.output, aws_conn_id='aws_default', steps=SPARK_STEPS, ) step_checker = EmrStepSensor( task_id='watch_step', - job_flow_id="{{ task_instance.xcom_pull('create_job_flow', key='return_value') }}", + job_flow_id=cluster_creator.output, step_id="{{ task_instance.xcom_pull(task_ids='add_steps', key='return_value')[0] }}", aws_conn_id='aws_default', ) cluster_remover = EmrTerminateJobFlowOperator( task_id='remove_cluster', - job_flow_id="{{ task_instance.xcom_pull(task_ids='create_job_flow', key='return_value') }}", + job_flow_id=cluster_creator.output, aws_conn_id='aws_default', ) - cluster_creator >> step_adder >> step_checker >> cluster_remover + step_adder >> step_checker >> cluster_remover # [END howto_operator_emr_manual_steps_tasks] + + # Task dependencies created via `XComArgs`: + # cluster_creator >> step_adder + # cluster_creator >> step_checker + # cluster_creator >> cluster_remover diff --git a/airflow/providers/amazon/aws/example_dags/example_google_api_to_s3_transfer_advanced.py b/airflow/providers/amazon/aws/example_dags/example_google_api_to_s3_transfer_advanced.py index 9b842c286bcd0..9f2b26f7be3ba 100644 --- a/airflow/providers/amazon/aws/example_dags/example_google_api_to_s3_transfer_advanced.py +++ b/airflow/providers/amazon/aws/example_dags/example_google_api_to_s3_transfer_advanced.py @@ -37,7 +37,7 @@ from airflow import DAG from airflow.operators.dummy import DummyOperator -from airflow.operators.python import BranchPythonOperator +from airflow.operators.python import BranchPythonOperator, get_current_context from airflow.providers.amazon.aws.transfers.google_api_to_s3 import GoogleApiToS3Operator from airflow.utils.dates import days_ago @@ -53,12 +53,13 @@ # [START howto_operator_google_api_to_s3_transfer_advanced_task_1_2] -def _check_and_transform_video_ids(xcom_key, task_ids, task_instance, **kwargs): - video_ids_response = task_instance.xcom_pull(task_ids=task_ids, key=xcom_key) +def _check_and_transform_video_ids(task_output): + video_ids_response = task_output video_ids = [item['id']['videoId'] for item in video_ids_response['items']] if video_ids: - task_instance.xcom_push(key='video_ids', value={'id': ','.join(video_ids)}) + context = get_current_context() + context["task_instance"].xcom_push(key='video_ids', value={'id': ','.join(video_ids)}) return 'video_data_to_s3' return 'no_video_ids' @@ -98,7 +99,7 @@ def _check_and_transform_video_ids(xcom_key, task_ids, task_instance, **kwargs): # [START howto_operator_google_api_to_s3_transfer_advanced_task_1_1] task_check_and_transform_video_ids = BranchPythonOperator( python_callable=_check_and_transform_video_ids, - op_args=[task_video_ids_to_s3.google_api_response_via_xcom, task_video_ids_to_s3.task_id], + op_args=[task_video_ids_to_s3.output[task_video_ids_to_s3.google_api_response_via_xcom]], task_id='check_and_transform_video_ids', ) # [END howto_operator_google_api_to_s3_transfer_advanced_task_1_1] @@ -121,4 +122,7 @@ def _check_and_transform_video_ids(xcom_key, task_ids, task_instance, **kwargs): # [START howto_operator_google_api_to_s3_transfer_advanced_task_2_1] task_no_video_ids = DummyOperator(task_id='no_video_ids') # [END howto_operator_google_api_to_s3_transfer_advanced_task_2_1] - task_video_ids_to_s3 >> task_check_and_transform_video_ids >> [task_video_data_to_s3, task_no_video_ids] + task_check_and_transform_video_ids >> [task_video_data_to_s3, task_no_video_ids] + + # Task dependency created via `XComArgs`: + # task_video_ids_to_s3 >> task_check_and_transform_video_ids diff --git a/airflow/providers/apache/beam/.latest-doc-only-change.txt b/airflow/providers/apache/beam/.latest-doc-only-change.txt index e7e8156d80b9e..20a51db63f4ac 100644 --- a/airflow/providers/apache/beam/.latest-doc-only-change.txt +++ b/airflow/providers/apache/beam/.latest-doc-only-change.txt @@ -1 +1 @@ -b916b7507921129dc48d6add1bdc4b923b60c9b9 +d02ded65eaa7d2281e249b3fa028605d1b4c52fb diff --git a/airflow/providers/apache/cassandra/.latest-doc-only-change.txt b/airflow/providers/apache/cassandra/.latest-doc-only-change.txt index e7e8156d80b9e..da9862433fd0e 100644 --- a/airflow/providers/apache/cassandra/.latest-doc-only-change.txt +++ b/airflow/providers/apache/cassandra/.latest-doc-only-change.txt @@ -1 +1 @@ -b916b7507921129dc48d6add1bdc4b923b60c9b9 +91f4d80ff09093de49478214c5bd027e02c92a0e diff --git a/airflow/providers/apache/druid/CHANGELOG.rst b/airflow/providers/apache/druid/CHANGELOG.rst index 3505973e659ba..7700a2ef71849 100644 --- a/airflow/providers/apache/druid/CHANGELOG.rst +++ b/airflow/providers/apache/druid/CHANGELOG.rst @@ -19,6 +19,28 @@ Changelog --------- +2.0.1 +..... + +Breaking changes +~~~~~~~~~~~~~~~~ + + +Features +~~~~~~~~ + + +Bug Fixes +~~~~~~~~~ + +* ``Fix error in Druid connection attribute retrieval (#17095)`` +* ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Prepare documentation for July release of providers. (#17015)`` + * ``Removes pylint from our toolchain (#16682)`` + 2.0.0 ..... diff --git a/airflow/providers/apache/druid/provider.yaml b/airflow/providers/apache/druid/provider.yaml index 66974e68ffd9d..10fb84b6e122a 100644 --- a/airflow/providers/apache/druid/provider.yaml +++ b/airflow/providers/apache/druid/provider.yaml @@ -22,6 +22,7 @@ description: | `Apache Druid `__. versions: + - 2.0.1 - 2.0.0 - 1.1.0 - 1.0.1 diff --git a/airflow/providers/apache/hdfs/.latest-doc-only-change.txt b/airflow/providers/apache/hdfs/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/apache/hdfs/.latest-doc-only-change.txt +++ b/airflow/providers/apache/hdfs/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/apache/hive/CHANGELOG.rst b/airflow/providers/apache/hive/CHANGELOG.rst index 4f86fbecb4c00..26b9b3e96aada 100644 --- a/airflow/providers/apache/hive/CHANGELOG.rst +++ b/airflow/providers/apache/hive/CHANGELOG.rst @@ -31,6 +31,8 @@ Features appropriate section above if needed. Do not delete the lines(!): * ``Removes pylint from our toolchain (#16682)`` * ``Prepare documentation for July release of providers. (#17015)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` + * ``Updating Apache example DAGs to use XComArgs (#16869)`` 2.0.0 ..... diff --git a/airflow/providers/apache/kylin/.latest-doc-only-change.txt b/airflow/providers/apache/kylin/.latest-doc-only-change.txt index b620a033bd63e..da9862433fd0e 100644 --- a/airflow/providers/apache/kylin/.latest-doc-only-change.txt +++ b/airflow/providers/apache/kylin/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +91f4d80ff09093de49478214c5bd027e02c92a0e diff --git a/airflow/providers/apache/livy/.latest-doc-only-change.txt b/airflow/providers/apache/livy/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/apache/livy/.latest-doc-only-change.txt +++ b/airflow/providers/apache/livy/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/apache/pig/.latest-doc-only-change.txt b/airflow/providers/apache/pig/.latest-doc-only-change.txt index 8e901d0d2aed3..da9862433fd0e 100644 --- a/airflow/providers/apache/pig/.latest-doc-only-change.txt +++ b/airflow/providers/apache/pig/.latest-doc-only-change.txt @@ -1 +1 @@ -40a2476a5db14ee26b5108d72635da116eab720b +91f4d80ff09093de49478214c5bd027e02c92a0e diff --git a/airflow/providers/apache/pinot/.latest-doc-only-change.txt b/airflow/providers/apache/pinot/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/apache/pinot/.latest-doc-only-change.txt +++ b/airflow/providers/apache/pinot/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/apache/spark/.latest-doc-only-change.txt b/airflow/providers/apache/spark/.latest-doc-only-change.txt index b620a033bd63e..da9862433fd0e 100644 --- a/airflow/providers/apache/spark/.latest-doc-only-change.txt +++ b/airflow/providers/apache/spark/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +91f4d80ff09093de49478214c5bd027e02c92a0e diff --git a/airflow/providers/apache/sqoop/CHANGELOG.rst b/airflow/providers/apache/sqoop/CHANGELOG.rst index ff3484eb06a7f..8a20450c48332 100644 --- a/airflow/providers/apache/sqoop/CHANGELOG.rst +++ b/airflow/providers/apache/sqoop/CHANGELOG.rst @@ -31,6 +31,7 @@ Bug Fixes appropriate section above if needed. Do not delete the lines(!): * ``Removes pylint from our toolchain (#16682)`` * ``Prepare documentation for July release of providers. (#17015)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` 2.0.0 ..... diff --git a/airflow/providers/asana/.latest-doc-only-change.txt b/airflow/providers/asana/.latest-doc-only-change.txt index b620a033bd63e..ba8d0c7834aba 100644 --- a/airflow/providers/asana/.latest-doc-only-change.txt +++ b/airflow/providers/asana/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +0dbd0f420cc08e011317e2a9f21f92fff4a64c1b diff --git a/airflow/providers/asana/example_dags/example_asana.py b/airflow/providers/asana/example_dags/example_asana.py index 76ecd61df9316..82aef7dd88439 100644 --- a/airflow/providers/asana/example_dags/example_asana.py +++ b/airflow/providers/asana/example_dags/example_asana.py @@ -29,10 +29,6 @@ ) from airflow.utils.dates import days_ago -default_args = { - "owner": "airflow", -} - ASANA_TASK_TO_UPDATE = os.environ.get("ASANA_TASK_TO_UPDATE") ASANA_TASK_TO_DELETE = os.environ.get("ASANA_TASK_TO_DELETE") # This example assumes a default project ID has been specified in the connection. If you @@ -45,7 +41,6 @@ with DAG( "example_asana", - default_args=default_args, start_date=days_ago(1), tags=["example"], ) as dag: diff --git a/airflow/providers/cncf/kubernetes/CHANGELOG.rst b/airflow/providers/cncf/kubernetes/CHANGELOG.rst index b03eed650961a..3135c57a1576f 100644 --- a/airflow/providers/cncf/kubernetes/CHANGELOG.rst +++ b/airflow/providers/cncf/kubernetes/CHANGELOG.rst @@ -22,6 +22,37 @@ Changelog 2.0.1 ..... +Breaking changes +~~~~~~~~~~~~~~~~ + + +Features +~~~~~~~~ + + +Bug Fixes +~~~~~~~~~ + +* ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` +* ``BugFix: Using 'json' string in template_field causes issue with K8s Operators (#16930)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Simplify 'default_args' in Kubernetes example DAGs (#16870)`` + * ``Enable using custom pod launcher in Kubernetes Pod Operator (#16945)`` + * ``Prepare documentation for July release of providers. (#17015)`` + * ``Updating task dependencies (#16624)`` + * ``Removes pylint from our toolchain (#16682)`` + +2.1.0 +..... + + +Features +~~~~~~~~ + +* ``Enable using custom pod launcher in Kubernetes Pod Operator (#16945)`` + Bug Fixes ~~~~~~~~~ @@ -32,6 +63,7 @@ Bug Fixes * ``Updating task dependencies (#16624)`` * ``Removes pylint from our toolchain (#16682)`` * ``Prepare documentation for July release of providers. (#17015)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` 2.0.0 ..... diff --git a/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py b/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py index ef338540031f0..8be6757308243 100644 --- a/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +++ b/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py @@ -29,7 +29,7 @@ class SparkKubernetesOperator(BaseOperator): For more detail about Spark Application Object have a look at the reference: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/v1beta2-1.1.0-2.4.5/docs/api-docs.md#sparkapplication - :param application_file: filepath to kubernetes custom_resource_definition of sparkApplication + :param application_file: Defines Kubernetes 'custom_resource_definition' of 'sparkApplication' as either a path to a '.json' file or a JSON string. :type application_file: str :param namespace: kubernetes namespace to put sparkApplication :type namespace: str diff --git a/airflow/providers/databricks/.latest-doc-only-change.txt b/airflow/providers/databricks/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/databricks/.latest-doc-only-change.txt +++ b/airflow/providers/databricks/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/datadog/.latest-doc-only-change.txt b/airflow/providers/datadog/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/datadog/.latest-doc-only-change.txt +++ b/airflow/providers/datadog/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/dingding/.latest-doc-only-change.txt b/airflow/providers/dingding/.latest-doc-only-change.txt index 5bda81758205d..ba8d0c7834aba 100644 --- a/airflow/providers/dingding/.latest-doc-only-change.txt +++ b/airflow/providers/dingding/.latest-doc-only-change.txt @@ -1 +1 @@ -814e471d137aad68bd64a21d20736e7b88403f97 +0dbd0f420cc08e011317e2a9f21f92fff4a64c1b diff --git a/airflow/providers/dingding/example_dags/example_dingding.py b/airflow/providers/dingding/example_dags/example_dingding.py index 70141d6ae7610..727fcc7e66669 100644 --- a/airflow/providers/dingding/example_dags/example_dingding.py +++ b/airflow/providers/dingding/example_dags/example_dingding.py @@ -24,11 +24,6 @@ from airflow.providers.dingding.operators.dingding import DingdingOperator from airflow.utils.dates import days_ago -args = { - 'owner': 'airflow', - 'retries': 3, -} - # [START howto_operator_dingding_failure_callback] def failure_callback(context): @@ -55,12 +50,11 @@ def failure_callback(context): ).execute(context) -args['on_failure_callback'] = failure_callback # [END howto_operator_dingding_failure_callback] with DAG( dag_id='example_dingding_operator', - default_args=args, + default_args={'retries': 3, 'on_failure_callback': failure_callback}, schedule_interval='@once', dagrun_timeout=timedelta(minutes=60), start_date=days_ago(2), diff --git a/airflow/providers/docker/CHANGELOG.rst b/airflow/providers/docker/CHANGELOG.rst index 8f9cc6f7e258b..69272d0a8a549 100644 --- a/airflow/providers/docker/CHANGELOG.rst +++ b/airflow/providers/docker/CHANGELOG.rst @@ -27,11 +27,13 @@ Features * ``Adds option to disable mounting temporary folder in DockerOperator (#16932)`` - Bug Fixes ~~~~~~~~~ * ``[FIX] Docker provider - retry docker in docker (#17061)`` +* ``fix string encoding when using xcom / json (#13536)`` +* if ``xcom_all`` is set to ``False``, only the last line of the log (separated by ``\n``) will be + included in the XCom value The ``DockerOperator`` in version 2.0.0 did not work for remote Docker Engine or Docker-In-Docker case. That was an unintended side effect of #15843 that has been fixed in #16932. There is a fallback mode @@ -43,6 +45,8 @@ using the new parameter to disable mounting the folder. * ``Removes pylint from our toolchain (#16682)`` * ``Prepare documentation for July release of providers. (#17015)`` * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` + * ``Prepares documentation for RC2 release of Docker Provider (#17066)`` + * ``Updating Docker example DAGs to use XComArgs (#16871)`` 2.0.0 ..... diff --git a/airflow/providers/docker/example_dags/example_docker.py b/airflow/providers/docker/example_dags/example_docker.py index a5826bc805e9a..3cb1eb8f1e5ea 100644 --- a/airflow/providers/docker/example_dags/example_docker.py +++ b/airflow/providers/docker/example_dags/example_docker.py @@ -22,19 +22,17 @@ from airflow.providers.docker.operators.docker import DockerOperator from airflow.utils.dates import days_ago -default_args = { - 'owner': 'airflow', - 'depends_on_past': False, - 'email': ['airflow@example.com'], - 'email_on_failure': False, - 'email_on_retry': False, - 'retries': 1, - 'retry_delay': timedelta(minutes=5), -} - dag = DAG( 'docker_sample', - default_args=default_args, + default_args={ + 'owner': 'airflow', + 'depends_on_past': False, + 'email': ['airflow@example.com'], + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), + }, schedule_interval=timedelta(minutes=10), start_date=days_ago(2), ) diff --git a/airflow/providers/docker/example_dags/example_docker_copy_data.py b/airflow/providers/docker/example_dags/example_docker_copy_data.py index 9dc8478041b1b..a6ab50e6b39b7 100644 --- a/airflow/providers/docker/example_dags/example_docker_copy_data.py +++ b/airflow/providers/docker/example_dags/example_docker_copy_data.py @@ -35,19 +35,17 @@ from airflow.providers.docker.operators.docker import DockerOperator from airflow.utils.dates import days_ago -default_args = { - "owner": "airflow", - "depends_on_past": False, - "email": ["airflow@example.com"], - "email_on_failure": False, - "email_on_retry": False, - "retries": 1, - "retry_delay": timedelta(minutes=5), -} - dag = DAG( "docker_sample_copy_data", - default_args=default_args, + default_args={ + "owner": "airflow", + "depends_on_past": False, + "email": ["airflow@example.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=5), + }, schedule_interval=timedelta(minutes=10), start_date=days_ago(2), ) @@ -65,16 +63,11 @@ dag=dag, ) - -def is_data_available(*args, **kwargs): - """Return True if data exists in XCom table for view_file task, false otherwise.""" - ti = kwargs["ti"] - data = ti.xcom_pull(key=None, task_ids="view_file") - return not data == "" - - t_is_data_available = ShortCircuitOperator( - task_id="check_if_data_available", python_callable=is_data_available, dag=dag + task_id="check_if_data_available", + python_callable=lambda task_output: not task_output == "", + op_kwargs=dict(task_output=t_view.output), + dag=dag, ) t_move = DockerOperator( @@ -90,8 +83,8 @@ def is_data_available(*args, **kwargs): "/bin/bash", "-c", "/bin/sleep 30; " - "/bin/mv {{params.source_location}}/{{ ti.xcom_pull('view_file') }} {{params.target_location}};" - "/bin/echo '{{params.target_location}}/{{ ti.xcom_pull('view_file') }}';", + "/bin/mv {{ params.source_location }}/" + f"{t_view.output}" + " {{ params.target_location }};" + "/bin/echo '{{ params.target_location }}/" + f"{t_view.output}';", ], task_id="move_data", do_xcom_push=True, @@ -99,20 +92,18 @@ def is_data_available(*args, **kwargs): dag=dag, ) -print_templated_cmd = """ - cat {{ ti.xcom_pull('move_data') }} -""" - t_print = DockerOperator( api_version="1.19", docker_url="tcp://localhost:2375", image="centos:latest", mounts=[Mount(source="/your/host/output_dir/path", target="/your/output_dir/path", type="bind")], - command=print_templated_cmd, + command=f"cat {t_move.output}", task_id="print", dag=dag, ) -t_view.set_downstream(t_is_data_available) t_is_data_available.set_downstream(t_move) t_move.set_downstream(t_print) + +# Task dependencies created via `XComArgs`: +# t_view >> t_is_data_available diff --git a/airflow/providers/docker/example_dags/example_docker_swarm.py b/airflow/providers/docker/example_dags/example_docker_swarm.py index d27337f4d48b3..36db9c3b465b9 100644 --- a/airflow/providers/docker/example_dags/example_docker_swarm.py +++ b/airflow/providers/docker/example_dags/example_docker_swarm.py @@ -21,17 +21,15 @@ from airflow.providers.docker.operators.docker_swarm import DockerSwarmOperator from airflow.utils.dates import days_ago -default_args = { - 'owner': 'airflow', - 'depends_on_past': False, - 'email': ['airflow@example.com'], - 'email_on_failure': False, - 'email_on_retry': False, -} - dag = DAG( 'docker_swarm_sample', - default_args=default_args, + default_args={ + 'owner': 'airflow', + 'depends_on_past': False, + 'email': ['airflow@example.com'], + 'email_on_failure': False, + 'email_on_retry': False, + }, schedule_interval=timedelta(minutes=10), start_date=days_ago(1), catchup=False, diff --git a/airflow/providers/docker/operators/docker.py b/airflow/providers/docker/operators/docker.py index 5387e48d1623b..76062304f9f43 100644 --- a/airflow/providers/docker/operators/docker.py +++ b/airflow/providers/docker/operators/docker.py @@ -299,10 +299,10 @@ def _run_image_with_mounts(self, target_mounts, add_tmp_variable: bool) -> Optio line = '' res_lines = [] for line in lines: - line = line.strip() if hasattr(line, 'decode'): # Note that lines returned can also be byte sequences so we have to handle decode here line = line.decode('utf-8') + line = line.strip() res_lines.append(line) self.log.info(line) @@ -310,11 +310,8 @@ def _run_image_with_mounts(self, target_mounts, add_tmp_variable: bool) -> Optio if result['StatusCode'] != 0: res_lines = "\n".join(res_lines) raise AirflowException('docker container failed: ' + repr(result) + f"lines {res_lines}") - - ret = None if self.do_xcom_push: - ret = self.cli.logs(container=self.container['Id']) if self.xcom_all else line.encode('utf-8') - return ret + return res_lines if self.xcom_all else line finally: if self.auto_remove: self.cli.remove_container(self.container['Id']) diff --git a/airflow/providers/ftp/.latest-doc-only-change.txt b/airflow/providers/ftp/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/ftp/.latest-doc-only-change.txt +++ b/airflow/providers/ftp/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/google/CHANGELOG.rst b/airflow/providers/google/CHANGELOG.rst index 0560bb8ae9bde..062406ea2e5e1 100644 --- a/airflow/providers/google/CHANGELOG.rst +++ b/airflow/providers/google/CHANGELOG.rst @@ -18,13 +18,14 @@ Changelog --------- -4.1.0 +5.0.0 ..... Breaking changes ~~~~~~~~~~~~~~~~ * ``Updated GoogleAdsHook to support newer API versions after google deprecated v5. Google Ads v8 is the new default API. (#17111)`` +* ``Google Ads Hook: Support newer versions of the google-ads library (#17160)`` .. warning:: The underlying google-ads library had breaking changes. @@ -41,16 +42,25 @@ Features ~~~~~~~~ * ``Standardise dataproc location param to region (#16034)`` +* ``Adding custom Salesforce connection type + SalesforceToS3Operator updates (#17162)`` Bug Fixes ~~~~~~~~~ * ``Update alias for field_mask in Google Memmcache (#16975)`` +* ``fix: dataprocpysparkjob project_id as self.project_id (#17075)`` +* ``Fix GCStoGCS operator with replace diabled and existing destination object (#16991)`` .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``Removes pylint from our toolchain (#16682)`` * ``Prepare documentation for July release of providers. (#17015)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` + * ``Fixes several failing tests after broken main (#17222)`` + * ``Fixes statich check failures (#17218)`` + * ``[CASSANDRA-16814] Fix cassandra to gcs type inconsistency. (#17183)`` + * ``Updating Google Cloud example DAGs to use XComArgs (#16875)`` + * ``Updating miscellaneous Google example DAGs to use XComArgs (#16876)`` 4.0.0 ..... diff --git a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py b/airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py index 45207352b7138..ce5e2a00c99aa 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py @@ -67,7 +67,7 @@ task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) - dataset_id = '{{ task_instance.xcom_pull("create_dataset_task", key="dataset_id") }}' + dataset_id = create_dataset_task.output['dataset_id'] import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -80,7 +80,7 @@ create_model = AutoMLTrainModelOperator(task_id="create_model", model=MODEL, location=GCP_AUTOML_LOCATION) - model_id = "{{ task_instance.xcom_pull('create_model', key='model_id') }}" + model_id = create_model.output['model_id'] delete_model_task = AutoMLDeleteModelOperator( task_id="delete_model_task", @@ -96,4 +96,10 @@ project_id=GCP_PROJECT_ID, ) - create_dataset_task >> import_dataset_task >> create_model >> delete_model_task >> delete_datasets_task + import_dataset_task >> create_model + delete_model_task >> delete_datasets_task + + # Task dependencies created via `XComArgs`: + # create_dataset_task >> import_dataset_task + # create_dataset_task >> create_model + # create_dataset_task >> delete_datasets_task diff --git a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_extraction.py b/airflow/providers/google/cloud/example_dags/example_automl_nl_text_extraction.py index 6e28fc7daab78..0cf240bcb850c 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_extraction.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_nl_text_extraction.py @@ -67,7 +67,7 @@ task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) - dataset_id = '{{ task_instance.xcom_pull("create_dataset_task", key="dataset_id") }}' + dataset_id = create_dataset_task.output['dataset_id'] import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -80,7 +80,7 @@ create_model = AutoMLTrainModelOperator(task_id="create_model", model=MODEL, location=GCP_AUTOML_LOCATION) - model_id = "{{ task_instance.xcom_pull('create_model', key='model_id') }}" + model_id = create_model.output['model_id'] delete_model_task = AutoMLDeleteModelOperator( task_id="delete_model_task", @@ -96,4 +96,11 @@ project_id=GCP_PROJECT_ID, ) - create_dataset_task >> import_dataset_task >> create_model >> delete_model_task >> delete_datasets_task + import_dataset_task >> create_model + delete_model_task >> delete_datasets_task + + # Task dependencies created via `XComArgs`: + # create_dataset_task >> import_dataset_task + # create_dataset_task >> create_model + # create_model >> delete_model_task + # create_dataset_task >> delete_datasets_task diff --git a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py b/airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py index 00e49cf54e253..ff8249af1ce10 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py @@ -68,7 +68,7 @@ task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) - dataset_id = '{{ task_instance.xcom_pull("create_dataset_task", key="dataset_id") }}' + dataset_id = create_dataset_task.output['dataset_id'] import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -81,7 +81,7 @@ create_model = AutoMLTrainModelOperator(task_id="create_model", model=MODEL, location=GCP_AUTOML_LOCATION) - model_id = "{{ task_instance.xcom_pull('create_model', key='model_id') }}" + model_id = create_model.output['model_id'] delete_model_task = AutoMLDeleteModelOperator( task_id="delete_model_task", @@ -97,4 +97,11 @@ project_id=GCP_PROJECT_ID, ) - create_dataset_task >> import_dataset_task >> create_model >> delete_model_task >> delete_datasets_task + import_dataset_task >> create_model + delete_model_task >> delete_datasets_task + + # Task dependencies created via `XComArgs`: + # create_dataset_task >> import_dataset_task + # create_dataset_task >> create_model + # create_model >> delete_model_task + # create_dataset_task >> delete_datasets_task diff --git a/airflow/providers/google/cloud/example_dags/example_automl_tables.py b/airflow/providers/google/cloud/example_dags/example_automl_tables.py index 39f479b6a774b..910771541273a 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_tables.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_tables.py @@ -101,7 +101,7 @@ def get_target_column_spec(columns_specs: List[Dict], column_name: str) -> str: project_id=GCP_PROJECT_ID, ) - dataset_id = "{{ task_instance.xcom_pull('create_dataset_task', key='dataset_id') }}" + dataset_id = create_dataset_task.output['dataset_id'] # [END howto_operator_automl_create_dataset] MODEL["dataset_id"] = dataset_id @@ -156,7 +156,7 @@ def get_target_column_spec(columns_specs: List[Dict], column_name: str) -> str: project_id=GCP_PROJECT_ID, ) - model_id = "{{ task_instance.xcom_pull('create_model_task', key='model_id') }}" + model_id = create_model_task.output['model_id'] # [END howto_operator_automl_create_model] # [START howto_operator_automl_delete_model] @@ -176,15 +176,21 @@ def get_target_column_spec(columns_specs: List[Dict], column_name: str) -> str: ) ( - create_dataset_task - >> import_dataset_task + import_dataset_task >> list_tables_spec_task >> list_columns_spec_task >> update_dataset_task >> create_model_task - >> delete_model_task - >> delete_datasets_task ) + delete_model_task >> delete_datasets_task + + # Task dependencies created via `XComArgs`: + # create_dataset_task >> import_dataset_task + # create_dataset_task >> list_tables_spec_task + # create_dataset_task >> list_columns_spec_task + # create_dataset_task >> create_model_task + # create_model_task >> delete_model_task + # create_dataset_task >> delete_datasets_task # Example DAG for AutoML datasets operations @@ -201,7 +207,7 @@ def get_target_column_spec(columns_specs: List[Dict], column_name: str) -> str: project_id=GCP_PROJECT_ID, ) - dataset_id = '{{ task_instance.xcom_pull("create_dataset_task", key="dataset_id") }}' + dataset_id = create_dataset_task.output['dataset_id'] import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -243,14 +249,19 @@ def get_target_column_spec(columns_specs: List[Dict], column_name: str) -> str: # [END howto_operator_delete_dataset] ( - create_dataset_task - >> import_dataset_task + import_dataset_task >> list_tables_spec_task >> list_columns_spec_task >> list_datasets_task >> delete_datasets_task ) + # Task dependencies created via `XComArgs`: + # create_dataset_task >> import_dataset_task + # create_dataset_task >> list_tables_spec_task + # create_dataset_task >> list_columns_spec_task + + with models.DAG( "example_gcp_get_deploy", schedule_interval=None, # Override to match your needs diff --git a/airflow/providers/google/cloud/example_dags/example_automl_translation.py b/airflow/providers/google/cloud/example_dags/example_automl_translation.py index c0216925525fd..4a7a246b642e5 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_translation.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_translation.py @@ -74,7 +74,7 @@ task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) - dataset_id = '{{ task_instance.xcom_pull("create_dataset_task", key="dataset_id") }}' + dataset_id = create_dataset_task.output["dataset_id"] import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -87,7 +87,7 @@ create_model = AutoMLTrainModelOperator(task_id="create_model", model=MODEL, location=GCP_AUTOML_LOCATION) - model_id = "{{ task_instance.xcom_pull('create_model', key='model_id') }}" + model_id = create_model.output["model_id"] delete_model_task = AutoMLDeleteModelOperator( task_id="delete_model_task", @@ -103,4 +103,11 @@ project_id=GCP_PROJECT_ID, ) - create_dataset_task >> import_dataset_task >> create_model >> delete_model_task >> delete_datasets_task + import_dataset_task >> create_model + delete_model_task >> delete_datasets_task + + # Task dependencies created via `XComArgs`: + # create_dataset_task >> import_dataset_task + # create_dataset_task >> create_model + # create_model >> delete_model_task + # create_dataset_task >> delete_datasets_task diff --git a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py b/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py index 140ce0ee9193a..5f03a22c39755 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py @@ -71,7 +71,7 @@ task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) - dataset_id = '{{ task_instance.xcom_pull("create_dataset_task", key="dataset_id") }}' + dataset_id = create_dataset_task.output["dataset_id"] import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -84,7 +84,7 @@ create_model = AutoMLTrainModelOperator(task_id="create_model", model=MODEL, location=GCP_AUTOML_LOCATION) - model_id = "{{ task_instance.xcom_pull('create_model', key='model_id') }}" + model_id = create_model.output["model_id"] delete_model_task = AutoMLDeleteModelOperator( task_id="delete_model_task", @@ -100,4 +100,11 @@ project_id=GCP_PROJECT_ID, ) - create_dataset_task >> import_dataset_task >> create_model >> delete_model_task >> delete_datasets_task + import_dataset_task >> create_model + delete_model_task >> delete_datasets_task + + # Task dependencies created via `XComArgs`: + # create_dataset_task >> import_dataset_task + # create_dataset_task >> create_model + # create_model >> delete_model_task + # create_dataset_task >> delete_datasets_task diff --git a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py b/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py index f1984129950d3..b5347703b5311 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py @@ -72,7 +72,7 @@ task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) - dataset_id = '{{ task_instance.xcom_pull("create_dataset_task", key="dataset_id") }}' + dataset_id = create_dataset_task.output["dataset_id"] import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -85,7 +85,7 @@ create_model = AutoMLTrainModelOperator(task_id="create_model", model=MODEL, location=GCP_AUTOML_LOCATION) - model_id = "{{ task_instance.xcom_pull('create_model', key='model_id') }}" + model_id = create_model.output["model_id"] delete_model_task = AutoMLDeleteModelOperator( task_id="delete_model_task", @@ -101,4 +101,11 @@ project_id=GCP_PROJECT_ID, ) - create_dataset_task >> import_dataset_task >> create_model >> delete_model_task >> delete_datasets_task + import_dataset_task >> create_model + delete_model_task >> delete_datasets_task + + # Task dependencies created via `XComArgs`: + # create_dataset_task >> import_dataset_task + # create_dataset_task >> create_model + # create_model >> delete_model_task + # create_dataset_task >> delete_datasets_task diff --git a/airflow/providers/google/cloud/example_dags/example_automl_vision_classification.py b/airflow/providers/google/cloud/example_dags/example_automl_vision_classification.py index ed96adc754a0c..eb8cf0137d5ca 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_vision_classification.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_vision_classification.py @@ -69,7 +69,7 @@ task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) - dataset_id = '{{ task_instance.xcom_pull("create_dataset_task", key="dataset_id") }}' + dataset_id = create_dataset_task.output["dataset_id"] import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -82,7 +82,7 @@ create_model = AutoMLTrainModelOperator(task_id="create_model", model=MODEL, location=GCP_AUTOML_LOCATION) - model_id = "{{ task_instance.xcom_pull('create_model', key='model_id') }}" + model_id = create_model.output["model_id"] delete_model_task = AutoMLDeleteModelOperator( task_id="delete_model_task", @@ -98,4 +98,11 @@ project_id=GCP_PROJECT_ID, ) - create_dataset_task >> import_dataset_task >> create_model >> delete_model_task >> delete_datasets_task + import_dataset_task >> create_model + delete_model_task >> delete_datasets_task + + # Task dependencies created via `XComArgs`: + # create_dataset_task >> import_dataset_task + # create_dataset_task >> create_model + # create_model >> delete_model_task + # create_dataset_task >> delete_datasets_task diff --git a/airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py b/airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py index f883c375f24f6..76c3720297423 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py +++ b/airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py @@ -71,7 +71,7 @@ task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) - dataset_id = '{{ task_instance.xcom_pull("create_dataset_task", key="dataset_id") }}' + dataset_id = create_dataset_task.output["dataset_id"] import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -84,7 +84,7 @@ create_model = AutoMLTrainModelOperator(task_id="create_model", model=MODEL, location=GCP_AUTOML_LOCATION) - model_id = "{{ task_instance.xcom_pull('create_model', key='model_id') }}" + model_id = create_model.output["model_id"] delete_model_task = AutoMLDeleteModelOperator( task_id="delete_model_task", @@ -100,4 +100,11 @@ project_id=GCP_PROJECT_ID, ) - create_dataset_task >> import_dataset_task >> create_model >> delete_model_task >> delete_datasets_task + import_dataset_task >> create_model + delete_model_task >> delete_datasets_task + + # Task dependencies created via `XComArgs`: + # create_dataset_task >> import_dataset_task + # create_dataset_task >> create_model + # create_model >> delete_model_task + # create_dataset_task >> delete_datasets_task diff --git a/airflow/providers/google/cloud/example_dags/example_azure_fileshare_to_gcs.py b/airflow/providers/google/cloud/example_dags/example_azure_fileshare_to_gcs.py index f0c22c78ef87a..aa49c860a8892 100644 --- a/airflow/providers/google/cloud/example_dags/example_azure_fileshare_to_gcs.py +++ b/airflow/providers/google/cloud/example_dags/example_azure_fileshare_to_gcs.py @@ -24,19 +24,18 @@ AZURE_SHARE_NAME = os.environ.get('AZURE_SHARE_NAME', 'test-azure-share') AZURE_DIRECTORY_NAME = "test-azure-dir" -default_args = { - 'owner': 'airflow', - 'depends_on_past': False, - 'email': ['airflow@example.com'], - 'email_on_failure': False, - 'email_on_retry': False, - 'retries': 1, - 'retry_delay': timedelta(minutes=5), -} with DAG( dag_id='azure_fileshare_to_gcs_example', - default_args=default_args, + default_args={ + 'owner': 'airflow', + 'depends_on_past': False, + 'email': ['airflow@example.com'], + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), + }, schedule_interval=None, start_date=datetime(2018, 11, 1), tags=['example'], diff --git a/airflow/providers/google/cloud/example_dags/example_bigquery_dts.py b/airflow/providers/google/cloud/example_dags/example_bigquery_dts.py index 965f3f918e9b8..c6991af642b1f 100644 --- a/airflow/providers/google/cloud/example_dags/example_bigquery_dts.py +++ b/airflow/providers/google/cloud/example_dags/example_bigquery_dts.py @@ -75,9 +75,7 @@ task_id="gcp_bigquery_create_transfer", ) - transfer_config_id = ( - "{{ task_instance.xcom_pull('gcp_bigquery_create_transfer', key='transfer_config_id') }}" - ) + transfer_config_id = gcp_bigquery_create_transfer.output["transfer_config_id"] # [END howto_bigquery_create_data_transfer] # [START howto_bigquery_start_transfer] @@ -86,14 +84,13 @@ transfer_config_id=transfer_config_id, requested_run_time={"seconds": int(time.time() + 60)}, ) - run_id = "{{ task_instance.xcom_pull('gcp_bigquery_start_transfer', key='run_id') }}" # [END howto_bigquery_start_transfer] # [START howto_bigquery_dts_sensor] gcp_run_sensor = BigQueryDataTransferServiceTransferRunSensor( task_id="gcp_run_sensor", transfer_config_id=transfer_config_id, - run_id=run_id, + run_id=gcp_bigquery_start_transfer.output["run_id"], expected_statuses={"SUCCEEDED"}, ) # [END howto_bigquery_dts_sensor] @@ -104,9 +101,10 @@ ) # [END howto_bigquery_delete_data_transfer] - ( - gcp_bigquery_create_transfer - >> gcp_bigquery_start_transfer - >> gcp_run_sensor - >> gcp_bigquery_delete_transfer - ) + gcp_run_sensor >> gcp_bigquery_delete_transfer + + # Task dependencies created via `XComArgs`: + # gcp_bigquery_create_transfer >> gcp_bigquery_start_transfer + # gcp_bigquery_create_transfer >> gcp_run_sensor + # gcp_bigquery_start_transfer >> gcp_run_sensor + # gcp_bigquery_create_transfer >> gcp_bigquery_delete_transfer diff --git a/airflow/providers/google/cloud/example_dags/example_bigquery_queries.py b/airflow/providers/google/cloud/example_dags/example_bigquery_queries.py index acca7575b4f11..24788c9e05174 100644 --- a/airflow/providers/google/cloud/example_dags/example_bigquery_queries.py +++ b/airflow/providers/google/cloud/example_dags/example_bigquery_queries.py @@ -160,7 +160,7 @@ get_data_result = BashOperator( task_id="get_data_result", - bash_command="echo \"{{ task_instance.xcom_pull('get_data') }}\"", + bash_command=f"echo {get_data.output}", ) # [START howto_operator_bigquery_check] @@ -199,3 +199,5 @@ execute_insert_query >> get_data >> get_data_result >> delete_dataset execute_insert_query >> execute_query_save >> bigquery_execute_multi_query >> delete_dataset execute_insert_query >> [check_count, check_value, check_interval] >> delete_dataset + + globals()[dag_id] = dag_with_locations diff --git a/airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py b/airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py index b7315ed65f269..1b75e34f66ff1 100644 --- a/airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py +++ b/airflow/providers/google/cloud/example_dags/example_cloud_memorystore.py @@ -96,7 +96,7 @@ # [START howto_operator_create_instance_result] create_instance_result = BashOperator( task_id="create-instance-result", - bash_command="echo \"{{ task_instance.xcom_pull('create-instance') }}\"", + bash_command=f"echo {create_instance.output}", ) # [END howto_operator_create_instance_result] @@ -120,7 +120,7 @@ # [START howto_operator_get_instance_result] get_instance_result = BashOperator( - task_id="get-instance-result", bash_command="echo \"{{ task_instance.xcom_pull('get-instance') }}\"" + task_id="get-instance-result", bash_command=f"echo {get_instance.output}" ) # [END howto_operator_get_instance_result] @@ -142,7 +142,7 @@ # [START howto_operator_list_instances_result] list_instances_result = BashOperator( - task_id="list-instances-result", bash_command="echo \"{{ task_instance.xcom_pull('get-instance') }}\"" + task_id="list-instances-result", bash_command=f"echo {get_instance.output}" ) # [END howto_operator_list_instances_result] @@ -236,14 +236,15 @@ create_instance >> get_instance >> get_instance_result create_instance >> update_instance - create_instance >> create_instance_result create_instance >> export_instance create_instance_2 >> import_instance create_instance >> list_instances >> list_instances_result list_instances >> delete_instance export_instance >> update_instance update_instance >> delete_instance + create_instance >> create_instance_result get_instance >> set_acl_permission >> export_instance + get_instance >> list_instances_result export_instance >> import_instance export_instance >> delete_instance failover_instance >> delete_instance_2 @@ -251,6 +252,7 @@ export_instance >> create_instance_and_import >> scale_instance >> export_and_delete_instance + with models.DAG( "gcp_cloud_memorystore_memcached", schedule_interval=None, # Override to match your needs diff --git a/airflow/providers/google/cloud/example_dags/example_cloud_sql.py b/airflow/providers/google/cloud/example_dags/example_cloud_sql.py index d1d9fd45022e3..39a8f2e897bd8 100644 --- a/airflow/providers/google/cloud/example_dags/example_cloud_sql.py +++ b/airflow/providers/google/cloud/example_dags/example_cloud_sql.py @@ -209,9 +209,7 @@ # write access to the destination GCS bucket. # [START howto_operator_cloudsql_export_gcs_permissions] sql_gcp_add_bucket_permission_task = GCSBucketCreateAclEntryOperator( - entity="user-{{ task_instance.xcom_pull(" - "'sql_instance_create_task', key='service_account_email') " - "}}", + entity=f"user-{sql_instance_create_task.output['service_account_email']}", role="WRITER", bucket=export_url_split[1], # netloc (bucket) task_id='sql_gcp_add_bucket_permission_task', @@ -236,9 +234,7 @@ # read access to the target GCS object. # [START howto_operator_cloudsql_import_gcs_permissions] sql_gcp_add_object_permission_task = GCSObjectCreateAclEntryOperator( - entity="user-{{ task_instance.xcom_pull(" - "'sql_instance_create_task2', key='service_account_email')" - " }}", + entity=f"user-{sql_instance_create_2_task.output['service_account_email']}", role="READER", bucket=import_url_split[1], # netloc (bucket) object_name=import_url_split[2][1:], # path (strip first '/') @@ -248,9 +244,7 @@ # For import to work we also need to add the Cloud SQL instance's Service Account # write access to the whole bucket!. sql_gcp_add_bucket_permission_2_task = GCSBucketCreateAclEntryOperator( - entity="user-{{ task_instance.xcom_pull(" - "'sql_instance_create_task2', key='service_account_email') " - "}}", + entity=f"user-{sql_instance_create_2_task.output['service_account_email']}", role="WRITER", bucket=import_url_split[1], # netloc task_id='sql_gcp_add_bucket_permission_2_task', @@ -300,14 +294,10 @@ project_id=GCP_PROJECT_ID, instance=INSTANCE_NAME, task_id='sql_instance_delete_task' ) sql_instance_delete_task2 = CloudSQLDeleteInstanceOperator( - instance=INSTANCE_NAME2, task_id='sql_instance_delete_task2' + project_id=GCP_PROJECT_ID, instance=INSTANCE_NAME2, task_id='sql_instance_delete_task2' ) # [END howto_operator_cloudsql_delete] - sql_instance_delete_2_task = CloudSQLDeleteInstanceOperator( - project_id=GCP_PROJECT_ID, instance=INSTANCE_NAME2, task_id='sql_instance_delete_2_task' - ) - ( sql_instance_create_task >> sql_instance_create_2_task @@ -330,5 +320,10 @@ >> sql_instance_failover_replica_delete_task >> sql_instance_read_replica_delete_task >> sql_instance_delete_task - >> sql_instance_delete_2_task + >> sql_instance_delete_task2 ) + + # Task dependencies created via `XComArgs`: + # sql_instance_create_task >> sql_gcp_add_bucket_permission_task + # sql_instance_create_2_task >> sql_gcp_add_object_permission_task + # sql_instance_create_2_task >> sql_gcp_add_bucket_permission_2_task diff --git a/airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py b/airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py index c1bc8c0b9cd93..d4562545cb691 100644 --- a/airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py +++ b/airflow/providers/google/cloud/example_dags/example_cloud_storage_transfer_service_aws.py @@ -106,10 +106,6 @@ # [END howto_operator_gcp_transfer_create_job_body_aws] -# [START howto_operator_gcp_transfer_default_args] -default_args = {'owner': 'airflow'} -# [END howto_operator_gcp_transfer_default_args] - with models.DAG( 'example_gcp_transfer_aws', schedule_interval=None, # Override to match your needs diff --git a/airflow/providers/google/cloud/example_dags/example_datacatalog.py b/airflow/providers/google/cloud/example_dags/example_datacatalog.py index e6e94f2913925..a764c11a76e72 100644 --- a/airflow/providers/google/cloud/example_dags/example_datacatalog.py +++ b/airflow/providers/google/cloud/example_dags/example_datacatalog.py @@ -72,14 +72,14 @@ # [START howto_operator_gcp_datacatalog_create_entry_group_result] create_entry_group_result = BashOperator( task_id="create_entry_group_result", - bash_command="echo \"{{ task_instance.xcom_pull('create_entry_group', key='entry_group_id') }}\"", + bash_command=f"echo {create_entry_group.output['entry_group_id']}", ) # [END howto_operator_gcp_datacatalog_create_entry_group_result] # [START howto_operator_gcp_datacatalog_create_entry_group_result2] create_entry_group_result2 = BashOperator( task_id="create_entry_group_result2", - bash_command="echo \"{{ task_instance.xcom_pull('create_entry_group') }}\"", + bash_command=f"echo {create_entry_group.output}", ) # [END howto_operator_gcp_datacatalog_create_entry_group_result2] @@ -100,14 +100,14 @@ # [START howto_operator_gcp_datacatalog_create_entry_gcs_result] create_entry_gcs_result = BashOperator( task_id="create_entry_gcs_result", - bash_command="echo \"{{ task_instance.xcom_pull('create_entry_gcs', key='entry_id') }}\"", + bash_command=f"echo {create_entry_gcs.output['entry_id']}", ) # [END howto_operator_gcp_datacatalog_create_entry_gcs_result] # [START howto_operator_gcp_datacatalog_create_entry_gcs_result2] create_entry_gcs_result2 = BashOperator( task_id="create_entry_gcs_result2", - bash_command="echo \"{{ task_instance.xcom_pull('create_entry_gcs') }}\"", + bash_command=f"echo {create_entry_gcs.output}", ) # [END howto_operator_gcp_datacatalog_create_entry_gcs_result2] @@ -125,14 +125,12 @@ # [START howto_operator_gcp_datacatalog_create_tag_result] create_tag_result = BashOperator( task_id="create_tag_result", - bash_command="echo \"{{ task_instance.xcom_pull('create_tag', key='tag_id') }}\"", + bash_command=f"echo {create_tag.output['tag_id']}", ) # [END howto_operator_gcp_datacatalog_create_tag_result] # [START howto_operator_gcp_datacatalog_create_tag_result2] - create_tag_result2 = BashOperator( - task_id="create_tag_result2", bash_command="echo \"{{ task_instance.xcom_pull('create_tag') }}\"" - ) + create_tag_result2 = BashOperator(task_id="create_tag_result2", bash_command=f"echo {create_tag.output}") # [END howto_operator_gcp_datacatalog_create_tag_result2] # [START howto_operator_gcp_datacatalog_create_tag_template] @@ -154,14 +152,14 @@ # [START howto_operator_gcp_datacatalog_create_tag_template_result] create_tag_template_result = BashOperator( task_id="create_tag_template_result", - bash_command="echo \"{{ task_instance.xcom_pull('create_tag_template', key='tag_template_id') }}\"", + bash_command=f"echo {create_tag_template.output['tag_template_id']}", ) # [END howto_operator_gcp_datacatalog_create_tag_template_result] # [START howto_operator_gcp_datacatalog_create_tag_template_result2] create_tag_template_result2 = BashOperator( task_id="create_tag_template_result2", - bash_command="echo \"{{ task_instance.xcom_pull('create_tag_template') }}\"", + bash_command=f"echo {create_tag_template.output}", ) # [END howto_operator_gcp_datacatalog_create_tag_template_result2] @@ -180,17 +178,14 @@ # [START howto_operator_gcp_datacatalog_create_tag_template_field_result] create_tag_template_field_result = BashOperator( task_id="create_tag_template_field_result", - bash_command=( - "echo \"{{ task_instance.xcom_pull('create_tag_template_field'," - + " key='tag_template_field_id') }}\"" - ), + bash_command=f"echo {create_tag_template_field.output['tag_template_field_id']}", ) # [END howto_operator_gcp_datacatalog_create_tag_template_field_result] # [START howto_operator_gcp_datacatalog_create_tag_template_field_result2] create_tag_template_field_result2 = BashOperator( task_id="create_tag_template_field_result2", - bash_command="echo \"{{ task_instance.xcom_pull('create_tag_template_field') }}\"", + bash_command=f"echo {create_tag_template_field.output}", ) # [END howto_operator_gcp_datacatalog_create_tag_template_field_result2] @@ -213,7 +208,7 @@ location=LOCATION, entry_group=ENTRY_GROUP_ID, entry=ENTRY_ID, - tag="{{ task_instance.xcom_pull('create_tag', key='tag_id') }}", + tag=create_tag.output["tag_id"], ) # [END howto_operator_gcp_datacatalog_delete_tag] @@ -246,7 +241,7 @@ # [START howto_operator_gcp_datacatalog_get_entry_group_result] get_entry_group_result = BashOperator( task_id="get_entry_group_result", - bash_command="echo \"{{ task_instance.xcom_pull('get_entry_group') }}\"", + bash_command=f"echo {get_entry_group.output}", ) # [END howto_operator_gcp_datacatalog_get_entry_group_result] @@ -257,9 +252,7 @@ # [END howto_operator_gcp_datacatalog_get_entry] # [START howto_operator_gcp_datacatalog_get_entry_result] - get_entry_result = BashOperator( - task_id="get_entry_result", bash_command="echo \"{{ task_instance.xcom_pull('get_entry') }}\"" - ) + get_entry_result = BashOperator(task_id="get_entry_result", bash_command=f"echo {get_entry.output}") # [END howto_operator_gcp_datacatalog_get_entry_result] # [START howto_operator_gcp_datacatalog_get_tag_template] @@ -271,7 +264,7 @@ # [START howto_operator_gcp_datacatalog_get_tag_template_result] get_tag_template_result = BashOperator( task_id="get_tag_template_result", - bash_command="echo \"{{ task_instance.xcom_pull('get_tag_template') }}\"", + bash_command=f"{get_tag_template.output}", ) # [END howto_operator_gcp_datacatalog_get_tag_template_result] @@ -283,9 +276,7 @@ # [END howto_operator_gcp_datacatalog_list_tags] # [START howto_operator_gcp_datacatalog_list_tags_result] - list_tags_result = BashOperator( - task_id="list_tags_result", bash_command="echo \"{{ task_instance.xcom_pull('list_tags') }}\"" - ) + list_tags_result = BashOperator(task_id="list_tags_result", bash_command=f"echo {list_tags.output}") # [END howto_operator_gcp_datacatalog_list_tags_result] # Lookup @@ -330,7 +321,7 @@ # [START howto_operator_gcp_datacatalog_search_catalog_result] search_catalog_result = BashOperator( task_id="search_catalog_result", - bash_command="echo \"{{ task_instance.xcom_pull('search_catalog') }}\"", + bash_command=f"echo {search_catalog.output}", ) # [END howto_operator_gcp_datacatalog_search_catalog_result] @@ -354,7 +345,7 @@ location=LOCATION, entry_group=ENTRY_GROUP_ID, entry=ENTRY_ID, - tag_id="{{ task_instance.xcom_pull('create_tag', key='tag_id') }}", + tag_id=f"{create_tag.output['tag_id']}", ) # [END howto_operator_gcp_datacatalog_update_tag] diff --git a/airflow/providers/google/cloud/example_dags/example_dataproc.py b/airflow/providers/google/cloud/example_dags/example_dataproc.py index faebd94214c0e..914df0ef149fa 100644 --- a/airflow/providers/google/cloud/example_dags/example_dataproc.py +++ b/airflow/providers/google/cloud/example_dags/example_dataproc.py @@ -170,7 +170,7 @@ update_mask=UPDATE_MASK, graceful_decommission_timeout=TIMEOUT, project_id=PROJECT_ID, - region=REGION, + location=REGION, ) # [END how_to_cloud_dataproc_update_cluster_operator] @@ -179,7 +179,7 @@ task_id="create_workflow_template", template=WORKFLOW_TEMPLATE, project_id=PROJECT_ID, - region=REGION, + location=REGION, ) # [END how_to_cloud_dataproc_create_workflow_template] @@ -190,46 +190,46 @@ # [END how_to_cloud_dataproc_trigger_workflow_template] pig_task = DataprocSubmitJobOperator( - task_id="pig_task", job=PIG_JOB, region=REGION, project_id=PROJECT_ID + task_id="pig_task", job=PIG_JOB, location=REGION, project_id=PROJECT_ID ) spark_sql_task = DataprocSubmitJobOperator( - task_id="spark_sql_task", job=SPARK_SQL_JOB, region=REGION, project_id=PROJECT_ID + task_id="spark_sql_task", job=SPARK_SQL_JOB, location=REGION, project_id=PROJECT_ID ) spark_task = DataprocSubmitJobOperator( - task_id="spark_task", job=SPARK_JOB, region=REGION, project_id=PROJECT_ID + task_id="spark_task", job=SPARK_JOB, location=REGION, project_id=PROJECT_ID ) # [START cloud_dataproc_async_submit_sensor] spark_task_async = DataprocSubmitJobOperator( - task_id="spark_task_async", job=SPARK_JOB, region=REGION, project_id=PROJECT_ID, asynchronous=True + task_id="spark_task_async", job=SPARK_JOB, location=REGION, project_id=PROJECT_ID, asynchronous=True ) spark_task_async_sensor = DataprocJobSensor( task_id='spark_task_async_sensor_task', - region=REGION, + location=REGION, project_id=PROJECT_ID, - dataproc_job_id="{{task_instance.xcom_pull(task_ids='spark_task_async')}}", + dataproc_job_id=spark_task_async.output, poke_interval=10, ) # [END cloud_dataproc_async_submit_sensor] # [START how_to_cloud_dataproc_submit_job_to_cluster_operator] pyspark_task = DataprocSubmitJobOperator( - task_id="pyspark_task", job=PYSPARK_JOB, region=REGION, project_id=PROJECT_ID + task_id="pyspark_task", job=PYSPARK_JOB, location=REGION, project_id=PROJECT_ID ) # [END how_to_cloud_dataproc_submit_job_to_cluster_operator] sparkr_task = DataprocSubmitJobOperator( - task_id="sparkr_task", job=SPARKR_JOB, region=REGION, project_id=PROJECT_ID + task_id="sparkr_task", job=SPARKR_JOB, location=REGION, project_id=PROJECT_ID ) hive_task = DataprocSubmitJobOperator( - task_id="hive_task", job=HIVE_JOB, region=REGION, project_id=PROJECT_ID + task_id="hive_task", job=HIVE_JOB, location=REGION, project_id=PROJECT_ID ) hadoop_task = DataprocSubmitJobOperator( - task_id="hadoop_task", job=HADOOP_JOB, region=REGION, project_id=PROJECT_ID + task_id="hadoop_task", job=HADOOP_JOB, location=REGION, project_id=PROJECT_ID ) # [START how_to_cloud_dataproc_delete_cluster_operator] @@ -244,7 +244,11 @@ scale_cluster >> pig_task >> delete_cluster scale_cluster >> spark_sql_task >> delete_cluster scale_cluster >> spark_task >> delete_cluster - scale_cluster >> spark_task_async >> spark_task_async_sensor >> delete_cluster + scale_cluster >> spark_task_async + spark_task_async_sensor >> delete_cluster scale_cluster >> pyspark_task >> delete_cluster scale_cluster >> sparkr_task >> delete_cluster scale_cluster >> hadoop_task >> delete_cluster + + # Task dependency created via `XComArgs`: + # spark_task_async >> spark_task_async_sensor diff --git a/airflow/providers/google/cloud/example_dags/example_datastore.py b/airflow/providers/google/cloud/example_dags/example_datastore.py index 8badca072c774..3ef605d7365a2 100644 --- a/airflow/providers/google/cloud/example_dags/example_datastore.py +++ b/airflow/providers/google/cloud/example_dags/example_datastore.py @@ -79,28 +79,6 @@ TRANSACTION_OPTIONS: Dict[str, Any] = {"readWrite": {}} # [END how_to_transaction_def] -# [START how_to_commit_def] -COMMIT_BODY = { - "mode": "TRANSACTIONAL", - "mutations": [ - { - "insert": { - "key": KEYS[0], - "properties": {"string": {"stringValue": "airflow is awesome!"}}, - } - } - ], - "transaction": "{{ task_instance.xcom_pull('begin_transaction_commit') }}", -} -# [END how_to_commit_def] - -# [START how_to_query_def] -QUERY = { - "partitionId": {"projectId": GCP_PROJECT_ID, "namespaceId": ""}, - "readOptions": {"transaction": "{{ task_instance.xcom_pull('begin_transaction_query') }}"}, - "query": {}, -} -# [END how_to_query_def] with models.DAG( "example_gcp_datastore_operations", @@ -122,13 +100,28 @@ ) # [END how_to_begin_transaction] + # [START how_to_commit_def] + COMMIT_BODY = { + "mode": "TRANSACTIONAL", + "mutations": [ + { + "insert": { + "key": KEYS[0], + "properties": {"string": {"stringValue": "airflow is awesome!"}}, + } + } + ], + "transaction": begin_transaction_commit.output, + } + # [END how_to_commit_def] + # [START how_to_commit_task] commit_task = CloudDatastoreCommitOperator( task_id="commit_task", body=COMMIT_BODY, project_id=GCP_PROJECT_ID ) # [END how_to_commit_task] - allocate_ids >> begin_transaction_commit >> commit_task + allocate_ids >> begin_transaction_commit begin_transaction_query = CloudDatastoreBeginTransactionOperator( task_id="begin_transaction_query", @@ -136,11 +129,19 @@ project_id=GCP_PROJECT_ID, ) + # [START how_to_query_def] + QUERY = { + "partitionId": {"projectId": GCP_PROJECT_ID, "namespaceId": ""}, + "readOptions": {"transaction": begin_transaction_query.output}, + "query": {}, + } + # [END how_to_query_def] + # [START how_to_run_query] run_query = CloudDatastoreRunQueryOperator(task_id="run_query", body=QUERY, project_id=GCP_PROJECT_ID) # [END how_to_run_query] - allocate_ids >> begin_transaction_query >> run_query + allocate_ids >> begin_transaction_query begin_transaction_to_rollback = CloudDatastoreBeginTransactionOperator( task_id="begin_transaction_to_rollback", @@ -151,7 +152,11 @@ # [START how_to_rollback_transaction] rollback_transaction = CloudDatastoreRollbackOperator( task_id="rollback_transaction", - transaction="{{ task_instance.xcom_pull('begin_transaction_to_rollback') }}", + transaction=begin_transaction_to_rollback.output, ) - begin_transaction_to_rollback >> rollback_transaction # [END how_to_rollback_transaction] + + # Task dependencies created via `XComArgs`: + # begin_transaction_commit >> commit_task + # begin_transaction_to_rollback >> rollback_transaction + # begin_transaction_query >> run_query diff --git a/airflow/providers/google/cloud/example_dags/example_functions.py b/airflow/providers/google/cloud/example_dags/example_functions.py index c98a93fd830b2..3730d375f90e1 100644 --- a/airflow/providers/google/cloud/example_dags/example_functions.py +++ b/airflow/providers/google/cloud/example_dags/example_functions.py @@ -95,6 +95,7 @@ with models.DAG( 'example_gcp_function', + default_args=default_args, schedule_interval=None, # Override to match your needs start_date=dates.days_ago(1), tags=['example'], diff --git a/airflow/providers/google/cloud/example_dags/example_gcs.py b/airflow/providers/google/cloud/example_dags/example_gcs.py index 40a9c47ef6735..94bdfbcab6082 100644 --- a/airflow/providers/google/cloud/example_dags/example_gcs.py +++ b/airflow/providers/google/cloud/example_dags/example_gcs.py @@ -75,7 +75,7 @@ list_buckets_result = BashOperator( task_id="list_buckets_result", - bash_command="echo \"{{ task_instance.xcom_pull('list_buckets') }}\"", + bash_command=f"echo {list_buckets.output}", ) upload_file = LocalFilesystemToGCSOperator( @@ -156,6 +156,7 @@ copy_file >> delete_bucket_2 delete_files >> delete_bucket_1 + with models.DAG( "example_gcs_sensors", start_date=days_ago(1), diff --git a/airflow/providers/google/cloud/example_dags/example_mlengine.py b/airflow/providers/google/cloud/example_dags/example_mlengine.py index d82d166b5feeb..56214e924ec72 100644 --- a/airflow/providers/google/cloud/example_dags/example_mlengine.py +++ b/airflow/providers/google/cloud/example_dags/example_mlengine.py @@ -56,13 +56,13 @@ SUMMARY_TMP = os.environ.get("GCP_MLENGINE_DATAFLOW_TMP", "gs://INVALID BUCKET NAME/tmp/") SUMMARY_STAGING = os.environ.get("GCP_MLENGINE_DATAFLOW_STAGING", "gs://INVALID BUCKET NAME/staging/") -default_args = {"params": {"model_name": MODEL_NAME}} with models.DAG( "example_gcp_mlengine", schedule_interval=None, # Override to match your needs start_date=days_ago(1), tags=['example'], + params={"model_name": MODEL_NAME}, ) as dag: # [START howto_operator_gcp_mlengine_training] training = MLEngineStartTrainingJobOperator( @@ -100,7 +100,7 @@ # [START howto_operator_gcp_mlengine_print_model] get_model_result = BashOperator( - bash_command="echo \"{{ task_instance.xcom_pull('get-model') }}\"", + bash_command=f"echo {get_model.output}", task_id="get-model-result", ) # [END howto_operator_gcp_mlengine_print_model] @@ -158,7 +158,7 @@ # [START howto_operator_gcp_mlengine_print_versions] list_version_result = BashOperator( - bash_command="echo \"{{ task_instance.xcom_pull('list-version') }}\"", + bash_command=f"echo {list_version.output}", task_id="list-version-result", ) # [END howto_operator_gcp_mlengine_print_versions] @@ -192,6 +192,7 @@ training >> create_version training >> create_version_2 create_model >> get_model >> [get_model_result, delete_model] + create_model >> get_model >> delete_model create_model >> create_version >> create_version_2 >> set_defaults_version >> list_version create_version >> prediction create_version_2 >> prediction diff --git a/airflow/providers/google/cloud/example_dags/example_natural_language.py b/airflow/providers/google/cloud/example_dags/example_natural_language.py index 9038052b9ae69..558e929997548 100644 --- a/airflow/providers/google/cloud/example_dags/example_natural_language.py +++ b/airflow/providers/google/cloud/example_dags/example_natural_language.py @@ -63,7 +63,7 @@ # [START howto_operator_gcp_natural_language_analyze_entities_result] analyze_entities_result = BashOperator( - bash_command="echo \"{{ task_instance.xcom_pull('analyze_entities') }}\"", + bash_command=f"echo {analyze_entities.output}", task_id="analyze_entities_result", ) # [END howto_operator_gcp_natural_language_analyze_entities_result] @@ -76,7 +76,7 @@ # [START howto_operator_gcp_natural_language_analyze_entity_sentiment_result] analyze_entity_sentiment_result = BashOperator( - bash_command="echo \"{{ task_instance.xcom_pull('analyze_entity_sentiment') }}\"", + bash_command=f"echo {analyze_entity_sentiment.output}", task_id="analyze_entity_sentiment_result", ) # [END howto_operator_gcp_natural_language_analyze_entity_sentiment_result] @@ -89,7 +89,7 @@ # [START howto_operator_gcp_natural_language_analyze_sentiment_result] analyze_sentiment_result = BashOperator( - bash_command="echo \"{{ task_instance.xcom_pull('analyze_sentiment') }}\"", + bash_command=f"echo {analyze_sentiment.output}", task_id="analyze_sentiment_result", ) # [END howto_operator_gcp_natural_language_analyze_sentiment_result] @@ -102,7 +102,7 @@ # [START howto_operator_gcp_natural_language_analyze_classify_text_result] analyze_classify_text_result = BashOperator( - bash_command="echo \"{{ task_instance.xcom_pull('analyze_classify_text') }}\"", + bash_command=f"echo {analyze_classify_text.output}", task_id="analyze_classify_text_result", ) # [END howto_operator_gcp_natural_language_analyze_classify_text_result] diff --git a/airflow/providers/google/cloud/example_dags/example_pubsub.py b/airflow/providers/google/cloud/example_dags/example_pubsub.py index 2395d7d00aa39..0dbf4f41bf322 100644 --- a/airflow/providers/google/cloud/example_dags/example_pubsub.py +++ b/airflow/providers/google/cloud/example_dags/example_pubsub.py @@ -65,7 +65,7 @@ # [END howto_operator_gcp_pubsub_create_subscription] # [START howto_operator_gcp_pubsub_pull_message_with_sensor] - subscription = "{{ task_instance.xcom_pull('subscribe_task') }}" + subscription = subscribe_task.output pull_messages = PubSubPullSensor( task_id="pull_messages", @@ -92,7 +92,7 @@ unsubscribe_task = PubSubDeleteSubscriptionOperator( task_id="unsubscribe_task", project_id=GCP_PROJECT_ID, - subscription="{{ task_instance.xcom_pull('subscribe_task') }}", + subscription=subscription, ) # [END howto_operator_gcp_pubsub_unsubscribe] @@ -102,9 +102,13 @@ ) # [END howto_operator_gcp_pubsub_delete_topic] - create_topic >> subscribe_task >> [publish_task, pull_messages] + create_topic >> subscribe_task >> publish_task pull_messages >> pull_messages_result >> unsubscribe_task >> delete_topic + # Task dependencies created via `XComArgs`: + # subscribe_task >> pull_messages + # subscribe_task >> unsubscribe_task + with models.DAG( "example_gcp_pubsub_operator", @@ -124,7 +128,7 @@ # [END howto_operator_gcp_pubsub_create_subscription] # [START howto_operator_gcp_pubsub_pull_message_with_operator] - subscription = "{{ task_instance.xcom_pull('subscribe_task') }}" + subscription = subscribe_task.output pull_messages_operator = PubSubPullOperator( task_id="pull_messages", @@ -151,7 +155,7 @@ unsubscribe_task = PubSubDeleteSubscriptionOperator( task_id="unsubscribe_task", project_id=GCP_PROJECT_ID, - subscription="{{ task_instance.xcom_pull('subscribe_task') }}", + subscription=subscription, ) # [END howto_operator_gcp_pubsub_unsubscribe] @@ -170,3 +174,7 @@ >> unsubscribe_task >> delete_topic ) + + # Task dependencies created via `XComArgs`: + # subscribe_task >> pull_messages_operator + # subscribe_task >> unsubscribe_task diff --git a/airflow/providers/google/cloud/example_dags/example_tasks.py b/airflow/providers/google/cloud/example_dags/example_tasks.py index dc2759fd2c2dc..49e21de0d1a6b 100644 --- a/airflow/providers/google/cloud/example_dags/example_tasks.py +++ b/airflow/providers/google/cloud/example_dags/example_tasks.py @@ -114,8 +114,9 @@ get_queue_result = BashOperator( task_id="get_queue_result", - bash_command="echo \"{{ task_instance.xcom_pull('get_queue') }}\"", + bash_command=f"echo {get_queue.output}", ) + get_queue >> get_queue_result update_queue = CloudTasksQueueUpdateOperator( diff --git a/airflow/providers/google/cloud/example_dags/example_vision.py b/airflow/providers/google/cloud/example_dags/example_vision.py index 9e38a314dc5c5..5f352ef65b5ca 100644 --- a/airflow/providers/google/cloud/example_dags/example_vision.py +++ b/airflow/providers/google/cloud/example_dags/example_vision.py @@ -130,10 +130,12 @@ ) # [END howto_operator_vision_product_set_create] + product_set_create_output = product_set_create.output + # [START howto_operator_vision_product_set_get] product_set_get = CloudVisionGetProductSetOperator( location=GCP_VISION_LOCATION, - product_set_id="{{ task_instance.xcom_pull('product_set_create') }}", + product_set_id=product_set_create_output, task_id='product_set_get', ) # [END howto_operator_vision_product_set_get] @@ -141,7 +143,7 @@ # [START howto_operator_vision_product_set_update] product_set_update = CloudVisionUpdateProductSetOperator( location=GCP_VISION_LOCATION, - product_set_id="{{ task_instance.xcom_pull('product_set_create') }}", + product_set_id=product_set_create_output, product_set=ProductSet(display_name='My Product Set 2'), task_id='product_set_update', ) @@ -150,7 +152,7 @@ # [START howto_operator_vision_product_set_delete] product_set_delete = CloudVisionDeleteProductSetOperator( location=GCP_VISION_LOCATION, - product_set_id="{{ task_instance.xcom_pull('product_set_create') }}", + product_set_id=product_set_create_output, task_id='product_set_delete', ) # [END howto_operator_vision_product_set_delete] @@ -165,10 +167,12 @@ ) # [END howto_operator_vision_product_create] + product_create_output = product_create.output + # [START howto_operator_vision_product_get] product_get = CloudVisionGetProductOperator( location=GCP_VISION_LOCATION, - product_id="{{ task_instance.xcom_pull('product_create') }}", + product_id=product_create_output, task_id='product_get', ) # [END howto_operator_vision_product_get] @@ -176,7 +180,7 @@ # [START howto_operator_vision_product_update] product_update = CloudVisionUpdateProductOperator( location=GCP_VISION_LOCATION, - product_id="{{ task_instance.xcom_pull('product_create') }}", + product_id=product_create_output, product=Product(display_name='My Product 2', description='My updated description'), task_id='product_update', ) @@ -185,7 +189,7 @@ # [START howto_operator_vision_product_delete] product_delete = CloudVisionDeleteProductOperator( location=GCP_VISION_LOCATION, - product_id="{{ task_instance.xcom_pull('product_create') }}", + product_id=product_create_output, task_id='product_delete', ) # [END howto_operator_vision_product_delete] @@ -194,7 +198,7 @@ reference_image_create = CloudVisionCreateReferenceImageOperator( location=GCP_VISION_LOCATION, reference_image=reference_image, - product_id="{{ task_instance.xcom_pull('product_create') }}", + product_id=product_create_output, reference_image_id=GCP_VISION_REFERENCE_IMAGE_ID, retry=Retry(maximum=10.0), timeout=5, @@ -205,7 +209,7 @@ # [START howto_operator_vision_reference_image_delete] reference_image_delete = CloudVisionDeleteReferenceImageOperator( location=GCP_VISION_LOCATION, - product_id="{{ task_instance.xcom_pull('product_create') }}", + product_id=product_create_output, reference_image_id=GCP_VISION_REFERENCE_IMAGE_ID, retry=Retry(maximum=10.0), timeout=5, @@ -216,8 +220,8 @@ # [START howto_operator_vision_add_product_to_product_set] add_product_to_product_set = CloudVisionAddProductToProductSetOperator( location=GCP_VISION_LOCATION, - product_set_id="{{ task_instance.xcom_pull('product_set_create') }}", - product_id="{{ task_instance.xcom_pull('product_create') }}", + product_set_id=product_set_create_output, + product_id=product_create_output, retry=Retry(maximum=10.0), timeout=5, task_id='add_product_to_product_set', @@ -227,8 +231,8 @@ # [START howto_operator_vision_remove_product_from_product_set] remove_product_from_product_set = CloudVisionRemoveProductFromProductSetOperator( location=GCP_VISION_LOCATION, - product_set_id="{{ task_instance.xcom_pull('product_set_create') }}", - product_id="{{ task_instance.xcom_pull('product_create') }}", + product_set_id=product_set_create_output, + product_id=product_create_output, retry=Retry(maximum=10.0), timeout=5, task_id='remove_product_from_product_set', @@ -239,18 +243,31 @@ product_create >> product_get >> product_update >> product_delete # ProductSet path - product_set_create >> product_set_get >> product_set_update >> product_set_delete + product_set_get >> product_set_update >> product_set_delete # ReferenceImage path - product_create >> reference_image_create >> reference_image_delete >> product_delete + reference_image_create >> reference_image_delete >> product_delete # Product/ProductSet path product_create >> add_product_to_product_set - product_set_create >> add_product_to_product_set add_product_to_product_set >> remove_product_from_product_set remove_product_from_product_set >> product_delete remove_product_from_product_set >> product_set_delete + # Task dependencies created via `XComArgs`: + # product_set_create >> product_set_get + # product_set_create >> product_set_update + # product_set_create >> product_set_delete + # product_create >> product_get + # product_create >> product_delete + # product_create >> reference_image_create + # product_create >> reference_image_delete + # product_set_create >> add_product_to_product_set + # product_create >> add_product_to_product_set + # product_set_create >> remove_product_from_product_set + # product_create >> remove_product_from_product_set + + with models.DAG( 'example_gcp_vision_explicit_id', start_date=days_ago(1), schedule_interval=None ) as dag_explicit_id: @@ -489,7 +506,7 @@ # [START howto_operator_vision_detect_safe_search_result] detect_safe_search_result = BashOperator( - bash_command="echo {{ task_instance.xcom_pull('detect_safe_search') }}", + bash_command=f"echo {detect_safe_search.output}", task_id="detect_safe_search_result", ) # [END howto_operator_vision_detect_safe_search_result] diff --git a/airflow/providers/google/cloud/example_dags/example_workflows.py b/airflow/providers/google/cloud/example_dags/example_workflows.py index 0fab435959201..35b5b015a60e4 100644 --- a/airflow/providers/google/cloud/example_dags/example_workflows.py +++ b/airflow/providers/google/cloud/example_dags/example_workflows.py @@ -131,13 +131,15 @@ ) # [END how_to_create_execution] + create_execution_id = create_execution.output["execution_id"] + # [START how_to_wait_for_execution] wait_for_execution = WorkflowExecutionSensor( task_id="wait_for_execution", location=LOCATION, project_id=PROJECT_ID, workflow_id=WORKFLOW_ID, - execution_id='{{ task_instance.xcom_pull("create_execution", key="execution_id") }}', + execution_id=create_execution_id, ) # [END how_to_wait_for_execution] @@ -147,7 +149,7 @@ location=LOCATION, project_id=PROJECT_ID, workflow_id=WORKFLOW_ID, - execution_id='{{ task_instance.xcom_pull("create_execution", key="execution_id") }}', + execution_id=create_execution_id, ) # [END how_to_get_execution] @@ -179,18 +181,23 @@ location=LOCATION, project_id=PROJECT_ID, workflow_id=SLEEP_WORKFLOW_ID, - execution_id='{{ task_instance.xcom_pull("create_execution_for_cancel", key="execution_id") }}', + execution_id=create_execution_id, ) # [END how_to_cancel_execution] create_workflow >> update_workflows >> [get_workflow, list_workflows] update_workflows >> [create_execution, create_execution_for_cancel] - create_execution >> wait_for_execution >> [get_execution, list_executions] + wait_for_execution >> [get_execution, list_executions] create_workflow_for_cancel >> create_execution_for_cancel >> cancel_execution [cancel_execution, list_executions] >> delete_workflow + # Task dependencies created via `XComArgs`: + # create_execution >> wait_for_execution + # create_execution >> get_execution + # create_execution >> cancel_execution + if __name__ == '__main__': dag.clear(dag_run_state=None) diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index 66980aee35cd2..4a0f6c007adc1 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -2196,7 +2196,7 @@ class BigQueryInsertJobOperator(BaseOperator): "impersonation_chain", ) template_ext = (".json",) - template_fields_renderers = {"configuration": "json"} + template_fields_renderers = {"configuration": "json", "configuration.query.query": "sql"} ui_color = BigQueryUIColors.QUERY.value def __init__( diff --git a/airflow/providers/google/leveldb/example_dags/example_leveldb.py b/airflow/providers/google/leveldb/example_dags/example_leveldb.py index 665791c651d7d..e4a732c71d6bf 100644 --- a/airflow/providers/google/leveldb/example_dags/example_leveldb.py +++ b/airflow/providers/google/leveldb/example_dags/example_leveldb.py @@ -23,13 +23,8 @@ from airflow.providers.google.leveldb.operators.leveldb import LevelDBOperator from airflow.utils.dates import days_ago -default_args = { - 'owner': 'airflow', -} - with models.DAG( 'example_leveldb', - default_args=default_args, start_date=days_ago(2), schedule_interval=None, tags=['example'], diff --git a/airflow/providers/google/marketing_platform/example_dags/example_display_video.py b/airflow/providers/google/marketing_platform/example_dags/example_display_video.py index cab062595682b..5c74c519ae1ba 100644 --- a/airflow/providers/google/marketing_platform/example_dags/example_display_video.py +++ b/airflow/providers/google/marketing_platform/example_dags/example_display_video.py @@ -89,7 +89,7 @@ ) as dag1: # [START howto_google_display_video_createquery_report_operator] create_report = GoogleDisplayVideo360CreateReportOperator(body=REPORT, task_id="create_report") - report_id = "{{ task_instance.xcom_pull('create_report', key='report_id') }}" + report_id = create_report.output["report_id"] # [END howto_google_display_video_createquery_report_operator] # [START howto_google_display_video_runquery_report_operator] @@ -115,7 +115,14 @@ delete_report = GoogleDisplayVideo360DeleteReportOperator(report_id=report_id, task_id="delete_report") # [END howto_google_display_video_deletequery_report_operator] - create_report >> run_report >> wait_for_report >> get_report >> delete_report + run_report >> wait_for_report >> get_report >> delete_report + + # Task dependencies created via `XComArgs`: + # create_report >> run_report + # create_report >> wait_for_report + # create_report >> get_report + # create_report >> delete_report + with models.DAG( "example_display_video_misc", @@ -183,7 +190,7 @@ upload_sdf_to_big_query = GCSToBigQueryOperator( task_id="upload_sdf_to_big_query", bucket=BUCKET, - source_objects=['{{ task_instance.xcom_pull("upload_sdf_to_bigquery")}}'], + source_objects=[save_sdf_in_gcs.output], destination_project_dataset_table=f"{BQ_DATA_SET}.gcs_to_bq_table", schema_fields=[ {"name": "name", "type": "STRING", "mode": "NULLABLE"}, @@ -193,4 +200,7 @@ ) # [END howto_google_display_video_gcs_to_big_query_operator] - create_sdf_download_task >> wait_for_operation >> save_sdf_in_gcs >> upload_sdf_to_big_query + create_sdf_download_task >> wait_for_operation >> save_sdf_in_gcs + + # Task dependency created via `XComArgs`: + # save_sdf_in_gcs >> upload_sdf_to_big_query diff --git a/airflow/providers/google/marketing_platform/example_dags/example_search_ads.py b/airflow/providers/google/marketing_platform/example_dags/example_search_ads.py index 66c860d54172f..f2e0b399fec05 100644 --- a/airflow/providers/google/marketing_platform/example_dags/example_search_ads.py +++ b/airflow/providers/google/marketing_platform/example_dags/example_search_ads.py @@ -54,7 +54,7 @@ # [END howto_search_ads_generate_report_operator] # [START howto_search_ads_get_report_id] - report_id = "{{ task_instance.xcom_pull('generate_report', key='report_id') }}" + report_id = generate_report.output["report_id"] # [END howto_search_ads_get_report_id] # [START howto_search_ads_get_report_operator] @@ -67,4 +67,8 @@ ) # [END howto_search_ads_getfile_report_operator] - generate_report >> wait_for_report >> download_report + wait_for_report >> download_report + + # Task dependencies created via `XComArgs`: + # generate_report >> wait_for_report + # generate_report >> download_report diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 9077a0f6e75b8..92b67e8157574 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -29,7 +29,7 @@ description: | - `Google Workspace `__ (formerly Google Suite) versions: - - 4.1.0 + - 5.0.0 - 4.0.0 - 3.0.0 - 2.2.0 diff --git a/airflow/providers/google/suite/example_dags/example_sheets.py b/airflow/providers/google/suite/example_dags/example_sheets.py index 171cdcfeb9032..f3e5067e5881d 100644 --- a/airflow/providers/google/suite/example_dags/example_sheets.py +++ b/airflow/providers/google/suite/example_dags/example_sheets.py @@ -57,7 +57,7 @@ # [START print_spreadsheet_url] print_spreadsheet_url = BashOperator( task_id="print_spreadsheet_url", - bash_command="echo {{ task_instance.xcom_pull('create_spreadsheet', key='spreadsheet_url') }}", + bash_command=f"echo {create_spreadsheet.output['spreadsheet_url']}", ) # [END print_spreadsheet_url] diff --git a/airflow/providers/grpc/.latest-doc-only-change.txt b/airflow/providers/grpc/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/grpc/.latest-doc-only-change.txt +++ b/airflow/providers/grpc/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/hashicorp/.latest-doc-only-change.txt b/airflow/providers/hashicorp/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/hashicorp/.latest-doc-only-change.txt +++ b/airflow/providers/hashicorp/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/http/.latest-doc-only-change.txt b/airflow/providers/http/.latest-doc-only-change.txt index b620a033bd63e..ba8d0c7834aba 100644 --- a/airflow/providers/http/.latest-doc-only-change.txt +++ b/airflow/providers/http/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +0dbd0f420cc08e011317e2a9f21f92fff4a64c1b diff --git a/airflow/providers/http/example_dags/example_http.py b/airflow/providers/http/example_dags/example_http.py index b7964f85ebb66..8c93765008a21 100644 --- a/airflow/providers/http/example_dags/example_http.py +++ b/airflow/providers/http/example_dags/example_http.py @@ -26,17 +26,15 @@ from airflow.providers.http.sensors.http import HttpSensor from airflow.utils.dates import days_ago -default_args = { - 'owner': 'airflow', - 'depends_on_past': False, - 'email': ['airflow@example.com'], - 'email_on_failure': False, - 'email_on_retry': False, - 'retries': 1, - 'retry_delay': timedelta(minutes=5), -} - -dag = DAG('example_http_operator', default_args=default_args, tags=['example'], start_date=days_ago(2)) +dag = DAG( + 'example_http_operator', + default_args={ + 'retries': 1, + 'retry_delay': timedelta(minutes=5), + }, + tags=['example'], + start_date=days_ago(2), +) dag.doc_md = __doc__ diff --git a/airflow/providers/jdbc/.latest-doc-only-change.txt b/airflow/providers/jdbc/.latest-doc-only-change.txt index 8e901d0d2aed3..ba8d0c7834aba 100644 --- a/airflow/providers/jdbc/.latest-doc-only-change.txt +++ b/airflow/providers/jdbc/.latest-doc-only-change.txt @@ -1 +1 @@ -40a2476a5db14ee26b5108d72635da116eab720b +0dbd0f420cc08e011317e2a9f21f92fff4a64c1b diff --git a/airflow/providers/jdbc/example_dags/example_jdbc_queries.py b/airflow/providers/jdbc/example_dags/example_jdbc_queries.py index 54cac66e9bf76..651ca9c28196e 100644 --- a/airflow/providers/jdbc/example_dags/example_jdbc_queries.py +++ b/airflow/providers/jdbc/example_dags/example_jdbc_queries.py @@ -25,13 +25,8 @@ from airflow.providers.jdbc.operators.jdbc import JdbcOperator from airflow.utils.dates import days_ago -args = { - 'owner': 'airflow', -} - with DAG( dag_id='example_jdbc_operator', - default_args=args, schedule_interval='0 0 * * *', start_date=days_ago(2), dagrun_timeout=timedelta(minutes=60), diff --git a/airflow/providers/jenkins/CHANGELOG.rst b/airflow/providers/jenkins/CHANGELOG.rst index a1746128e7932..520cd0fe972bb 100644 --- a/airflow/providers/jenkins/CHANGELOG.rst +++ b/airflow/providers/jenkins/CHANGELOG.rst @@ -31,6 +31,8 @@ Bug Fixes appropriate section above if needed. Do not delete the lines(!): * ``Removes pylint from our toolchain (#16682)`` * ``Prepare documentation for July release of providers. (#17015)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` + * ``Updating Jenkins example DAGs to use XComArgs (#16874)`` 2.0.0 ..... diff --git a/airflow/providers/jenkins/example_dags/example_jenkins_job_trigger.py b/airflow/providers/jenkins/example_dags/example_jenkins_job_trigger.py index 9e245e61ae282..66f4803007f9a 100644 --- a/airflow/providers/jenkins/example_dags/example_jenkins_job_trigger.py +++ b/airflow/providers/jenkins/example_dags/example_jenkins_job_trigger.py @@ -24,18 +24,18 @@ from airflow.providers.jenkins.hooks.jenkins import JenkinsHook from airflow.providers.jenkins.operators.jenkins_job_trigger import JenkinsJobTriggerOperator -default_args = { - "owner": "airflow", - "retries": 1, - "retry_delay": timedelta(minutes=5), - "depends_on_past": False, - "concurrency": 8, - "max_active_runs": 8, -} - - with DAG( - "test_jenkins", default_args=default_args, start_date=datetime(2017, 6, 1), schedule_interval=None + "test_jenkins", + default_args={ + "owner": "airflow", + "retries": 1, + "retry_delay": timedelta(minutes=5), + "depends_on_past": False, + "concurrency": 8, + "max_active_runs": 8, + }, + start_date=datetime(2017, 6, 1), + schedule_interval=None, ) as dag: job_trigger = JenkinsJobTriggerOperator( task_id="trigger_job", @@ -45,7 +45,7 @@ jenkins_connection_id="your_jenkins_connection", # T he connection must be configured first ) - def grab_artifact_from_jenkins(**context): + def grab_artifact_from_jenkins(url): """ Grab an artifact from the previous job The python-jenkins library doesn't expose a method for that @@ -53,7 +53,6 @@ def grab_artifact_from_jenkins(**context): """ hook = JenkinsHook("your_jenkins_connection") jenkins_server = hook.get_jenkins_server() - url = context['task_instance'].xcom_pull(task_ids='trigger_job') # The JenkinsJobTriggerOperator store the job url in the xcom variable corresponding to the task # You can then use it to access things or to get the job number # This url looks like : http://jenkins_url/job/job_name/job_number/ @@ -62,6 +61,11 @@ def grab_artifact_from_jenkins(**context): response = jenkins_server.jenkins_open(request) return response # We store the artifact content in a xcom variable for later use - artifact_grabber = PythonOperator(task_id='artifact_grabber', python_callable=grab_artifact_from_jenkins) + artifact_grabber = PythonOperator( + task_id='artifact_grabber', + python_callable=grab_artifact_from_jenkins, + op_kwargs=dict(url=job_trigger.output), + ) - job_trigger >> artifact_grabber + # Task dependency created via `XComArgs`: + # job_trigger >> artifact_grabber diff --git a/airflow/providers/jira/.latest-doc-only-change.txt b/airflow/providers/jira/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/jira/.latest-doc-only-change.txt +++ b/airflow/providers/jira/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/microsoft/azure/CHANGELOG.rst b/airflow/providers/microsoft/azure/CHANGELOG.rst index adbd740e8bbd2..673f44709111a 100644 --- a/airflow/providers/microsoft/azure/CHANGELOG.rst +++ b/airflow/providers/microsoft/azure/CHANGELOG.rst @@ -37,6 +37,8 @@ Bug Fixes appropriate section above if needed. Do not delete the lines(!): * ``Removes pylint from our toolchain (#16682)`` * ``Prepare documentation for July release of providers. (#17015)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` + * ``Remove/refactor default_args pattern for Microsoft example DAGs (#16873)`` 3.0.0 ..... diff --git a/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt b/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt +++ b/airflow/providers/microsoft/mssql/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt b/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt index b620a033bd63e..7087d520d738d 100644 --- a/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt +++ b/airflow/providers/microsoft/winrm/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +48ca9374bfe4a0784b5eb9ec74c1e3262a833677 diff --git a/airflow/providers/mysql/CHANGELOG.rst b/airflow/providers/mysql/CHANGELOG.rst index 425caeb86d695..5ee34dde3c8ba 100644 --- a/airflow/providers/mysql/CHANGELOG.rst +++ b/airflow/providers/mysql/CHANGELOG.rst @@ -30,11 +30,14 @@ Features * ``Added template_fields_renderers for MySQL Operator (#16914)`` * ``Extended template_fields_renderers for MySQL provider (#16987)`` +* ``Parse template parameters field for MySQL operator (#17080)`` .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``Removes pylint from our toolchain (#16682)`` * ``Prepare documentation for July release of providers. (#17015)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` + * ``Remove/refactor default_args pattern for miscellaneous providers (#16872)`` 2.0.0 ..... diff --git a/airflow/providers/mysql/example_dags/example_mysql.py b/airflow/providers/mysql/example_dags/example_mysql.py index 8b425a657100b..029881ff940d8 100644 --- a/airflow/providers/mysql/example_dags/example_mysql.py +++ b/airflow/providers/mysql/example_dags/example_mysql.py @@ -23,13 +23,8 @@ from airflow.providers.mysql.operators.mysql import MySqlOperator from airflow.utils.dates import days_ago -default_args = { - 'owner': 'airflow', -} - dag = DAG( 'example_mysql', - default_args=default_args, start_date=days_ago(2), tags=['example'], ) diff --git a/airflow/providers/neo4j/.latest-doc-only-change.txt b/airflow/providers/neo4j/.latest-doc-only-change.txt index 8e901d0d2aed3..ba8d0c7834aba 100644 --- a/airflow/providers/neo4j/.latest-doc-only-change.txt +++ b/airflow/providers/neo4j/.latest-doc-only-change.txt @@ -1 +1 @@ -40a2476a5db14ee26b5108d72635da116eab720b +0dbd0f420cc08e011317e2a9f21f92fff4a64c1b diff --git a/airflow/providers/neo4j/example_dags/example_neo4j.py b/airflow/providers/neo4j/example_dags/example_neo4j.py index 7d6f2fc7df1b0..3e8b6740cfec8 100644 --- a/airflow/providers/neo4j/example_dags/example_neo4j.py +++ b/airflow/providers/neo4j/example_dags/example_neo4j.py @@ -23,13 +23,8 @@ from airflow.providers.neo4j.operators.neo4j import Neo4jOperator from airflow.utils.dates import days_ago -default_args = { - 'owner': 'airflow', -} - dag = DAG( 'example_neo4j', - default_args=default_args, start_date=days_ago(2), tags=['example'], ) diff --git a/airflow/providers/odbc/.latest-doc-only-change.txt b/airflow/providers/odbc/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/odbc/.latest-doc-only-change.txt +++ b/airflow/providers/odbc/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/opsgenie/.latest-doc-only-change.txt b/airflow/providers/opsgenie/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/opsgenie/.latest-doc-only-change.txt +++ b/airflow/providers/opsgenie/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/oracle/.latest-doc-only-change.txt b/airflow/providers/oracle/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/oracle/.latest-doc-only-change.txt +++ b/airflow/providers/oracle/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/pagerduty/.latest-doc-only-change.txt b/airflow/providers/pagerduty/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/pagerduty/.latest-doc-only-change.txt +++ b/airflow/providers/pagerduty/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/papermill/.latest-doc-only-change.txt b/airflow/providers/papermill/.latest-doc-only-change.txt index 8e901d0d2aed3..ba8d0c7834aba 100644 --- a/airflow/providers/papermill/.latest-doc-only-change.txt +++ b/airflow/providers/papermill/.latest-doc-only-change.txt @@ -1 +1 @@ -40a2476a5db14ee26b5108d72635da116eab720b +0dbd0f420cc08e011317e2a9f21f92fff4a64c1b diff --git a/airflow/providers/papermill/example_dags/example_papermill.py b/airflow/providers/papermill/example_dags/example_papermill.py index 8c6bf2817956f..3ade5e262c645 100644 --- a/airflow/providers/papermill/example_dags/example_papermill.py +++ b/airflow/providers/papermill/example_dags/example_papermill.py @@ -31,13 +31,8 @@ from airflow.providers.papermill.operators.papermill import PapermillOperator from airflow.utils.dates import days_ago -default_args = { - 'owner': 'airflow', -} - with DAG( dag_id='example_papermill_operator', - default_args=default_args, schedule_interval='0 0 * * *', start_date=days_ago(2), dagrun_timeout=timedelta(minutes=60), @@ -69,7 +64,6 @@ def check_notebook(inlets, execution_date): with DAG( dag_id='example_papermill_operator_2', - default_args=default_args, schedule_interval='0 0 * * *', start_date=days_ago(2), dagrun_timeout=timedelta(minutes=60), diff --git a/airflow/providers/plexus/.latest-doc-only-change.txt b/airflow/providers/plexus/.latest-doc-only-change.txt index 8e901d0d2aed3..ba8d0c7834aba 100644 --- a/airflow/providers/plexus/.latest-doc-only-change.txt +++ b/airflow/providers/plexus/.latest-doc-only-change.txt @@ -1 +1 @@ -40a2476a5db14ee26b5108d72635da116eab720b +0dbd0f420cc08e011317e2a9f21f92fff4a64c1b diff --git a/airflow/providers/plexus/example_dags/example_plexus.py b/airflow/providers/plexus/example_dags/example_plexus.py index a077bb71bdd07..2f0a492a67122 100644 --- a/airflow/providers/plexus/example_dags/example_plexus.py +++ b/airflow/providers/plexus/example_dags/example_plexus.py @@ -22,11 +22,10 @@ HOME = '/home/acc' T3_PRERUN_SCRIPT = 'cp {home}/imdb/run_scripts/mlflow.sh {home}/ && chmod +x mlflow.sh'.format(home=HOME) -args = {'owner': 'core scientific', 'retries': 1} dag = DAG( 'test', - default_args=args, + default_args={'owner': 'core scientific', 'retries': 1}, description='testing plexus operator', start_date=days_ago(1), schedule_interval='@once', diff --git a/airflow/providers/postgres/CHANGELOG.rst b/airflow/providers/postgres/CHANGELOG.rst index f877230af433e..18f43284dff82 100644 --- a/airflow/providers/postgres/CHANGELOG.rst +++ b/airflow/providers/postgres/CHANGELOG.rst @@ -31,6 +31,8 @@ Features appropriate section above if needed. Do not delete the lines(!): * ``Removes pylint from our toolchain (#16682)`` * ``Prepare documentation for July release of providers. (#17015)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` + * ``Remove/refactor default_args pattern for miscellaneous providers (#16872)`` 2.0.0 ..... diff --git a/airflow/providers/postgres/example_dags/example_postgres.py b/airflow/providers/postgres/example_dags/example_postgres.py index 5537b1ca64a20..93b5f42702dd6 100644 --- a/airflow/providers/postgres/example_dags/example_postgres.py +++ b/airflow/providers/postgres/example_dags/example_postgres.py @@ -21,8 +21,6 @@ from airflow import DAG from airflow.providers.postgres.operators.postgres import PostgresOperator -default_args = {"owner": "airflow"} - # create_pet_table, populate_pet_table, get_all_pets, and get_birth_date are examples of tasks created by # instantiating the Postgres Operator @@ -30,7 +28,6 @@ dag_id="postgres_operator_dag", start_date=datetime.datetime(2020, 2, 2), schedule_interval="@once", - default_args=default_args, catchup=False, ) as dag: # [START postgres_operator_howto_guide_create_pet_table] diff --git a/airflow/providers/postgres/hooks/postgres.py b/airflow/providers/postgres/hooks/postgres.py index 35ed59f68f634..4e54cf926249a 100644 --- a/airflow/providers/postgres/hooks/postgres.py +++ b/airflow/providers/postgres/hooks/postgres.py @@ -125,6 +125,7 @@ def copy_expert(self, sql: str, filename: str) -> None: So if users want to be aware when the input file does not exist, they have to check its existence by themselves. """ + self.log.info("Running copy expert: %s, filename: %s", sql, filename) if not os.path.isfile(filename): with open(filename, 'w'): pass diff --git a/airflow/providers/postgres/operators/postgres.py b/airflow/providers/postgres/operators/postgres.py index b28cd48678ed1..e4ef23fce23e6 100644 --- a/airflow/providers/postgres/operators/postgres.py +++ b/airflow/providers/postgres/operators/postgres.py @@ -65,7 +65,6 @@ def __init__( self.hook = None def execute(self, context): - self.log.info('Executing: %s', self.sql) self.hook = PostgresHook(postgres_conn_id=self.postgres_conn_id, schema=self.database) self.hook.run(self.sql, self.autocommit, parameters=self.parameters) for output in self.hook.conn.notices: diff --git a/airflow/providers/presto/.latest-doc-only-change.txt b/airflow/providers/presto/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/presto/.latest-doc-only-change.txt +++ b/airflow/providers/presto/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/qubole/.latest-doc-only-change.txt b/airflow/providers/qubole/.latest-doc-only-change.txt index b620a033bd63e..ba8d0c7834aba 100644 --- a/airflow/providers/qubole/.latest-doc-only-change.txt +++ b/airflow/providers/qubole/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +0dbd0f420cc08e011317e2a9f21f92fff4a64c1b diff --git a/airflow/providers/qubole/example_dags/example_qubole.py b/airflow/providers/qubole/example_dags/example_qubole.py index c79a305e4b686..9162753586443 100644 --- a/airflow/providers/qubole/example_dags/example_qubole.py +++ b/airflow/providers/qubole/example_dags/example_qubole.py @@ -55,7 +55,7 @@ """ ) - def compare_result_fn(**kwargs): + def compare_result_fn(ti): """ Compares the results of two QuboleOperator tasks. @@ -64,7 +64,6 @@ def compare_result_fn(**kwargs): :return: True if the files are the same, False otherwise. :rtype: bool """ - ti = kwargs['ti'] qubole_result_1 = hive_show_table.get_results(ti) qubole_result_2 = hive_s3_location.get_results(ti) return filecmp.cmp(qubole_result_1, qubole_result_2) diff --git a/airflow/providers/salesforce/.latest-doc-only-change.txt b/airflow/providers/salesforce/.latest-doc-only-change.txt index b620a033bd63e..526180ad73ee9 100644 --- a/airflow/providers/salesforce/.latest-doc-only-change.txt +++ b/airflow/providers/salesforce/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +763919d4152ffa13433e2489fec85ed286b7b196 diff --git a/airflow/providers/segment/.latest-doc-only-change.txt b/airflow/providers/segment/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/segment/.latest-doc-only-change.txt +++ b/airflow/providers/segment/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/sendgrid/.latest-doc-only-change.txt b/airflow/providers/sendgrid/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/sendgrid/.latest-doc-only-change.txt +++ b/airflow/providers/sendgrid/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/sftp/CHANGELOG.rst b/airflow/providers/sftp/CHANGELOG.rst index aa5d25f40a12f..2c4cc36047167 100644 --- a/airflow/providers/sftp/CHANGELOG.rst +++ b/airflow/providers/sftp/CHANGELOG.rst @@ -31,6 +31,7 @@ Features appropriate section above if needed. Do not delete the lines(!): * ``Remove redundant logging in SFTP Hook (#16704)`` * ``Prepare documentation for July release of providers. (#17015)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` 2.0.0 ..... diff --git a/airflow/providers/singularity/.latest-doc-only-change.txt b/airflow/providers/singularity/.latest-doc-only-change.txt index b620a033bd63e..ba8d0c7834aba 100644 --- a/airflow/providers/singularity/.latest-doc-only-change.txt +++ b/airflow/providers/singularity/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +0dbd0f420cc08e011317e2a9f21f92fff4a64c1b diff --git a/airflow/providers/singularity/example_dags/example_singularity.py b/airflow/providers/singularity/example_dags/example_singularity.py index fc33b0dd65db1..b37cab72261d4 100644 --- a/airflow/providers/singularity/example_dags/example_singularity.py +++ b/airflow/providers/singularity/example_dags/example_singularity.py @@ -23,19 +23,12 @@ from airflow.providers.singularity.operators.singularity import SingularityOperator from airflow.utils.dates import days_ago -default_args = { - 'owner': 'airflow', - 'depends_on_past': False, - 'email': ['airflow@example.com'], - 'email_on_failure': False, - 'email_on_retry': False, - 'retries': 1, - 'retry_delay': timedelta(minutes=5), -} - with DAG( 'singularity_sample', - default_args=default_args, + default_args={ + 'retries': 1, + 'retry_delay': timedelta(minutes=5), + }, schedule_interval=timedelta(minutes=10), start_date=days_ago(0), ) as dag: diff --git a/airflow/providers/slack/.latest-doc-only-change.txt b/airflow/providers/slack/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/slack/.latest-doc-only-change.txt +++ b/airflow/providers/slack/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/snowflake/CHANGELOG.rst b/airflow/providers/snowflake/CHANGELOG.rst index 4c94ebb025f5c..0f0c2fe41c0fd 100644 --- a/airflow/providers/snowflake/CHANGELOG.rst +++ b/airflow/providers/snowflake/CHANGELOG.rst @@ -40,6 +40,8 @@ Bug Fixes * ``Logging and returning info about query execution SnowflakeHook (#15736)`` * ``Removes pylint from our toolchain (#16682)`` * ``Prepare documentation for July release of providers. (#17015)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` + * ``Remove/refactor default_args pattern for miscellaneous providers (#16872)`` 2.0.0 ..... diff --git a/airflow/providers/snowflake/example_dags/example_snowflake.py b/airflow/providers/snowflake/example_dags/example_snowflake.py index 71242c4c3de97..a15eb0815c1f0 100644 --- a/airflow/providers/snowflake/example_dags/example_snowflake.py +++ b/airflow/providers/snowflake/example_dags/example_snowflake.py @@ -47,13 +47,9 @@ "Results in an ASCII table:\n```{{ results_df | tabulate(tablefmt='pretty', headers='keys') }}```" ) -default_args = { - 'owner': 'airflow', -} dag = DAG( 'example_snowflake', - default_args=default_args, start_date=days_ago(2), tags=['example'], ) diff --git a/airflow/providers/sqlite/.latest-doc-only-change.txt b/airflow/providers/sqlite/.latest-doc-only-change.txt index 8e901d0d2aed3..ba8d0c7834aba 100644 --- a/airflow/providers/sqlite/.latest-doc-only-change.txt +++ b/airflow/providers/sqlite/.latest-doc-only-change.txt @@ -1 +1 @@ -40a2476a5db14ee26b5108d72635da116eab720b +0dbd0f420cc08e011317e2a9f21f92fff4a64c1b diff --git a/airflow/providers/sqlite/example_dags/example_sqlite.py b/airflow/providers/sqlite/example_dags/example_sqlite.py index 29579395b0df3..88b4ba2293469 100644 --- a/airflow/providers/sqlite/example_dags/example_sqlite.py +++ b/airflow/providers/sqlite/example_dags/example_sqlite.py @@ -27,11 +27,8 @@ from airflow.providers.sqlite.operators.sqlite import SqliteOperator from airflow.utils.dates import days_ago -default_args = {'owner': 'airflow'} - dag = DAG( dag_id='example_sqlite', - default_args=default_args, schedule_interval='@daily', start_date=days_ago(2), tags=['example'], diff --git a/airflow/providers/ssh/CHANGELOG.rst b/airflow/providers/ssh/CHANGELOG.rst index 263825b042606..9779594fc8bfb 100644 --- a/airflow/providers/ssh/CHANGELOG.rst +++ b/airflow/providers/ssh/CHANGELOG.rst @@ -37,6 +37,7 @@ Bug Fixes appropriate section above if needed. Do not delete the lines(!): * ``Removes pylint from our toolchain (#16682)`` * ``Prepare documentation for July release of providers. (#17015)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` 2.0.0 ..... diff --git a/airflow/providers/tableau/CHANGELOG.rst b/airflow/providers/tableau/CHANGELOG.rst index 4e0a38f0449d1..14d614c8a8d5e 100644 --- a/airflow/providers/tableau/CHANGELOG.rst +++ b/airflow/providers/tableau/CHANGELOG.rst @@ -27,10 +27,14 @@ Features * ``Allow disable SSL for TableauHook (#16365)`` * ``Deprecate Tableau personal token authentication (#16916)`` +* ``Fix bool conversion Verify parameter in Tableau Hook (#17125)`` .. Below changes are excluded from the changelog. Move them to appropriate section above if needed. Do not delete the lines(!): * ``Prepare documentation for July release of providers. (#17015)`` + * ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` + * ``Refactored waiting function for Tableau Jobs (#17034)`` + * ``Remove/refactor default_args pattern for miscellaneous providers (#16872)`` 2.0.0 ..... diff --git a/airflow/providers/tableau/example_dags/example_tableau_refresh_workbook.py b/airflow/providers/tableau/example_dags/example_tableau_refresh_workbook.py index da1cc8be411cd..650f24a3b24b6 100644 --- a/airflow/providers/tableau/example_dags/example_tableau_refresh_workbook.py +++ b/airflow/providers/tableau/example_dags/example_tableau_refresh_workbook.py @@ -27,17 +27,8 @@ from airflow.providers.tableau.sensors.tableau_job_status import TableauJobStatusSensor from airflow.utils.dates import days_ago -DEFAULT_ARGS = { - 'owner': 'airflow', - 'depends_on_past': False, - 'email': ['airflow@example.com'], - 'email_on_failure': False, - 'email_on_retry': False, -} - with DAG( dag_id='example_tableau_refresh_workbook', - default_args=DEFAULT_ARGS, dagrun_timeout=timedelta(hours=2), schedule_interval=None, start_date=days_ago(2), diff --git a/airflow/providers/telegram/.latest-doc-only-change.txt b/airflow/providers/telegram/.latest-doc-only-change.txt index 8e901d0d2aed3..ba8d0c7834aba 100644 --- a/airflow/providers/telegram/.latest-doc-only-change.txt +++ b/airflow/providers/telegram/.latest-doc-only-change.txt @@ -1 +1 @@ -40a2476a5db14ee26b5108d72635da116eab720b +0dbd0f420cc08e011317e2a9f21f92fff4a64c1b diff --git a/airflow/providers/telegram/example_dags/example_telegram.py b/airflow/providers/telegram/example_dags/example_telegram.py index 6485e80ab35e3..31d66293ff774 100644 --- a/airflow/providers/telegram/example_dags/example_telegram.py +++ b/airflow/providers/telegram/example_dags/example_telegram.py @@ -23,13 +23,8 @@ from airflow.providers.telegram.operators.telegram import TelegramOperator from airflow.utils.dates import days_ago -default_args = { - 'owner': 'airflow', -} - dag = DAG( 'example_telegram', - default_args=default_args, start_date=days_ago(2), tags=['example'], ) diff --git a/airflow/providers/trino/.latest-doc-only-change.txt b/airflow/providers/trino/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/trino/.latest-doc-only-change.txt +++ b/airflow/providers/trino/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/vertica/.latest-doc-only-change.txt b/airflow/providers/vertica/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/vertica/.latest-doc-only-change.txt +++ b/airflow/providers/vertica/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/providers/yandex/.latest-doc-only-change.txt b/airflow/providers/yandex/.latest-doc-only-change.txt index b620a033bd63e..ba8d0c7834aba 100644 --- a/airflow/providers/yandex/.latest-doc-only-change.txt +++ b/airflow/providers/yandex/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +0dbd0f420cc08e011317e2a9f21f92fff4a64c1b diff --git a/airflow/providers/yandex/example_dags/example_yandexcloud_dataproc.py b/airflow/providers/yandex/example_dags/example_yandexcloud_dataproc.py index 330cdf6196503..42071a6e6aa58 100644 --- a/airflow/providers/yandex/example_dags/example_yandexcloud_dataproc.py +++ b/airflow/providers/yandex/example_dags/example_yandexcloud_dataproc.py @@ -39,13 +39,8 @@ S3_BUCKET_NAME_FOR_JOB_LOGS = '' -default_args = { - 'owner': 'airflow', -} - with DAG( 'example_yandexcloud_dataproc_operator', - default_args=default_args, schedule_interval=None, start_date=days_ago(1), tags=['example'], diff --git a/airflow/providers/zendesk/.latest-doc-only-change.txt b/airflow/providers/zendesk/.latest-doc-only-change.txt index b620a033bd63e..e7e8156d80b9e 100644 --- a/airflow/providers/zendesk/.latest-doc-only-change.txt +++ b/airflow/providers/zendesk/.latest-doc-only-change.txt @@ -1 +1 @@ -866a601b76e219b3c043e1dbbc8fb22300866351 +b916b7507921129dc48d6add1bdc4b923b60c9b9 diff --git a/airflow/settings.py b/airflow/settings.py index 857dffe8ad3b1..e68b52d53a311 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -431,8 +431,18 @@ def import_local_settings(): del globals()["policy"] log.info("Loaded airflow_local_settings from %s .", airflow_local_settings.__file__) + except ModuleNotFoundError as e: + if e.name == "airflow_local_settings": + log.debug("No airflow_local_settings to import.", exc_info=True) + else: + log.critical( + "Failed to import airflow_local_settings due to a transitive module not found error.", + exc_info=True, + ) + raise except ImportError: - log.debug("Failed to import airflow_local_settings.", exc_info=True) + log.critical("Failed to import airflow_local_settings.", exc_info=True) + raise def initialize(): diff --git a/airflow/www/views.py b/airflow/www/views.py index 117b173f84fc3..8423f911c71a9 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -279,6 +279,29 @@ def task_group_to_dict(task_group): } +def get_key_paths(input_dict): + """Return a list of dot-separated dictionary paths""" + for key, value in input_dict.items(): + if isinstance(value, dict): + for sub_key in get_key_paths(value): + yield '.'.join((key, sub_key)) + else: + yield key + + +def get_value_from_path(key_path, content): + """Return the value from a dictionary based on dot-separated path of keys""" + elem = content + for x in key_path.strip(".").split("."): + try: + x = int(x) + elem = elem[x] + except ValueError: + elem = elem.get(x) + + return elem + + def dag_edges(dag): """ Create the list of edges needed to construct the Graph view. @@ -995,11 +1018,31 @@ def rendered_templates(self): renderer = task.template_fields_renderers.get(template_field, template_field) if renderer in renderers: if isinstance(content, (dict, list)): - content = json.dumps(content, sort_keys=True, indent=4) - html_dict[template_field] = renderers[renderer](content) + json_content = json.dumps(content, sort_keys=True, indent=4) + html_dict[template_field] = renderers[renderer](json_content) + else: + html_dict[template_field] = renderers[renderer](content) else: html_dict[template_field] = Markup("
{}
").format(pformat(content)) + if isinstance(content, dict): + if template_field == 'op_kwargs': + for key, value in content.items(): + renderer = task.template_fields_renderers.get(key, key) + if renderer in renderers: + html_dict['.'.join([template_field, key])] = renderers[renderer](value) + else: + html_dict['.'.join([template_field, key])] = Markup( + "
{}
" + ).format(pformat(value)) + else: + for dict_keys in get_key_paths(content): + template_path = '.'.join((template_field, dict_keys)) + renderer = task.template_fields_renderers.get(template_path, template_path) + if renderer in renderers: + content_value = get_value_from_path(dict_keys, content) + html_dict[template_path] = renderers[renderer](content_value) + return self.render_template( 'airflow/ti_code.html', html_dict=html_dict, diff --git a/breeze b/breeze index 60674b2f3d21a..7aa42952fb7cb 100755 --- a/breeze +++ b/breeze @@ -864,6 +864,12 @@ function breeze::parse_arguments() { echo shift 2 ;; + --mssql-version) + export MSSQL_VERSION="${2}" + echo "MSSQL version: ${MSSQL_VERSION}" + echo + shift 2 + ;; -l | --skip-mounting-local-sources) MOUNT_SELECTED_LOCAL_SOURCES="false" MOUNT_ALL_LOCAL_SOURCES="false" @@ -1581,6 +1587,10 @@ function breeze::prepare_formatted_versions() { tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/") readonly FORMATTED_MYSQL_VERSIONS + FORMATTED_MSSQL_VERSIONS=$(echo "${_breeze_allowed_mssql_versions=""}" | + tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/") + readonly FORMATTED_MSSQL_VERSIONS + FORMATTED_DEFAULT_CI_EXTRAS=$(echo "${DEFAULT_CI_EXTRAS=}" | tr ',' ' ' | fold -w "${indented_screen_width}" -s | sed "s/ /,/g; s/^/${list_prefix}/") readonly FORMATTED_DEFAULT_CI_EXTRAS @@ -2258,9 +2268,15 @@ ${FORMATTED_BACKENDS} ${FORMATTED_POSTGRES_VERSIONS} --mysql-version MYSQL_VERSION - Mysql version used. One of: + MySql version used. One of: ${FORMATTED_MYSQL_VERSIONS} + +--mssql-version MSSQL_VERSION + MSSql version used. One of: + +${FORMATTED_MSSQL_VERSIONS} + " } diff --git a/chart/CHANGELOG.txt b/chart/CHANGELOG.txt index dc9d33ff24b9b..2fd41bf3f9cec 100644 --- a/chart/CHANGELOG.txt +++ b/chart/CHANGELOG.txt @@ -1,4 +1,4 @@ -Airflow Helm Chart 1.1.0, 2021-07-25 +Airflow Helm Chart 1.1.0, 2021-07-26 ------------------------------------ New Features diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 6368eceec5c29..837f3cec96171 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -19,7 +19,7 @@ --- apiVersion: v2 name: airflow -version: 1.1.0 +version: 1.2.0-rc1 appVersion: 2.1.2 description: The official Helm chart to deploy Apache Airflow, a platform to programmatically author, schedule, and monitor workflows diff --git a/dev/README_RELEASE_HELM_CHART.md b/dev/README_RELEASE_HELM_CHART.md index 1e5672268c4ec..2955e8908ab95 100644 --- a/dev/README_RELEASE_HELM_CHART.md +++ b/dev/README_RELEASE_HELM_CHART.md @@ -486,7 +486,7 @@ svn mkdir ${VERSION} cd ${VERSION} # Move the artifacts to svn folder & commit (don't copy or copy & remove - index.yaml) -for f in ../../../airflow-dev/helm-chart/$RC/*; do svn cp $f ${$(basename $f)/rc?/}; done +for f in ../../../airflow-dev/helm-chart/$RC/*; do svn cp $f ${$(basename $f)/}; done svn rm index.yaml svn commit -m "Release Airflow Helm Chart Check ${VERSION} from ${RC}" diff --git a/docs/apache-airflow-providers-airbyte/commits.rst b/docs/apache-airflow-providers-airbyte/commits.rst index c9f26d171c961..bccf7e3cd80f6 100644 --- a/docs/apache-airflow-providers-airbyte/commits.rst +++ b/docs/apache-airflow-providers-airbyte/commits.rst @@ -28,14 +28,30 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-19 ``AirbyteHook - Consider incomplete status (#16965)`` +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` +`b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` +`1e1b21241 `_ 2021-07-08 ``Updating Airbyte example DAG to use XComArgs (#16867)`` +`866a601b7 `_ 2021-06-28 ``Removes pylint from our toolchain (#16682)`` +================================================================================================ =========== =================================================================== + 2.0.0 ..... -Latest change: 2021-06-16 +Latest change: 2021-06-18 ================================================================================================ =========== ================================================================= Commit Committed Subject ================================================================================================ =========== ================================================================= +`bbc627a3d `_ 2021-06-18 ``Prepares documentation for rc2 release of Providers (#16501)`` `cbf8001d7 `_ 2021-06-16 ``Synchronizes updated changelog after buggfix release (#16464)`` `1fba5402b `_ 2021-06-15 ``More documentation update for June providers release (#16405)`` `9c94b72d4 `_ 2021-06-07 ``Updated documentation for June 2021 provider release (#16294)`` diff --git a/docs/apache-airflow-providers-airbyte/index.rst b/docs/apache-airflow-providers-airbyte/index.rst index 0aaeb074723ef..454f24a19c890 100644 --- a/docs/apache-airflow-providers-airbyte/index.rst +++ b/docs/apache-airflow-providers-airbyte/index.rst @@ -57,7 +57,7 @@ Package apache-airflow-providers-airbyte `Airbyte `__ -Release: 2.0.0 +Release: 2.1.0 Provider package ---------------- @@ -106,7 +106,7 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-airbyte 2.0.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-airbyte 2.0.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-airbyte 2.1.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-airbyte 2.1.0 wheel package `_ (`asc `__, `sha512 `__) .. include:: ../../airflow/providers/airbyte/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-amazon/commits.rst b/docs/apache-airflow-providers-amazon/commits.rst index e2fda9ca53426..3a2970ca4cffe 100644 --- a/docs/apache-airflow-providers-amazon/commits.rst +++ b/docs/apache-airflow-providers-amazon/commits.rst @@ -31,11 +31,20 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-26 ``Updating Amazon-AWS example DAGs to use XComArgs (#16868)`` +`763919d41 `_ 2021-07-25 ``Adding custom Salesforce connection type + SalesforceToS3Operator updates (#17162)`` +`8b100fcb4 `_ 2021-07-21 ``ECSOperator / pass context to self.xcom_pull as it was missing (when using reattach) (#17141)`` +`f44d7bd9c `_ 2021-07-21 ``Made S3ToRedshiftOperator transaction safe (#17117)`` +`32582b5bf `_ 2021-07-21 ``Adding SalesforceToS3Operator to Amazon Provider (#17094)`` +`bb1d79cb8 `_ 2021-07-19 ``Fixed template_fields_renderers for Amazon provider (#17087)`` +`8f77a54b5 `_ 2021-07-19 ``removing try-catch block (#17081)`` +`cda78333b `_ 2021-07-18 ``Added docs & doc ref's for AWS transfer operators between SFTP & S3 (#16964)`` +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `fc0250f1d `_ 2021-07-15 ``Allow attaching to previously launched task in ECSOperator (#16685)`` `d3f300fba `_ 2021-07-11 ``Fix wrong template_fields_renderers for AWS operators (#16820)`` @@ -46,7 +55,7 @@ Commit `3a57d9fc6 `_ 2021-06-26 ``Bump ''sphinxcontrib-spelling'' and minor improvements (#16675)`` `2543c74c1 `_ 2021-06-24 ``AWS DataSync cancel task on exception (#11011) (#16589)`` `2ab2cbf93 `_ 2021-06-23 ``Update Boto3 API calls in ECSOperator (#16050)`` -================================================================================================ =========== =============================================================================== +================================================================================================ =========== ================================================================================================= 2.0.0 ..... diff --git a/docs/apache-airflow-providers-amazon/index.rst b/docs/apache-airflow-providers-amazon/index.rst index 0ecf787a730b8..4aebf642b3408 100644 --- a/docs/apache-airflow-providers-amazon/index.rst +++ b/docs/apache-airflow-providers-amazon/index.rst @@ -108,6 +108,7 @@ Dependent package `apache-airflow-providers-mongo `_ ``mongo`` `apache-airflow-providers-mysql `_ ``mysql`` `apache-airflow-providers-postgres `_ ``postgres`` +`apache-airflow-providers-salesforce `_ ``salesforce`` `apache-airflow-providers-ssh `_ ``ssh`` ============================================================================================================== =============== diff --git a/docs/apache-airflow-providers-apache-druid/commits.rst b/docs/apache-airflow-providers-apache-druid/commits.rst index 959849941b29c..eb768ed817c39 100644 --- a/docs/apache-airflow-providers-apache-druid/commits.rst +++ b/docs/apache-airflow-providers-apache-druid/commits.rst @@ -28,14 +28,29 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-20 ``Fix error in Druid connection attribute retrieval (#17095)`` +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` +`b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` +`866a601b7 `_ 2021-06-28 ``Removes pylint from our toolchain (#16682)`` +================================================================================================ =========== =================================================================== + 2.0.0 ..... -Latest change: 2021-06-16 +Latest change: 2021-06-18 ================================================================================================ =========== ======================================================================= Commit Committed Subject ================================================================================================ =========== ======================================================================= +`bbc627a3d `_ 2021-06-18 ``Prepares documentation for rc2 release of Providers (#16501)`` `cbf8001d7 `_ 2021-06-16 ``Synchronizes updated changelog after buggfix release (#16464)`` `1fba5402b `_ 2021-06-15 ``More documentation update for June providers release (#16405)`` `9c94b72d4 `_ 2021-06-07 ``Updated documentation for June 2021 provider release (#16294)`` diff --git a/docs/apache-airflow-providers-apache-druid/index.rst b/docs/apache-airflow-providers-apache-druid/index.rst index 9a32edad6efef..088d35f450576 100644 --- a/docs/apache-airflow-providers-apache-druid/index.rst +++ b/docs/apache-airflow-providers-apache-druid/index.rst @@ -44,7 +44,7 @@ Package apache-airflow-providers-apache-druid `Apache Druid `__. -Release: 2.0.0 +Release: 2.0.1 Provider package ---------------- @@ -93,7 +93,7 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-druid 2.0.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-druid 2.0.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-druid 2.0.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-druid 2.0.1 wheel package `_ (`asc `__, `sha512 `__) .. include:: ../../airflow/providers/apache/druid/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-apache-hive/commits.rst b/docs/apache-airflow-providers-apache-hive/commits.rst index 763e4c75c5e5c..28916c671ed62 100644 --- a/docs/apache-airflow-providers-apache-hive/commits.rst +++ b/docs/apache-airflow-providers-apache-hive/commits.rst @@ -31,15 +31,17 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-23 ``Updating Apache example DAGs to use XComArgs (#16869)`` +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `866a601b7 `_ 2021-06-28 ``Removes pylint from our toolchain (#16682)`` `ce44b6289 `_ 2021-06-25 ``Add Python 3.9 support (#15515)`` -================================================================================================ =========== ================================================================= +================================================================================================ =========== =================================================================== 2.0.0 ..... diff --git a/docs/apache-airflow-providers-apache-sqoop/commits.rst b/docs/apache-airflow-providers-apache-sqoop/commits.rst index defa9510a7cbd..c9d4b06c9c18b 100644 --- a/docs/apache-airflow-providers-apache-sqoop/commits.rst +++ b/docs/apache-airflow-providers-apache-sqoop/commits.rst @@ -33,13 +33,14 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `fc917af8b `_ 2021-07-10 ``Fix Minor Bugs in Apache Sqoop Hook and Operator (#16350)`` `866a601b7 `_ 2021-06-28 ``Removes pylint from our toolchain (#16682)`` -================================================================================================ =========== ================================================================= +================================================================================================ =========== =================================================================== 2.0.0 ..... diff --git a/docs/apache-airflow-providers-cncf-kubernetes/commits.rst b/docs/apache-airflow-providers-cncf-kubernetes/commits.rst index 8b452e6afa121..4c3f83f488655 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/commits.rst +++ b/docs/apache-airflow-providers-cncf-kubernetes/commits.rst @@ -31,11 +31,14 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-21 ``Simplify 'default_args' in Kubernetes example DAGs (#16870)`` +`3939e8416 `_ 2021-07-20 ``Enable using custom pod launcher in Kubernetes Pod Operator (#16945)`` +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `b2c66e45b `_ 2021-07-11 ``BugFix: Using 'json' string in template_field causes issue with K8s Operators (#16930)`` `9d6ae609b `_ 2021-06-28 ``Updating task dependencies (#16624)`` diff --git a/docs/apache-airflow-providers-docker/commits.rst b/docs/apache-airflow-providers-docker/commits.rst index 93605c11f74a1..55faca0c52cda 100644 --- a/docs/apache-airflow-providers-docker/commits.rst +++ b/docs/apache-airflow-providers-docker/commits.rst @@ -31,11 +31,14 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-26 ``Updating Docker example DAGs to use XComArgs (#16871)`` +`cd3307ff2 `_ 2021-07-26 ``fix string encoding when using xcom / json (#13536)`` +`24d02bfa8 `_ 2021-07-19 ``Prepares documentation for RC2 release of Docker Provider (#17066)`` `b076ac592 `_ 2021-07-18 ``[FIX] Docker provider - retry docker in docker (#17061)`` `d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` diff --git a/docs/apache-airflow-providers-google/commits.rst b/docs/apache-airflow-providers-google/commits.rst index 91e0a86565f1a..096205261ff0f 100644 --- a/docs/apache-airflow-providers-google/commits.rst +++ b/docs/apache-airflow-providers-google/commits.rst @@ -35,19 +35,29 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-26 ``Updating Google Cloud example DAGs to use XComArgs (#16875)`` +`aaf44ccac `_ 2021-07-26 ``Updating miscellaneous Google example DAGs to use XComArgs (#16876)`` +`d01cc945d `_ 2021-07-26 ``Fixes several failing tests after broken main (#17222)`` +`babc4250f `_ 2021-07-26 ``Fixes statich check failures (#17218)`` +`5d2224795 `_ 2021-07-25 ``Google Ads Hook: Support newer versions of the google-ads library (#17160)`` +`966b25019 `_ 2021-07-25 ``Fix GCStoGCS operator with replace diabled and existing destination object (#16991)`` +`763919d41 `_ 2021-07-25 ``Adding custom Salesforce connection type + SalesforceToS3Operator updates (#17162)`` +`81bd40933 `_ 2021-07-23 ``[CASSANDRA-16814] Fix cassandra to gcs type inconsistency. (#17183)`` +`026ffe65d `_ 2021-07-19 ``fix: dataprocpysparkjob project_id as self.project_id (#17075)`` +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `a3f5c9380 `_ 2021-07-13 ``Update alias for field_mask in Google Memmcache (#16975)`` `b0f7f91fe `_ 2021-07-07 ``Standardise dataproc location param to region (#16034)`` `866a601b7 `_ 2021-06-28 ``Removes pylint from our toolchain (#16682)`` -================================================================================================ =========== ================================================================= +================================================================================================ =========== ======================================================================================= 4.0.0 ..... diff --git a/docs/apache-airflow-providers-google/index.rst b/docs/apache-airflow-providers-google/index.rst index 99d04554825c3..b7c06c8d4b55d 100644 --- a/docs/apache-airflow-providers-google/index.rst +++ b/docs/apache-airflow-providers-google/index.rst @@ -68,7 +68,7 @@ Google services including: - `Google Workspace `__ (formerly Google Suite) -Release: 4.1.0 +Release: 5.0.0 Provider package ---------------- @@ -165,7 +165,7 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-google 4.1.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-google 4.1.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-google 5.0.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-google 5.0.0 wheel package `_ (`asc `__, `sha512 `__) .. include:: ../../airflow/providers/google/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-jenkins/commits.rst b/docs/apache-airflow-providers-jenkins/commits.rst index dfddc5bdcf28a..032050385de80 100644 --- a/docs/apache-airflow-providers-jenkins/commits.rst +++ b/docs/apache-airflow-providers-jenkins/commits.rst @@ -31,15 +31,17 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-26 ``Updating Jenkins example DAGs to use XComArgs (#16874)`` +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `9ff781ad8 `_ 2021-07-13 ``Fixed to check number key from jenkins response (#16963)`` `866a601b7 `_ 2021-06-28 ``Removes pylint from our toolchain (#16682)`` -================================================================================================ =========== ================================================================= +================================================================================================ =========== =================================================================== 2.0.0 ..... diff --git a/docs/apache-airflow-providers-microsoft-azure/commits.rst b/docs/apache-airflow-providers-microsoft-azure/commits.rst index 21c5ed38689e4..da37e4ed52fa1 100644 --- a/docs/apache-airflow-providers-microsoft-azure/commits.rst +++ b/docs/apache-airflow-providers-microsoft-azure/commits.rst @@ -31,11 +31,13 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-26 ``Remove/refactor default_args pattern for Microsoft example DAGs (#16873)`` +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `866a601b7 `_ 2021-06-28 ``Removes pylint from our toolchain (#16682)`` `caf0a8499 `_ 2021-06-25 ``Add support for managed identity in WASB hook (#16628)`` diff --git a/docs/apache-airflow-providers-mysql/commits.rst b/docs/apache-airflow-providers-mysql/commits.rst index ef0a5d0e3f066..fdd1c14d1ac8d 100644 --- a/docs/apache-airflow-providers-mysql/commits.rst +++ b/docs/apache-airflow-providers-mysql/commits.rst @@ -31,16 +31,19 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-26 ``Remove/refactor default_args pattern for miscellaneous providers (#16872)`` +`a1d3b271e `_ 2021-07-19 ``Parse template parameters field for MySQL operator (#17080)`` +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `6ab00bfcf `_ 2021-07-14 ``Extended template_fields_renderers for MySQL provider (#16987)`` `c3b8212b6 `_ 2021-07-10 ``Added template_fields_renderers for MySQL Operator (#16914)`` `866a601b7 `_ 2021-06-28 ``Removes pylint from our toolchain (#16682)`` -================================================================================================ =========== ================================================================== +================================================================================================ =========== ============================================================================= 2.0.0 ..... diff --git a/docs/apache-airflow-providers-postgres/commits.rst b/docs/apache-airflow-providers-postgres/commits.rst index 589cbdf463535..b8eed905623f2 100644 --- a/docs/apache-airflow-providers-postgres/commits.rst +++ b/docs/apache-airflow-providers-postgres/commits.rst @@ -31,15 +31,17 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-26 ``Remove/refactor default_args pattern for miscellaneous providers (#16872)`` +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `866a601b7 `_ 2021-06-28 ``Removes pylint from our toolchain (#16682)`` `3ee916e9e `_ 2021-06-23 ``Add schema as DbApiHook instance attribute (#16521)`` -================================================================================================ =========== ================================================================= +================================================================================================ =========== ============================================================================= 2.0.0 ..... diff --git a/docs/apache-airflow-providers-sftp/commits.rst b/docs/apache-airflow-providers-sftp/commits.rst index 4a6d04bff98d2..63309a206a252 100644 --- a/docs/apache-airflow-providers-sftp/commits.rst +++ b/docs/apache-airflow-providers-sftp/commits.rst @@ -33,13 +33,14 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `81be82bfb `_ 2021-07-02 ``Remove redundant logging in SFTP Hook (#16704)`` `50e334df3 `_ 2021-06-24 ``Add support for non-RSA type key for SFTP hook (#16314)`` -================================================================================================ =========== ================================================================= +================================================================================================ =========== =================================================================== 2.0.0 ..... diff --git a/docs/apache-airflow-providers-snowflake/commits.rst b/docs/apache-airflow-providers-snowflake/commits.rst index 0d188bc3dd374..5830bd943c4d1 100644 --- a/docs/apache-airflow-providers-snowflake/commits.rst +++ b/docs/apache-airflow-providers-snowflake/commits.rst @@ -31,16 +31,18 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-26 ``Remove/refactor default_args pattern for miscellaneous providers (#16872)`` +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `5999cb9a6 `_ 2021-07-07 ``Adding: Snowflake Role in snowflake provider hook (#16735)`` `8b41c2e0b `_ 2021-07-01 ``Logging and returning info about query execution SnowflakeHook (#15736)`` `866a601b7 `_ 2021-06-28 ``Removes pylint from our toolchain (#16682)`` -================================================================================================ =========== =========================================================================== +================================================================================================ =========== ============================================================================= 2.0.0 ..... diff --git a/docs/apache-airflow-providers-ssh/commits.rst b/docs/apache-airflow-providers-ssh/commits.rst index 5ca3fac6e780a..17eb4fcf81d6e 100644 --- a/docs/apache-airflow-providers-ssh/commits.rst +++ b/docs/apache-airflow-providers-ssh/commits.rst @@ -36,6 +36,7 @@ Latest change: 2021-07-15 ================================================================================================ =========== ========================================================================================= Commit Committed Subject ================================================================================================ =========== ========================================================================================= +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `a2dc01b34 `_ 2021-07-03 ``SSHHook: Using correct hostname for host_key when using non-default ssh port (#15964)`` `7777d4f2f `_ 2021-07-01 ``Correctly load openssh-gerenated private keys in SSHHook (#16756)`` diff --git a/docs/apache-airflow-providers-tableau/commits.rst b/docs/apache-airflow-providers-tableau/commits.rst index 2c1bb7fd4f6e7..e2953446f54f1 100644 --- a/docs/apache-airflow-providers-tableau/commits.rst +++ b/docs/apache-airflow-providers-tableau/commits.rst @@ -31,15 +31,19 @@ For high-level changelog, see :doc:`package information including changelog `_ 2021-07-26 ``Remove/refactor default_args pattern for miscellaneous providers (#16872)`` +`29b6be848 `_ 2021-07-21 ``Refactored waiting function for Tableau Jobs (#17034)`` +`ef3c75df1 `_ 2021-07-21 ``Fix bool conversion Verify parameter in Tableau Hook (#17125)`` +`d02ded65e `_ 2021-07-15 ``Fixed wrongly escaped characters in amazon's changelog (#17020)`` `b916b7507 `_ 2021-07-15 ``Prepare documentation for July release of providers. (#17015)`` `53246ebef `_ 2021-07-15 ``Deprecate Tableau personal token authentication (#16916)`` `df0746e13 `_ 2021-07-10 ``Allow disable SSL for TableauHook (#16365)`` -================================================================================================ =========== ================================================================= +================================================================================================ =========== ============================================================================= 2.0.0 ..... diff --git a/tests/core/test_settings.py b/tests/core/test_settings.py index 0afdb1d7dd9e4..41d7399d22040 100644 --- a/tests/core/test_settings.py +++ b/tests/core/test_settings.py @@ -141,7 +141,7 @@ def test_import_local_settings_without_syspath(self, log_mock): from airflow import settings settings.import_local_settings() - log_mock.assert_called_once_with("Failed to import airflow_local_settings.", exc_info=True) + log_mock.assert_called_once_with("No airflow_local_settings to import.", exc_info=True) def test_policy_function(self): """ diff --git a/tests/executors/test_celery_executor.py b/tests/executors/test_celery_executor.py index 88ea95c4d9a93..cdda8ee0b5b19 100644 --- a/tests/executors/test_celery_executor.py +++ b/tests/executors/test_celery_executor.py @@ -126,14 +126,12 @@ def fake_execute_command(command): task_tuples_to_send = [ ( ('success', 'fake_simple_ti', execute_date, 0), - None, success_command, celery_executor.celery_configuration['task_default_queue'], celery_executor.execute_command, ), ( ('fail', 'fake_simple_ti', execute_date, 0), - None, fail_command, celery_executor.celery_configuration['task_default_queue'], celery_executor.execute_command, @@ -141,8 +139,8 @@ def fake_execute_command(command): ] # "Enqueue" them. We don't have a real SimpleTaskInstance, so directly edit the dict - for (key, simple_ti, command, queue, task) in task_tuples_to_send: - executor.queued_tasks[key] = (command, 1, queue, simple_ti) + for (key, command, queue, task) in task_tuples_to_send: + executor.queued_tasks[key] = (command, 1, queue, None) executor.task_publish_retries[key] = 1 executor._process_tasks(task_tuples_to_send) diff --git a/tests/providers/docker/operators/test_docker.py b/tests/providers/docker/operators/test_docker.py index 78bb9680299a0..58ddc2f70eab9 100644 --- a/tests/providers/docker/operators/test_docker.py +++ b/tests/providers/docker/operators/test_docker.py @@ -47,8 +47,7 @@ def setUp(self): self.client_mock = mock.Mock(spec=APIClient) self.client_mock.create_container.return_value = {'Id': 'some_id'} self.client_mock.images.return_value = [] - self.client_mock.attach.return_value = ['container log'] - self.client_mock.logs.return_value = ['container log'] + self.client_mock.attach.return_value = ['container log 1', 'container log 2'] self.client_mock.pull.return_value = {"status": "pull log"} self.client_mock.wait.return_value = {"StatusCode": 0} self.client_mock.create_host_config.return_value = mock.Mock() @@ -421,13 +420,48 @@ def test_execute_xcom_behavior(self): 'tty': True, } - xcom_push_operator = DockerOperator(**kwargs, do_xcom_push=True) + xcom_push_operator = DockerOperator(**kwargs, do_xcom_push=True, xcom_all=False) + xcom_all_operator = DockerOperator(**kwargs, do_xcom_push=True, xcom_all=True) no_xcom_push_operator = DockerOperator(**kwargs, do_xcom_push=False) xcom_push_result = xcom_push_operator.execute(None) + xcom_all_result = xcom_all_operator.execute(None) no_xcom_push_result = no_xcom_push_operator.execute(None) - assert xcom_push_result == b'container log' + assert xcom_push_result == 'container log 2' + assert xcom_all_result == ['container log 1', 'container log 2'] + assert no_xcom_push_result is None + + def test_execute_xcom_behavior_bytes(self): + self.client_mock.pull.return_value = [b'{"status":"pull log"}'] + self.client_mock.attach.return_value = [b'container log 1 ', b'container log 2'] + kwargs = { + 'api_version': '1.19', + 'command': 'env', + 'environment': {'UNIT': 'TEST'}, + 'private_environment': {'PRIVATE': 'MESSAGE'}, + 'image': 'ubuntu:latest', + 'network_mode': 'bridge', + 'owner': 'unittest', + 'task_id': 'unittest', + 'mounts': [Mount(source='/host/path', target='/container/path', type='bind')], + 'working_dir': '/container/path', + 'shm_size': 1000, + 'host_tmp_dir': '/host/airflow', + 'container_name': 'test_container', + 'tty': True, + } + + xcom_push_operator = DockerOperator(**kwargs, do_xcom_push=True, xcom_all=False) + xcom_all_operator = DockerOperator(**kwargs, do_xcom_push=True, xcom_all=True) + no_xcom_push_operator = DockerOperator(**kwargs, do_xcom_push=False) + + xcom_push_result = xcom_push_operator.execute(None) + xcom_all_result = xcom_all_operator.execute(None) + no_xcom_push_result = no_xcom_push_operator.execute(None) + + assert xcom_push_result == 'container log 2' + assert xcom_all_result == ['container log 1', 'container log 2'] assert no_xcom_push_result is None def test_extra_hosts(self): diff --git a/tests/www/views/test_views.py b/tests/www/views/test_views.py index ebb4f462a2985..a15f708e5f354 100644 --- a/tests/www/views/test_views.py +++ b/tests/www/views/test_views.py @@ -22,7 +22,7 @@ from airflow.configuration import initialize_config from airflow.plugins_manager import AirflowPlugin, EntryPointSource -from airflow.www.views import get_safe_url, truncate_task_duration +from airflow.www.views import get_key_paths, get_safe_url, get_value_from_path, truncate_task_duration from tests.test_utils.config import conf_vars from tests.test_utils.mock_plugins import mock_plugin_manager from tests.test_utils.www import check_content_in_response, check_content_not_in_response @@ -243,3 +243,26 @@ def get_task_instance(session, task): dagrun.refresh_from_db(session=session) # dagrun should be set to QUEUED assert dagrun.get_state() == State.QUEUED + + +TEST_CONTENT_DICT = {"key1": {"key2": "val2", "key3": "val3", "key4": {"key5": "val5"}}} + + +@pytest.mark.parametrize( + "test_content_dict, expected_paths", [(TEST_CONTENT_DICT, ("key1.key2", "key1.key3", "key1.key4.key5"))] +) +def test_generate_key_paths(test_content_dict, expected_paths): + for key_path in get_key_paths(test_content_dict): + assert key_path in expected_paths + + +@pytest.mark.parametrize( + "test_content_dict, test_key_path, expected_value", + [ + (TEST_CONTENT_DICT, "key1.key2", "val2"), + (TEST_CONTENT_DICT, "key1.key3", "val3"), + (TEST_CONTENT_DICT, "key1.key4.key5", "val5"), + ], +) +def test_get_value_from_path(test_content_dict, test_key_path, expected_value): + assert expected_value == get_value_from_path(test_key_path, test_content_dict)