From 12913e2051631eaf9b03c35f41ec3fcb3b403ead Mon Sep 17 00:00:00 2001 From: flomom Date: Fri, 3 Feb 2023 17:35:40 +0100 Subject: [PATCH 1/8] Add google-cloud-sql operator to clone instances. --- .../providers/google/cloud/hooks/cloud_sql.py | 27 ++++++ .../google/cloud/operators/cloud_sql.py | 87 +++++++++++++++++++ .../google/cloud/hooks/test_cloud_sql.py | 26 ++++++ .../google/cloud/operators/test_cloud_sql.py | 29 +++++++ 4 files changed, 169 insertions(+) diff --git a/airflow/providers/google/cloud/hooks/cloud_sql.py b/airflow/providers/google/cloud/hooks/cloud_sql.py index 7f96d63156e89..f8bd80e3836c0 100644 --- a/airflow/providers/google/cloud/hooks/cloud_sql.py +++ b/airflow/providers/google/cloud/hooks/cloud_sql.py @@ -345,6 +345,33 @@ def import_instance(self, instance: str, body: dict, project_id: str) -> None: except HttpError as ex: raise AirflowException(f"Importing instance {instance} failed: {ex.content}") + @GoogleBaseHook.fallback_to_default_project_id + def clone_instance(self, instance: str, body: dict, project_id: str) -> None: + """ + Clones an instance to a target instance. + + :param instance: Database instance ID to be cloned. This does not include the + project ID. + :param instance: Database instance ID to be used for the clone. This does not include the + project ID. + :param body: The request body, as described in + https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1/instances/clone + :param project_id: Project ID of the project that contains the instance. If set + to None or missing, the default project_id from the Google Cloud connection is used. + # :return: None + # """ + try: + response = ( + self.get_conn() + .instances() + .clone(project=project_id, instance=instance, body=body) + .execute(num_retries=self.num_retries) + ) + operation_name = response["name"] + self._wait_for_operation_to_complete(project_id=project_id, operation_name=operation_name) + except HttpError as ex: + raise AirflowException(f"Cloning of instance {instance} failed: {ex.content}") + def _wait_for_operation_to_complete(self, project_id: str, operation_name: str) -> None: """ Waits for the named operation to complete - checks status of the diff --git a/airflow/providers/google/cloud/operators/cloud_sql.py b/airflow/providers/google/cloud/operators/cloud_sql.py index 494c9eaa5aeb3..6b15b9197ad1f 100644 --- a/airflow/providers/google/cloud/operators/cloud_sql.py +++ b/airflow/providers/google/cloud/operators/cloud_sql.py @@ -523,6 +523,93 @@ def execute(self, context: Context) -> bool | None: return hook.delete_instance(project_id=self.project_id, instance=self.instance) +class CloudSQLCloneInstanceOperator(CloudSQLBaseOperator): + """ + Clones an instance to a target instance + """ + + template_fields: Sequence[str] = ( + "project_id", + "instance", + "destination_instance_name", + "gcp_conn_id", + "api_version", + ) + + def __init__( + self, + *, + instance: str, + destination_instance_name: str, + clone_context: dict | None = None, + project_id: str | None = None, + gcp_conn_id: str = "google_cloud_default", + api_version: str = "v1beta4", + impersonation_chain: str | Sequence[str] | None = None, + **kwargs, + ) -> None: + """ + :param instance: Database instance ID to be cloned. This does not include the + project ID. + :param destination_instance_name: Database instance ID to be created. This does not include the + project ID. + :param clone_context: additional clone_context parameters as described in + https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1/instances/clone + :param project_id: Project ID of the project that contains the instance. If set + to None or missing, the default project_id from the Google Cloud connection is used. + :param gcp_conn_id: The connection ID used to connect to Google Cloud. + :param api_version: API version used (e.g. v1beta4). + :param impersonation_chain: Optional service account to impersonate using short-term + credentials, or chained list of accounts required to get the access_token + of the last account in the list, which will be impersonated in the request. + If set as a string, the account must grant the originating account + the Service Account Token Creator IAM role. + If set as a sequence, the identities from the list must grant + Service Account Token Creator IAM role to the directly preceding identity, with first + account from the list granting this role to the originating account (templated). + """ + self.destination_instance_name = destination_instance_name + self.clone_context = clone_context or {} + super().__init__( + project_id=project_id, + instance=instance, + gcp_conn_id=gcp_conn_id, + api_version=api_version, + impersonation_chain=impersonation_chain, + **kwargs + ) + + def _validate_inputs(self) -> None: + super()._validate_inputs() + if not self.destination_instance_name: + raise AirflowException("The required parameter 'destination_instance_name' is empty or None") + + def execute(self, context: Context): + hook = CloudSQLHook( + gcp_conn_id=self.gcp_conn_id, + api_version=self.api_version, + impersonation_chain=self.impersonation_chain, + ) + if not self._check_if_instance_exists(self.instance, hook): + raise AirflowException( + f"Cloud SQL instance with ID {self.instance} does not exist. " + "Please specify another instance to patch." + ) + else: + body = { + "cloneContext": { + "kind": "sql#cloneContext", + "destinationInstanceName": self.destination_instance_name, + **self.clone_context + } + } + return hook.clone_instance( + project_id=self.project_id, + body=body, + instance=self.instance, + ) + + class CloudSQLCreateInstanceDatabaseOperator(CloudSQLBaseOperator): """ Creates a new database inside a Cloud SQL instance. diff --git a/tests/providers/google/cloud/hooks/test_cloud_sql.py b/tests/providers/google/cloud/hooks/test_cloud_sql.py index 4be1c1277dcef..ce1b2456e7074 100644 --- a/tests/providers/google/cloud/hooks/test_cloud_sql.py +++ b/tests/providers/google/cloud/hooks/test_cloud_sql.py @@ -318,6 +318,32 @@ def test_delete_instance_with_in_progress_retry( operation_name="operation_id", project_id="example-project" ) + @mock.patch( + "airflow.providers.google.cloud.hooks.cloud_sql.CloudSQLHook.get_credentials_and_project_id", + return_value=(mock.MagicMock(), "example-project"), + ) + @mock.patch("airflow.providers.google.cloud.hooks.cloud_sql.CloudSQLHook.get_conn") + @mock.patch("airflow.providers.google.cloud.hooks.cloud_sql.CloudSQLHook._wait_for_operation_to_complete") + def test_instance_clone(self, wait_for_operation_to_complete, get_conn, mock_get_credentials): + clone_method = get_conn.return_value.instances.return_value.clone + execute_method = clone_method.return_value.execute + execute_method.return_value = {"name": "operation_id"} + wait_for_operation_to_complete.return_value = None + body = { + "cloneContext": { + "kind": "sql#cloneContext", + "destinationInstanceName": "clonedInstance", + } + } + self.cloudsql_hook.clone_instance(instance="instance", body=body) + + clone_method.assert_called_once_with(instance="instance", project="example-project", body=body) + execute_method.assert_called_once_with(num_retries=5) + wait_for_operation_to_complete.assert_called_once_with( + operation_name="operation_id", project_id="example-project" + ) + assert 1 == mock_get_credentials.call_count + @mock.patch( "airflow.providers.google.cloud.hooks.cloud_sql.CloudSQLHook.get_credentials_and_project_id", return_value=(mock.MagicMock(), "example-project"), diff --git a/tests/providers/google/cloud/operators/test_cloud_sql.py b/tests/providers/google/cloud/operators/test_cloud_sql.py index c072cfd4fe926..0d6b317637960 100644 --- a/tests/providers/google/cloud/operators/test_cloud_sql.py +++ b/tests/providers/google/cloud/operators/test_cloud_sql.py @@ -36,6 +36,7 @@ CloudSQLImportInstanceOperator, CloudSQLInstancePatchOperator, CloudSQLPatchInstanceDatabaseOperator, + CloudSQLCloneInstanceOperator, ) PROJECT_ID = os.environ.get("PROJECT_ID", "project-id") @@ -362,6 +363,34 @@ def test_instance_delete(self, mock_hook, _check_if_instance_exists): project_id=PROJECT_ID, instance=INSTANCE_NAME ) + @mock.patch( + "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLCloneInstanceOperator._check_if_instance_exists" + ) + @mock.patch("airflow.providers.google.cloud.operators.cloud_sql.CloudSQLHook") + def test_instance_clone(self, mock_hook, _check_if_instance_exists): + destination_instance_name = "clone-test-name" + _check_if_instance_exists.return_value = True + op = CloudSQLCloneInstanceOperator( + project_id=PROJECT_ID, instance=INSTANCE_NAME, destination_instance_name=destination_instance_name, + task_id="id" + ) + result = op.execute(None) + assert result + mock_hook.assert_called_once_with( + api_version="v1beta4", + gcp_conn_id="google_cloud_default", + impersonation_chain=None, + ) + body = { + "cloneContext": { + "kind": "sql#cloneContext", + "destinationInstanceName": destination_instance_name, + } + } + mock_hook.return_value.clone_instance.assert_called_once_with( + project_id=PROJECT_ID, instance=INSTANCE_NAME, body=body + ) + @mock.patch( "airflow.providers.google.cloud.operators.cloud_sql" ".CloudSQLDeleteInstanceOperator._check_if_instance_exists" From bc3addf033599f587d1d9b5d5cc04401b2d1651e Mon Sep 17 00:00:00 2001 From: flomom Date: Fri, 24 Feb 2023 17:22:28 +0100 Subject: [PATCH 2/8] Fix static checks. --- airflow/providers/google/cloud/hooks/cloud_sql.py | 2 +- airflow/providers/google/cloud/operators/cloud_sql.py | 4 ++-- tests/always/test_project_structure.py | 1 + tests/providers/google/cloud/operators/test_cloud_sql.py | 8 +++++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/airflow/providers/google/cloud/hooks/cloud_sql.py b/airflow/providers/google/cloud/hooks/cloud_sql.py index f8bd80e3836c0..d249cc95050f1 100644 --- a/airflow/providers/google/cloud/hooks/cloud_sql.py +++ b/airflow/providers/google/cloud/hooks/cloud_sql.py @@ -359,7 +359,7 @@ def clone_instance(self, instance: str, body: dict, project_id: str) -> None: :param project_id: Project ID of the project that contains the instance. If set to None or missing, the default project_id from the Google Cloud connection is used. # :return: None - # """ + """ try: response = ( self.get_conn() diff --git a/airflow/providers/google/cloud/operators/cloud_sql.py b/airflow/providers/google/cloud/operators/cloud_sql.py index 6b15b9197ad1f..6caf3e4187b19 100644 --- a/airflow/providers/google/cloud/operators/cloud_sql.py +++ b/airflow/providers/google/cloud/operators/cloud_sql.py @@ -576,7 +576,7 @@ def __init__( gcp_conn_id=gcp_conn_id, api_version=api_version, impersonation_chain=impersonation_chain, - **kwargs + **kwargs, ) def _validate_inputs(self) -> None: @@ -600,7 +600,7 @@ def execute(self, context: Context): "cloneContext": { "kind": "sql#cloneContext", "destinationInstanceName": self.destination_instance_name, - **self.clone_context + **self.clone_context, } } return hook.clone_instance( diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py index 438ff2f215b96..f088c7aef5705 100644 --- a/tests/always/test_project_structure.py +++ b/tests/always/test_project_structure.py @@ -288,6 +288,7 @@ class TestGoogleProviderProjectStructure(ExampleCoverageTest, AssetsCoverageTest "airflow.providers.google.cloud.operators.vertex_ai.auto_ml.AutoMLTrainingJobBaseOperator", "airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.UpdateEndpointOperator", "airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job." + "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLCloneInstanceOperator", "GetBatchPredictionJobOperator", } diff --git a/tests/providers/google/cloud/operators/test_cloud_sql.py b/tests/providers/google/cloud/operators/test_cloud_sql.py index 0d6b317637960..ab78bb0c1c4ee 100644 --- a/tests/providers/google/cloud/operators/test_cloud_sql.py +++ b/tests/providers/google/cloud/operators/test_cloud_sql.py @@ -27,6 +27,7 @@ from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.google.cloud.operators.cloud_sql import ( + CloudSQLCloneInstanceOperator, CloudSQLCreateInstanceDatabaseOperator, CloudSQLCreateInstanceOperator, CloudSQLDeleteInstanceDatabaseOperator, @@ -36,7 +37,6 @@ CloudSQLImportInstanceOperator, CloudSQLInstancePatchOperator, CloudSQLPatchInstanceDatabaseOperator, - CloudSQLCloneInstanceOperator, ) PROJECT_ID = os.environ.get("PROJECT_ID", "project-id") @@ -371,8 +371,10 @@ def test_instance_clone(self, mock_hook, _check_if_instance_exists): destination_instance_name = "clone-test-name" _check_if_instance_exists.return_value = True op = CloudSQLCloneInstanceOperator( - project_id=PROJECT_ID, instance=INSTANCE_NAME, destination_instance_name=destination_instance_name, - task_id="id" + project_id=PROJECT_ID, + instance=INSTANCE_NAME, + destination_instance_name=destination_instance_name, + task_id="id", ) result = op.execute(None) assert result From b5c14c1bd598e0401a5d5ace9cfc83f5ffe71b7b Mon Sep 17 00:00:00 2001 From: flomom Date: Mon, 27 Feb 2023 09:31:41 +0100 Subject: [PATCH 3/8] Fix static checks --- .../google/cloud/operators/cloud_sql.py | 39 +++++++++---------- tests/always/test_project_structure.py | 2 +- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/airflow/providers/google/cloud/operators/cloud_sql.py b/airflow/providers/google/cloud/operators/cloud_sql.py index 6caf3e4187b19..f86df52a0d942 100644 --- a/airflow/providers/google/cloud/operators/cloud_sql.py +++ b/airflow/providers/google/cloud/operators/cloud_sql.py @@ -526,6 +526,25 @@ def execute(self, context: Context) -> bool | None: class CloudSQLCloneInstanceOperator(CloudSQLBaseOperator): """ Clones an instance to a target instance + + :param instance: Database instance ID to be cloned. This does not include the + project ID. + :param destination_instance_name: Database instance ID to be created. This does not include the + project ID. + :param clone_context: additional clone_context parameters as described in + https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1/instances/clone + :param project_id: Project ID of the project that contains the instance. If set + to None or missing, the default project_id from the Google Cloud connection is used. + :param gcp_conn_id: The connection ID used to connect to Google Cloud. + :param api_version: API version used (e.g. v1beta4). + :param impersonation_chain: Optional service account to impersonate using short-term + credentials, or chained list of accounts required to get the access_token + of the last account in the list, which will be impersonated in the request. + If set as a string, the account must grant the originating account + the Service Account Token Creator IAM role. + If set as a sequence, the identities from the list must grant + Service Account Token Creator IAM role to the directly preceding identity, with first + account from the list granting this role to the originating account (templated). """ template_fields: Sequence[str] = ( @@ -548,26 +567,6 @@ def __init__( impersonation_chain: str | Sequence[str] | None = None, **kwargs, ) -> None: - """ - :param instance: Database instance ID to be cloned. This does not include the - project ID. - :param destination_instance_name: Database instance ID to be created. This does not include the - project ID. - :param clone_context: additional clone_context parameters as described in - https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1/instances/clone - :param project_id: Project ID of the project that contains the instance. If set - to None or missing, the default project_id from the Google Cloud connection is used. - :param gcp_conn_id: The connection ID used to connect to Google Cloud. - :param api_version: API version used (e.g. v1beta4). - :param impersonation_chain: Optional service account to impersonate using short-term - credentials, or chained list of accounts required to get the access_token - of the last account in the list, which will be impersonated in the request. - If set as a string, the account must grant the originating account - the Service Account Token Creator IAM role. - If set as a sequence, the identities from the list must grant - Service Account Token Creator IAM role to the directly preceding identity, with first - account from the list granting this role to the originating account (templated). - """ self.destination_instance_name = destination_instance_name self.clone_context = clone_context or {} super().__init__( diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py index f088c7aef5705..c75c3e64603c4 100644 --- a/tests/always/test_project_structure.py +++ b/tests/always/test_project_structure.py @@ -288,8 +288,8 @@ class TestGoogleProviderProjectStructure(ExampleCoverageTest, AssetsCoverageTest "airflow.providers.google.cloud.operators.vertex_ai.auto_ml.AutoMLTrainingJobBaseOperator", "airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.UpdateEndpointOperator", "airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job." - "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLCloneInstanceOperator", "GetBatchPredictionJobOperator", + "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLCloneInstanceOperator", } ASSETS_NOT_REQUIRED = { From f0175127d1468bd93985e6c7b2efaf2d8f4aa7bc Mon Sep 17 00:00:00 2001 From: flomom Date: Mon, 27 Feb 2023 17:55:51 +0100 Subject: [PATCH 4/8] Add CloudSQLCloneInstanceOperator to example DAG. --- tests/always/test_project_structure.py | 1 - .../cloud/cloud_sql/example_cloud_sql.py | 20 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py index c75c3e64603c4..438ff2f215b96 100644 --- a/tests/always/test_project_structure.py +++ b/tests/always/test_project_structure.py @@ -289,7 +289,6 @@ class TestGoogleProviderProjectStructure(ExampleCoverageTest, AssetsCoverageTest "airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.UpdateEndpointOperator", "airflow.providers.google.cloud.operators.vertex_ai.batch_prediction_job." "GetBatchPredictionJobOperator", - "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLCloneInstanceOperator", } ASSETS_NOT_REQUIRED = { diff --git a/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py b/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py index 20fe5ba39a60a..4950317aad862 100644 --- a/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +++ b/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py @@ -34,6 +34,7 @@ from airflow import models from airflow.models.xcom_arg import XComArg from airflow.providers.google.cloud.operators.cloud_sql import ( + CloudSQLCloneInstanceOperator, CloudSQLCreateInstanceDatabaseOperator, CloudSQLCreateInstanceOperator, CloudSQLDeleteInstanceDatabaseOperator, @@ -64,7 +65,7 @@ FAILOVER_REPLICA_NAME = f"{INSTANCE_NAME}-failover-replica" READ_REPLICA_NAME = f"{INSTANCE_NAME}-read-replica" - +CLONED_INSTANCE_NAME = f"{INSTANCE_NAME}-clone" # Bodies below represent Cloud SQL instance resources: # https://cloud.google.com/sql/docs/mysql/admin-api/v1beta4/instances @@ -232,6 +233,16 @@ ) # [END howto_operator_cloudsql_import] + # ############################################## # + # ### CLONE AN INSTANCE ######################## # + # ############################################## # + + sql_instance_clone = CloudSQLCloneInstanceOperator( + instance=INSTANCE_NAME, + destination_instance_name=CLONED_INSTANCE_NAME, + task_id="sql_instance_clone" + ) + # ############################################## # # ### DELETING A DATABASE FROM AN INSTANCE ##### # # ############################################## # @@ -260,6 +271,11 @@ sql_instance_failover_replica_delete_task.trigger_rule = TriggerRule.ALL_DONE sql_instance_read_replica_delete_task.trigger_rule = TriggerRule.ALL_DONE + sql_instance_clone_delete_task = CloudSQLDeleteInstanceOperator( + instance=CLONED_INSTANCE_NAME, + task_id="sql_instance_clone_delete_task", + ) + # [START howto_operator_cloudsql_delete] sql_instance_delete_task = CloudSQLDeleteInstanceOperator( instance=INSTANCE_NAME, task_id="sql_instance_delete_task" @@ -284,9 +300,11 @@ >> sql_export_task >> sql_gcp_add_object_permission_task >> sql_import_task + >> sql_instance_clone >> sql_db_delete_task >> sql_instance_failover_replica_delete_task >> sql_instance_read_replica_delete_task + >> sql_instance_clone_delete_task >> sql_instance_delete_task # TEST TEARDOWN >> delete_bucket From 17e3265690b5cbbfc6f018bf7bb6065426a46fbe Mon Sep 17 00:00:00 2001 From: flomom Date: Tue, 28 Feb 2023 12:02:37 +0100 Subject: [PATCH 5/8] Add CloudSQLCloneInstanceOperator to the provider documentation. --- .../google/cloud/operators/cloud_sql.py | 3 +- .../operators/cloud/cloud_sql.rst | 41 +++++++++++++++++++ .../cloud/cloud_sql/example_cloud_sql.py | 3 +- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/airflow/providers/google/cloud/operators/cloud_sql.py b/airflow/providers/google/cloud/operators/cloud_sql.py index f86df52a0d942..f606cbae1cc88 100644 --- a/airflow/providers/google/cloud/operators/cloud_sql.py +++ b/airflow/providers/google/cloud/operators/cloud_sql.py @@ -546,7 +546,7 @@ class CloudSQLCloneInstanceOperator(CloudSQLBaseOperator): Service Account Token Creator IAM role to the directly preceding identity, with first account from the list granting this role to the originating account (templated). """ - + # [START gcp_sql_clone_template_fields] template_fields: Sequence[str] = ( "project_id", "instance", @@ -554,6 +554,7 @@ class CloudSQLCloneInstanceOperator(CloudSQLBaseOperator): "gcp_conn_id", "api_version", ) + # [END gcp_sql_clone_template_fields] def __init__( self, diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst index 6855235d7a66c..bcc816f9dd2d8 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst @@ -473,6 +473,47 @@ More information See Google Cloud SQL API documentation to `patch an instance `_. +.. _howto/operator:CloudSQLCloneInstanceOperator: + +CloudSQLCloneInstanceOperator +----------------------------- + +Clones an Cloud SQL instance. + +For parameter definition, take a look at +:class:`~airflow.providers.google.cloud.operators.cloud_sql.CloudSQLCloneInstanceOperator`. + +Arguments +""""""""" +For `clone_context` object attributes please refer to +`CloneContext `_ + +Using the operator +"""""""""""""""""" +You can create the operator with or without project id. If project id is missing it will be retrieved from the Google +Cloud connection used. Both variants are shown: + +.. exampleinclude:: /../../tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_cloudsql_clone] + :end-before: [END howto_operator_cloudsql_clone] + +Templating +"""""""""" + +.. literalinclude:: /../../airflow/providers/google/cloud/operators/cloud_sql.py + :language: python + :dedent: 4 + :start-after: [START gcp_sql_clone_template_fields] + :end-before: [END gcp_sql_clone_template_fields] + +More information +"""""""""""""""" + +See Google Cloud SQL API documentation to `clone an instance +`_. + .. _howto/operator:CloudSQLExecuteQueryOperator: CloudSQLExecuteQueryOperator diff --git a/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py b/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py index 4950317aad862..68f8208143887 100644 --- a/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +++ b/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py @@ -236,12 +236,13 @@ # ############################################## # # ### CLONE AN INSTANCE ######################## # # ############################################## # - + #[START howto_operator_cloudsql_clone] sql_instance_clone = CloudSQLCloneInstanceOperator( instance=INSTANCE_NAME, destination_instance_name=CLONED_INSTANCE_NAME, task_id="sql_instance_clone" ) + # [END howto_operator_cloudsql_clone] # ############################################## # # ### DELETING A DATABASE FROM AN INSTANCE ##### # From ee5774e581de1c90ad0aed7a037dba8df21753d5 Mon Sep 17 00:00:00 2001 From: flomom Date: Wed, 1 Mar 2023 08:58:59 +0100 Subject: [PATCH 6/8] Correct typo. --- .../operators/cloud/cloud_sql.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst index bcc816f9dd2d8..b591156f65159 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst @@ -512,7 +512,7 @@ More information """""""""""""""" See Google Cloud SQL API documentation to `clone an instance -`_. +`_. .. _howto/operator:CloudSQLExecuteQueryOperator: From 14fc75cba2938aebe136c74e2531bef48805e40f Mon Sep 17 00:00:00 2001 From: flomom Date: Wed, 1 Mar 2023 13:14:13 +0100 Subject: [PATCH 7/8] Fix static checks --- airflow/providers/google/cloud/operators/cloud_sql.py | 5 +++++ .../operators/cloud/cloud_sql.rst | 2 +- .../providers/google/cloud/cloud_sql/example_cloud_sql.py | 6 ++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/airflow/providers/google/cloud/operators/cloud_sql.py b/airflow/providers/google/cloud/operators/cloud_sql.py index f606cbae1cc88..6276ed72e4150 100644 --- a/airflow/providers/google/cloud/operators/cloud_sql.py +++ b/airflow/providers/google/cloud/operators/cloud_sql.py @@ -527,6 +527,10 @@ class CloudSQLCloneInstanceOperator(CloudSQLBaseOperator): """ Clones an instance to a target instance + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudSQLCloneInstanceOperator` + :param instance: Database instance ID to be cloned. This does not include the project ID. :param destination_instance_name: Database instance ID to be created. This does not include the @@ -546,6 +550,7 @@ class CloudSQLCloneInstanceOperator(CloudSQLBaseOperator): Service Account Token Creator IAM role to the directly preceding identity, with first account from the list granting this role to the originating account (templated). """ + # [START gcp_sql_clone_template_fields] template_fields: Sequence[str] = ( "project_id", diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst index b591156f65159..635e8c8068c45 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst @@ -485,7 +485,7 @@ For parameter definition, take a look at Arguments """"""""" -For `clone_context` object attributes please refer to +For ``clone_context`` object attributes please refer to `CloneContext `_ Using the operator diff --git a/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py b/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py index 68f8208143887..33a999b823415 100644 --- a/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py +++ b/tests/system/providers/google/cloud/cloud_sql/example_cloud_sql.py @@ -236,11 +236,9 @@ # ############################################## # # ### CLONE AN INSTANCE ######################## # # ############################################## # - #[START howto_operator_cloudsql_clone] + # [START howto_operator_cloudsql_clone] sql_instance_clone = CloudSQLCloneInstanceOperator( - instance=INSTANCE_NAME, - destination_instance_name=CLONED_INSTANCE_NAME, - task_id="sql_instance_clone" + instance=INSTANCE_NAME, destination_instance_name=CLONED_INSTANCE_NAME, task_id="sql_instance_clone" ) # [END howto_operator_cloudsql_clone] From 28c55be8c6545f018a634a003e8057c0821a06a4 Mon Sep 17 00:00:00 2001 From: flomom Date: Thu, 2 Mar 2023 18:11:21 +0100 Subject: [PATCH 8/8] Fix documentation build. --- airflow/providers/google/cloud/hooks/cloud_sql.py | 2 +- .../operators/cloud/cloud_sql.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/airflow/providers/google/cloud/hooks/cloud_sql.py b/airflow/providers/google/cloud/hooks/cloud_sql.py index bb5add61885e9..edd34e43d7040 100644 --- a/airflow/providers/google/cloud/hooks/cloud_sql.py +++ b/airflow/providers/google/cloud/hooks/cloud_sql.py @@ -360,7 +360,7 @@ def clone_instance(self, instance: str, body: dict, project_id: str) -> None: https://cloud.google.com/sql/docs/mysql/admin-api/rest/v1/instances/clone :param project_id: Project ID of the project that contains the instance. If set to None or missing, the default project_id from the Google Cloud connection is used. - # :return: None + :return: None """ try: response = ( diff --git a/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst b/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst index 635e8c8068c45..80076b95f5f1b 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_sql.rst @@ -485,11 +485,13 @@ For parameter definition, take a look at Arguments """"""""" + For ``clone_context`` object attributes please refer to `CloneContext `_ Using the operator """""""""""""""""" + You can create the operator with or without project id. If project id is missing it will be retrieved from the Google Cloud connection used. Both variants are shown: