diff --git a/airflow/providers/amazon/aws/example_dags/example_gcs_to_s3.py b/airflow/providers/amazon/aws/example_dags/example_gcs_to_s3.py new file mode 100644 index 0000000000000..d9d04c73ffa31 --- /dev/null +++ b/airflow/providers/amazon/aws/example_dags/example_gcs_to_s3.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +from datetime import datetime + +from airflow import DAG +from airflow.providers.amazon.aws.transfers.gcs_to_s3 import GCSToS3Operator + +BUCKET = os.getenv("BUCKET", "bucket") +S3_KEY = os.getenv("S3_KEY", "s3:///") + +with DAG( + dag_id="example_gcs_to_s3", + schedule_interval=None, + start_date=datetime(2021, 1, 1), + tags=["example"], + catchup=False, +) as dag: + # [START howto_transfer_gcs_to_s3] + gcs_to_s3 = GCSToS3Operator( + task_id="gcs_to_s3", + bucket=BUCKET, + dest_s3_key=S3_KEY, + replace=True, + ) + # [END howto_transfer_gcs_to_s3] diff --git a/airflow/providers/amazon/aws/transfers/gcs_to_s3.py b/airflow/providers/amazon/aws/transfers/gcs_to_s3.py index 3dc8aa0b873ae..b521ce5360741 100644 --- a/airflow/providers/amazon/aws/transfers/gcs_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/gcs_to_s3.py @@ -32,6 +32,10 @@ class GCSToS3Operator(BaseOperator): """ Synchronizes a Google Cloud Storage bucket with an S3 bucket. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:GCSToS3Operator` + :param bucket: The Google Cloud Storage bucket to find the objects. (templated) :param prefix: Prefix string which filters objects whose name begin with this prefix. (templated) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 6cd923d4426d6..413b6dcfed7ca 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -482,6 +482,7 @@ transfers: python-module: airflow.providers.amazon.aws.transfers.dynamodb_to_s3 - source-integration-name: Google Cloud Storage (GCS) target-integration-name: Amazon Simple Storage Service (S3) + how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/gcs_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.gcs_to_s3 - source-integration-name: Amazon Glacier target-integration-name: Google Cloud Storage (GCS) diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/gcs_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/gcs_to_s3.rst new file mode 100644 index 0000000000000..f19d005e943cf --- /dev/null +++ b/docs/apache-airflow-providers-amazon/operators/transfer/gcs_to_s3.rst @@ -0,0 +1,52 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +=================================================== +Google Cloud Storage to Amazon S3 transfer operator +=================================================== + +Use the ``GCSToS3Operator`` transfer to copy the data from Google Cloud Storage to Amazon Simple Storage Service (S3). + +Prerequisite Tasks +------------------ + +.. include:: ../_partials/prerequisite_tasks.rst + +Operators +--------- + +.. _howto/operator:GCSToS3Operator: + +Google Cloud Storage to Amazon S3 +================================= + +To copy data from a Google Cloud Storage bucket to an Amazon S3 bucket you can use +:class:`~airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSToS3Operator` + +Example usage: + +.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_gcs_to_s3.py + :language: python + :dedent: 4 + :start-after: [START howto_transfer_gcs_to_s3] + :end-before: [END howto_transfer_gcs_to_s3] + +Reference +--------- + +* `Google Cloud Storage Client library `__ +* `AWS boto3 Library Documentation for Amazon S3 `__