Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion providers/amazon/docs/operators/sagemaker.rst
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,7 @@ you can use :class:`~airflow.providers.amazon.aws.sensors.sagemaker.SageMakerPro
Branch a DAG based on condition evaluation
==========================================

To branch an Airflow DAG based on upstream task outputs you can use
To branch an Airflow Dag based on upstream task outputs you can use
:class:`~airflow.providers.amazon.aws.operators.sagemaker.SageMakerConditionOperator`.

Simple usage with flat parameters (single condition):
Expand Down
30 changes: 15 additions & 15 deletions providers/amazon/src/airflow/providers/amazon/aws/bundles/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,14 @@

class S3DagBundle(BaseDagBundle):
"""
S3 DAG bundle - exposes a directory in S3 as a DAG bundle.
S3 Dag bundle - exposes a directory in S3 as a Dag bundle.

This allows Airflow to load DAGs directly from an S3 bucket.
This allows Airflow to load Dags directly from an S3 bucket.

:param aws_conn_id: Airflow connection ID for AWS. Defaults to AwsBaseHook.default_conn_name.
:param bucket_name: The name of the S3 bucket containing the DAG files.
:param prefix: Optional subdirectory within the S3 bucket where the DAGs are stored.
If None, DAGs are assumed to be at the root of the bucket (Optional).
:param bucket_name: The name of the S3 bucket containing the Dag files.
:param prefix: Optional subdirectory within the S3 bucket where the Dags are stored.
If None, Dags are assumed to be at the root of the bucket (Optional).
"""

supports_versioning = False
Expand All @@ -53,7 +53,7 @@ def __init__(
self.aws_conn_id = aws_conn_id
self.bucket_name = bucket_name
self.prefix = prefix
# Local path where S3 DAGs are downloaded
# Local path where S3 Dags are downloaded
self.s3_dags_dir: Path = self.base_dir

log = structlog.get_logger(__name__)
Expand All @@ -69,11 +69,11 @@ def __init__(
def _initialize(self):
with self.lock():
if not self.s3_dags_dir.exists():
self._log.info("Creating local DAGs directory: %s", self.s3_dags_dir)
self._log.info("Creating local Dags directory: %s", self.s3_dags_dir)
os.makedirs(self.s3_dags_dir)

if not self.s3_dags_dir.is_dir():
raise AirflowException(f"Local DAGs path: {self.s3_dags_dir} is not a directory.")
raise AirflowException(f"Local Dags path: {self.s3_dags_dir} is not a directory.")

if not self.s3_hook.check_for_bucket(bucket_name=self.bucket_name):
raise AirflowException(f"S3 bucket '{self.bucket_name}' does not exist.")
Expand Down Expand Up @@ -112,22 +112,22 @@ def __repr__(self):
)

def get_current_version(self) -> str | None:
"""Return the current version of the DAG bundle. Currently not supported."""
"""Return the current version of the Dag bundle. Currently not supported."""
return None

@property
def path(self) -> Path:
"""Return the local path to the DAG files."""
return self.s3_dags_dir # Path where DAGs are downloaded.
"""Return the local path to the Dag files."""
return self.s3_dags_dir # Path where Dags are downloaded.

def refresh(self) -> None:
"""Refresh the DAG bundle by re-downloading the DAGs from S3."""
"""Refresh the Dag bundle by re-downloading the Dags from S3."""
if self.version:
raise AirflowException("Refreshing a specific version is not supported")

with self.lock():
self._log.debug(
"Downloading DAGs from s3://%s/%s to %s", self.bucket_name, self.prefix, self.s3_dags_dir
"Downloading Dags from s3://%s/%s to %s", self.bucket_name, self.prefix, self.s3_dags_dir
)
self.s3_hook.sync_to_local_dir(
bucket_name=self.bucket_name,
Expand All @@ -138,15 +138,15 @@ def refresh(self) -> None:

def view_url(self, version: str | None = None) -> str | None:
"""
Return a URL for viewing the DAGs in S3. Currently, versioning is not supported.
Return a URL for viewing the Dags in S3. Currently, versioning is not supported.

This method is deprecated and will be removed when the minimum supported Airflow version is 3.1.
Use `view_url_template` instead.
"""
return self.view_url_template()

def view_url_template(self) -> str | None:
"""Return a URL for viewing the DAGs in S3. Currently, versioning is not supported."""
"""Return a URL for viewing the Dags in S3. Currently, versioning is not supported."""
if self.version:
raise AirflowException("S3 url with version is not supported")
if hasattr(self, "_view_url_template") and self._view_url_template:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ RUN curl "https://github.com/apache/airflow/main/scripts/docker/e
COPY <<"EOF" /install_dags_entrypoint.sh
#!/bin/bash

echo "Downloading DAGs from S3 bucket"
echo "Downloading Dags from S3 bucket"
aws s3 sync "$S3_URI" "$CONTAINER_DAG_PATH"

/home/airflow/entrypoint.sh "$@"
Expand Down Expand Up @@ -81,27 +81,27 @@ ENV AWS_DEFAULT_REGION=$aws_default_region
ENV AWS_SESSION_TOKEN=$aws_session_token


## Loading DAGs
# This Dockerfile supports 2 ways to load DAGs onto the container.
# One is to upload all the DAGs onto an S3 bucket, and then
## Loading Dags
# This Dockerfile supports 2 ways to load Dags onto the container.
# One is to upload all the Dags onto an S3 bucket, and then
# download them onto the container. The other is to copy a local folder with
# the DAGs onto the container.
# If you would like to use an alternative method of loading DAGs, feel free to make the
# the Dags onto the container.
# If you would like to use an alternative method of loading Dags, feel free to make the
# necessary changes to this file.

ARG host_dag_path=./dags
ENV HOST_DAG_PATH=$host_dag_path
ARG container_dag_path=/opt/airflow/dags
ENV CONTAINER_DAG_PATH=$container_dag_path
# Set host_dag_path to the path of the DAGs on the host
# Set host_dag_path to the path of the Dags on the host
# COPY --chown=airflow:root $HOST_DAG_PATH $CONTAINER_DAG_PATH


# Use these arguments to load DAGs onto the container from S3
# Use these arguments to load Dags onto the container from S3
ARG s3_uri
ENV S3_URI=$s3_uri
# If using S3 bucket as source of DAGs, uncommenting the next ENTRYPOINT command will overwrite this one.
# If using S3 bucket as source of Dags, uncommenting the next ENTRYPOINT command will overwrite this one.
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/home/airflow/entrypoint.sh"]

# Uncomment the line if using S3 bucket as the source of DAGs
# Uncomment the line if using S3 bucket as the source of Dags
# ENTRYPOINT ["/usr/bin/dumb-init", "--", "/install_dags_entrypoint.sh"]
Original file line number Diff line number Diff line change
Expand Up @@ -80,18 +80,18 @@ ENV REQUIREMENTS_PATH=$requirements_path
#ENV AWS_DEFAULT_REGION=$aws_default_region
#ENV AWS_SESSION_TOKEN=$aws_session_token

## Loading DAGs
# This Dockerfile supports 2 ways to load DAGs onto the container.
## Loading Dags
# This Dockerfile supports 2 ways to load Dags onto the container.
# One is to download them from S3 at runtime during the Lambda app invocation. The other
# is to copy the dags into the image at build time, this will make task execution
# much faster, since the images will already be present but the image will need to be rebuilt
# every time the DAGs are updated.
# If you would like to use an alternative method of loading DAGs, feel free to make the
# every time the Dags are updated.
# If you would like to use an alternative method of loading Dags, feel free to make the
# necessary changes to this file.

ARG host_dag_path=./dags
ENV HOST_DAG_PATH=$host_dag_path
# Uncomment the line below to copy the DAGs from the host to the container.
# Uncomment the line below to copy the Dags from the host to the container.
# COPY $HOST_DAG_PATH $AIRFLOW__CORE__DAGS_FOLDER

# Use these arguments to load DAGs at runtime. If you are using the provided Lambda function (app.py),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ def get_queue_url():


def fetch_dags_from_s3(s3_uri):
"""Fetch DAGs from S3 and sync them to the local dags directory."""
log.info("Fetching DAGs from S3 URI: %s", s3_uri)
"""Fetch Dags from S3 and sync them to the local dags directory."""
log.info("Fetching Dags from S3 URI: %s", s3_uri)
# Use a named temporary directory for the local dags folder, only tmp is writeable in Lambda
local_dags_dir = mkdtemp(prefix="airflow_dags_")
log.info("Setting AIRFLOW__CORE__DAGS_FOLDER to: %s", local_dags_dir)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -259,8 +259,8 @@ def _handle_failed_job(self, job):
# if the job fails before the Airflow process on the container has started. These failures
# can be caused by a Batch API failure, container misconfiguration etc.
# If the container is able to start up and run the Airflow process, any failures after that
# (i.e. DAG failures) will not be marked as Failed by AWS Batch, because Batch on assumes
# responsibility for ensuring the process started. Failures in the DAG will be caught by
# (i.e. Dag failures) will not be marked as Failed by AWS Batch, because Batch on assumes
# responsibility for ensuring the process started. Failures in the Dag will be caught by
# Airflow, which will be handled separately.
job_info = self.active_workers.id_to_job_info[job.job_id]
task_key = self.active_workers.id_to_key[job.job_id]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -586,7 +586,7 @@ def _get_caller(self, target_function_name: str = "execute") -> str:
@return_on_error("00000000-0000-0000-0000-000000000000")
def _generate_dag_key() -> str:
"""
Generate a DAG key.
Generate a Dag key.

The Object Identifier (OID) namespace is used to salt the dag_id value.
That salted value is used to generate a SHA-1 hash which, by definition,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@

class AppflowBaseOperator(AwsBaseOperator[AppflowHook]):
"""
Amazon AppFlow Base Operator class (not supposed to be used directly in DAGs).
Amazon AppFlow Base Operator class (not supposed to be used directly in Dags).

:param source: The source name (Supported: salesforce, zendesk)
:param flow_name: The flow name
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@

class ComprehendBaseOperator(AwsBaseOperator[ComprehendHook]):
"""
This is the base operator for Comprehend Service operators (not supposed to be used directly in DAGs).
This is the base operator for Comprehend Service operators (not supposed to be used directly in Dags).

:param input_data_config: The input properties for a PII entities detection job. (templated)
:param output_data_config: Provides `configuration` parameters for the output of PII entity detection
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1196,7 +1196,7 @@ class EmrServerlessStartJobOperator(AwsBaseOperator[EmrServerlessHook]):
This implies waiting for completion. This mode requires aiobotocore module to be installed.
(default: False, but can be overridden in config file by setting default_deferrable to True)
:param enable_application_ui_links: If True, the operator will generate one-time links to EMR Serverless
application UIs. The generated links will allow any user with access to the DAG to see the Spark or
application UIs. The generated links will allow any user with access to the Dag to see the Spark or
Tez UI or Spark stdout logs. Defaults to False.
:param cancel_on_kill: If True, the EMR Serverless job will be cancelled when the task is killed
while in deferrable mode. This ensures that orphan jobs are not left running in EMR Serverless
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,23 +41,23 @@ class MwaaTriggerDagRunOperator(AwsBaseOperator[MwaaHook]):
:ref:`howto/operator:MwaaTriggerDagRunOperator`

:param env_name: The MWAA environment name (templated)
:param trigger_dag_id: The ID of the DAG to be triggered (templated)
:param trigger_dag_id: The ID of the Dag to be triggered (templated)
:param trigger_run_id: The Run ID. This together with trigger_dag_id are a unique key. (templated)
:param logical_date: The logical date (previously called execution date). This is the time or interval
covered by this DAG run, according to the DAG definition. This together with trigger_dag_id are a
covered by this Dag run, according to the Dag definition. This together with trigger_dag_id are a
unique key. This field is required if your environment is running with Airflow 3. (templated)
:param data_interval_start: The beginning of the interval the DAG run covers
:param data_interval_end: The end of the interval the DAG run covers
:param data_interval_start: The beginning of the interval the Dag run covers
:param data_interval_end: The end of the interval the Dag run covers
:param conf: Additional configuration parameters. The value of this field can be set only when creating
the object. (templated)
:param note: Contains manually entered notes by the user about the DagRun. (templated)
:param airflow_version: The Airflow major version the MWAA environment runs.
This parameter is only used if the local web token method is used to call Airflow API. (templated)

:param wait_for_completion: Whether to wait for DAG run to stop. (default: False)
:param wait_for_completion: Whether to wait for Dag run to stop. (default: False)
:param waiter_delay: Time in seconds to wait between status checks. (default: 120)
:param waiter_max_attempts: Maximum number of attempts to check for DAG run completion. (default: 720)
:param deferrable: If True, the operator will wait asynchronously for the DAG run to stop.
:param waiter_max_attempts: Maximum number of attempts to check for Dag run completion. (default: 720)
:param deferrable: If True, the operator will wait asynchronously for the Dag run to stop.
This implies waiting for completion. This mode requires aiobotocore module to be installed.
(default: False)
:param aws_conn_id: The Airflow connection used for AWS credentials.
Expand Down Expand Up @@ -122,10 +122,10 @@ def __init__(
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict:
validated_event = validate_execute_complete_event(event)
if validated_event["status"] != "success":
raise AirflowException(f"DAG run failed: {validated_event}")
raise AirflowException(f"Dag run failed: {validated_event}")

dag_run_id = validated_event["dag_run_id"]
self.log.info("DAG run %s of DAG %s completed", dag_run_id, self.trigger_dag_id)
self.log.info("Dag run %s of Dag %s completed", dag_run_id, self.trigger_dag_id)
return self.hook.invoke_rest_api(
env_name=self.env_name,
path=f"/dags/{self.trigger_dag_id}/dagRuns/{dag_run_id}",
Expand Down Expand Up @@ -157,9 +157,9 @@ def execute(self, context: Context) -> dict:
)

dag_run_id = response["RestApiResponse"]["dag_run_id"]
self.log.info("DAG run %s of DAG %s created", dag_run_id, self.trigger_dag_id)
self.log.info("Dag run %s of Dag %s created", dag_run_id, self.trigger_dag_id)

task_description = f"DAG run {dag_run_id} of DAG {self.trigger_dag_id} to complete"
task_description = f"Dag run {dag_run_id} of Dag {self.trigger_dag_id} to complete"
if self.deferrable:
self.log.info("Deferring for %s", task_description)
self.defer(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2101,8 +2101,8 @@ def _cast(value: Any) -> Any:
Cast Jinja-rendered string values to appropriate Python types.

This is a compatibility shim for environments where
``render_template_as_native_obj=True`` is not available at the DAG or
task level (e.g., YAML DAGs). Once task-level native rendering
``render_template_as_native_obj=True`` is not available at the Dag or
task level (e.g., YAML Dags). Once task-level native rendering
is widely supported, this method can be removed in favor of letting
Airflow handle the casting natively.

Expand Down
Loading
Loading