diff --git a/airflow/utils/cli.py b/airflow/utils/cli.py index b601c4c07e2c3..0682e65313357 100644 --- a/airflow/utils/cli.py +++ b/airflow/utils/cli.py @@ -62,7 +62,7 @@ def _check_cli_args(args): def action_cli(func=None, check_db=True): def action_logging(f: T) -> T: """ - Decorates function to execute function at the same time submitting action_logging but in CLI context. + Decorate function to execute function at the same time submitting action_logging but in CLI context. It will call action logger callbacks twice, one for pre-execution and the other one for post-execution. @@ -86,7 +86,7 @@ def action_logging(f: T) -> T: @functools.wraps(f) def wrapper(*args, **kwargs): """ - A wrapper for cli functions; assumes Namespace instance as first positional argument. + Wrap cli functions; assume Namespace instance as first positional argument. :param args: Positional argument. It assumes to have Namespace instance at 1st positional argument @@ -127,7 +127,7 @@ def wrapper(*args, **kwargs): def _build_metrics(func_name, namespace): """ - Builds metrics dict from function args. + Build metrics dict from function args. It assumes that function arguments is from airflow.bin.cli module's function and has Namespace instance where it optionally contains "dag_id", "task_id", @@ -173,7 +173,7 @@ def _build_metrics(func_name, namespace): def process_subdir(subdir: str | None): - """Expands path to absolute by replacing 'DAGS_FOLDER', '~', '.', etc.""" + """Expand path to absolute by replacing 'DAGS_FOLDER', '~', '.', etc.""" if subdir: if not settings.DAGS_FOLDER: raise ValueError("DAGS_FOLDER variable in settings should be filled.") @@ -183,7 +183,7 @@ def process_subdir(subdir: str | None): def get_dag_by_file_location(dag_id: str): - """Returns DAG of a given dag_id by looking up file location.""" + """Return DAG of a given dag_id by looking up file location.""" from airflow.models import DagBag, DagModel # Benefit is that logging from other dags in dagbag will not appear @@ -219,7 +219,7 @@ def _search_for_dag_file(val: str | None) -> str | None: def get_dag(subdir: str | None, dag_id: str, from_db: bool = False) -> DAG: """ - Returns DAG of a given dag_id. + Return DAG of a given dag_id. First we'll try to use the given subdir. If that doesn't work, we'll try to find the correct path (assuming it's a file) and failing that, use the configured @@ -248,7 +248,7 @@ def get_dag(subdir: str | None, dag_id: str, from_db: bool = False) -> DAG: def get_dags(subdir: str | None, dag_id: str, use_regex: bool = False): - """Returns DAG(s) matching a given regex or dag_id.""" + """Return DAG(s) matching a given regex or dag_id.""" from airflow.models import DagBag if not use_regex: @@ -276,7 +276,7 @@ def get_dag_by_pickle(pickle_id: int, session: Session = NEW_SESSION) -> DAG: def setup_locations(process, pid=None, stdout=None, stderr=None, log=None): - """Creates logging paths.""" + """Create logging paths.""" if not stderr: stderr = os.path.join(settings.AIRFLOW_HOME, f"airflow-{process}.err") if not stdout: @@ -293,7 +293,7 @@ def setup_locations(process, pid=None, stdout=None, stderr=None, log=None): def setup_logging(filename): - """Creates log file handler for daemon process.""" + """Create log file handler for daemon process.""" root = logging.getLogger() handler = NonCachingFileHandler(filename) formatter = logging.Formatter(settings.SIMPLE_LOG_FORMAT) @@ -306,7 +306,7 @@ def setup_logging(filename): def sigint_handler(sig, frame): """ - Returns without error on SIGINT or SIGTERM signals in interactive command mode. + Return without error on SIGINT or SIGTERM signals in interactive command mode. e.g. CTRL+C or kill """ @@ -315,7 +315,7 @@ def sigint_handler(sig, frame): def sigquit_handler(sig, frame): """ - Helps debug deadlocks by printing stacktraces when this gets a SIGQUIT. + Help debug deadlocks by printing stacktraces when this gets a SIGQUIT. e.g. kill -s QUIT or CTRL+ """ @@ -340,7 +340,7 @@ class ColorMode: def should_use_colors(args) -> bool: - """Processes arguments and decides whether to enable color in output.""" + """Process arguments and decide whether to enable color in output.""" if args.color == ColorMode.ON: return True if args.color == ColorMode.OFF: @@ -361,7 +361,7 @@ def should_ignore_depends_on_past(args) -> bool: def suppress_logs_and_warning(f: T) -> T: - """Decorator to suppress logging and warning messages in cli functions.""" + """Suppress logging and warning messages in cli functions.""" @functools.wraps(f) def _wrapper(*args, **kwargs): diff --git a/airflow/utils/cli_action_loggers.py b/airflow/utils/cli_action_loggers.py index 575f4bfb08cad..0f5f346db3e2d 100644 --- a/airflow/utils/cli_action_loggers.py +++ b/airflow/utils/cli_action_loggers.py @@ -29,7 +29,7 @@ def register_pre_exec_callback(action_logger): - """Registers more action_logger function callback for pre-execution. + """Register more action_logger function callback for pre-execution. This function callback is expected to be called with keyword args. For more about the arguments that is being passed to the callback, @@ -43,7 +43,7 @@ def register_pre_exec_callback(action_logger): def register_post_exec_callback(action_logger): - """Registers more action_logger function callback for post-execution. + """Register more action_logger function callback for post-execution. This function callback is expected to be called with keyword args. For more about the arguments that is being passed to the callback, @@ -57,7 +57,7 @@ def register_post_exec_callback(action_logger): def on_pre_execution(**kwargs): - """Calls callbacks before execution. + """Call callbacks before execution. Note that any exception from callback will be logged but won't be propagated. @@ -73,7 +73,7 @@ def on_pre_execution(**kwargs): def on_post_execution(**kwargs): - """Calls callbacks after execution. + """Call callbacks after execution. As it's being called after execution, it can capture status of execution, duration, etc. Note that any exception from callback will be logged but @@ -91,7 +91,8 @@ def on_post_execution(**kwargs): def default_action_log(sub_command, user, task_id, dag_id, execution_date, host_name, full_command, **_): - """Default action logger callback that behaves similar to ``action_logging``. + """ + Behave similar to ``action_logging``; default action logger callback. The difference is this function uses the global ORM session, and pushes a ``Log`` row into the database instead of actually logging. diff --git a/airflow/utils/cli_app_builder.py b/airflow/utils/cli_app_builder.py index e8cf74e6ec6ed..dd3a40ffd8c73 100644 --- a/airflow/utils/cli_app_builder.py +++ b/airflow/utils/cli_app_builder.py @@ -31,7 +31,7 @@ @lru_cache(maxsize=None) def _return_appbuilder(app: Flask) -> AirflowAppBuilder: - """Returns an appbuilder instance for the given app.""" + """Return an appbuilder instance for the given app.""" init_appbuilder(app) init_plugins(app) return app.appbuilder # type: ignore[attr-defined] diff --git a/airflow/utils/code_utils.py b/airflow/utils/code_utils.py index 65172a0ebe17b..f43a6ce41d059 100644 --- a/airflow/utils/code_utils.py +++ b/airflow/utils/code_utils.py @@ -23,7 +23,7 @@ def get_python_source(x: Any) -> str | None: - """Helper function to get Python source (or not), preventing exceptions.""" + """Get Python source (or not), preventing exceptions.""" if isinstance(x, str): return x @@ -80,7 +80,7 @@ def prepare_code_snippet(file_path: str, line_no: int, context_lines_count: int def get_terminal_formatter(**opts): - """Returns the best formatter available in the current terminal.""" + """Return the best formatter available in the current terminal.""" if "256" in os.environ.get("TERM", ""): from pygments.formatters.terminal256 import Terminal256Formatter diff --git a/airflow/utils/configuration.py b/airflow/utils/configuration.py index 84c5e946ee1a2..15c8beb4d1b73 100644 --- a/airflow/utils/configuration.py +++ b/airflow/utils/configuration.py @@ -27,7 +27,7 @@ def tmp_configuration_copy(chmod=0o600, include_env=True, include_cmds=True): """ - Returns a path for a temporary file including a full copy of the configuration settings. + Return a path for a temporary file including a full copy of the configuration settings. :param include_env: Should the value of configuration from ``AIRFLOW__`` environment variables be included or not diff --git a/airflow/utils/dag_cycle_tester.py b/airflow/utils/dag_cycle_tester.py index 4cea52200da1e..3fb47371dc516 100644 --- a/airflow/utils/dag_cycle_tester.py +++ b/airflow/utils/dag_cycle_tester.py @@ -58,7 +58,7 @@ def check_cycle(dag: DAG) -> None: task_dict = dag.task_dict def _check_adjacent_tasks(task_id, current_task): - """Returns first untraversed child task, else None if all tasks traversed.""" + """Return first untraversed child task, else None if all tasks traversed.""" for adjacent_task in current_task.get_direct_relative_ids(): if visited[adjacent_task] == CYCLE_IN_PROGRESS: msg = f"Cycle detected in DAG: {dag.dag_id}. Faulty task: {task_id}" diff --git a/airflow/utils/dates.py b/airflow/utils/dates.py index a4f70c8dfec05..4f9d37d0c9018 100644 --- a/airflow/utils/dates.py +++ b/airflow/utils/dates.py @@ -140,7 +140,7 @@ def round_time( delta: str | timedelta | relativedelta, start_date: datetime = timezone.make_aware(datetime.min), ): - """Returns ``start_date + i * delta`` for given ``i`` where the result is closest to ``dt``. + """Return ``start_date + i * delta`` for given ``i`` where the result is closest to ``dt``. .. code-block:: pycon diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 92ad989800635..8e217e1d3990b 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -790,7 +790,7 @@ def _get_current_revision(session): def check_migrations(timeout): """ - Function to wait for all airflow migrations to complete. + Wait for all airflow migrations to complete. :param timeout: Timeout for the migration in seconds :return: None @@ -1380,11 +1380,12 @@ def _move_duplicate_data_to_new_table( def check_bad_references(session: Session) -> Iterable[str]: """ - Starting in Airflow 2.2, we began a process of replacing `execution_date` with `run_id` in many tables. + Go through each table and look for records that can't be mapped to a dag run. - Here we go through each table and look for records that can't be mapped to a dag run. When we find such "dangling" rows we back them up in a special table and delete them from the main table. + + Starting in Airflow 2.2, we began a process of replacing `execution_date` with `run_id` in many tables. """ from airflow.models.dagrun import DagRun from airflow.models.renderedtifields import RenderedTaskInstanceFields @@ -1534,7 +1535,7 @@ def _revision_greater(config, this_rev, base_rev): def _revisions_above_min_for_offline(config, revisions) -> None: """ - Checks that all supplied revision ids are above the minimum revision for the dialect. + Check that all supplied revision ids are above the minimum revision for the dialect. :param config: Alembic config :param revisions: list of Alembic revision ids @@ -1725,7 +1726,7 @@ def downgrade(*, to_revision, from_revision=None, show_sql_only=False, session: def drop_airflow_models(connection): """ - Drops all airflow models. + Drop all airflow models. :param connection: SQLAlchemy Connection :return: None @@ -1760,7 +1761,7 @@ def drop_airflow_moved_tables(connection): @provide_session def check(session: Session = NEW_SESSION): """ - Checks if the database works. + Check if the database works. :param session: session of the sqlalchemy """ diff --git a/airflow/utils/decorators.py b/airflow/utils/decorators.py index eaa624f6cfb63..21b6ff3412482 100644 --- a/airflow/utils/decorators.py +++ b/airflow/utils/decorators.py @@ -30,7 +30,7 @@ def apply_defaults(func: T) -> T: """ - This decorator is deprecated. + Use apply_default decorator for the `default_args` feature to work properly; deprecated. In previous versions, all subclasses of BaseOperator must use apply_default decorator for the" `default_args` feature to work properly. @@ -58,7 +58,7 @@ def wrapper(*args, **kwargs): def remove_task_decorator(python_source: str, task_decorator_name: str) -> str: """ - Removes @task or similar decorators as well as @setup and @teardown. + Remove @task or similar decorators as well as @setup and @teardown. :param python_source: The python source code :param task_decorator_name: the decorator name diff --git a/airflow/utils/dot_renderer.py b/airflow/utils/dot_renderer.py index f168566f7e33d..7b895f739f97e 100644 --- a/airflow/utils/dot_renderer.py +++ b/airflow/utils/dot_renderer.py @@ -37,7 +37,7 @@ def _refine_color(color: str): """ - Converts color in #RGB (12 bits) format to #RRGGBB (32 bits), if it possible. + Convert color in #RGB (12 bits) format to #RRGGBB (32 bits), if it possible. Otherwise, it returns the original value. Graphviz does not support colors in #RGB format. @@ -144,7 +144,7 @@ def _draw_nodes( def render_dag_dependencies(deps: dict[str, list[DagDependency]]) -> graphviz.Digraph: """ - Renders the DAG dependency to the DOT object. + Render the DAG dependency to the DOT object. :param deps: List of DAG dependencies :return: Graphviz object @@ -169,7 +169,7 @@ def render_dag_dependencies(deps: dict[str, list[DagDependency]]) -> graphviz.Di def render_dag(dag: DAG, tis: list[TaskInstance] | None = None) -> graphviz.Digraph: """ - Renders the DAG object to the DOT object. + Render the DAG object to the DOT object. If an task instance list is passed, the nodes will be painted according to task statuses. diff --git a/airflow/utils/edgemodifier.py b/airflow/utils/edgemodifier.py index e760beb50a457..a78e6c6499925 100644 --- a/airflow/utils/edgemodifier.py +++ b/airflow/utils/edgemodifier.py @@ -154,7 +154,7 @@ def set_downstream( def update_relative( self, other: DependencyMixin, upstream: bool = True, edge_modifier: EdgeModifier | None = None ) -> None: - """Called if we're not the "main" side of a relationship; we still run the same logic, though.""" + """Update relative if we're not the "main" side of a relationship; still run the same logic.""" if upstream: self.set_upstream(other) else: @@ -162,7 +162,7 @@ def update_relative( def add_edge_info(self, dag, upstream_id: str, downstream_id: str): """ - Adds or updates task info on the DAG for this specific pair of tasks. + Add or update task info on the DAG for this specific pair of tasks. Called either from our relationship trigger methods above, or directly by set_upstream/set_downstream in operators. @@ -172,5 +172,5 @@ def add_edge_info(self, dag, upstream_id: str, downstream_id: str): # Factory functions def Label(label: str): - """Creates an EdgeModifier that sets a human-readable label on the edge.""" + """Create an EdgeModifier that sets a human-readable label on the edge.""" return EdgeModifier(label=label) diff --git a/airflow/utils/email.py b/airflow/utils/email.py index 860b805e28644..8e139e5b52cdc 100644 --- a/airflow/utils/email.py +++ b/airflow/utils/email.py @@ -287,7 +287,7 @@ def send_mime_email( def get_email_address_list(addresses: str | Iterable[str]) -> list[str]: """ - Returns a list of email addresses from the provided input. + Return a list of email addresses from the provided input. :param addresses: A string or iterable of strings containing email addresses. :return: A list of email addresses. @@ -305,7 +305,7 @@ def get_email_address_list(addresses: str | Iterable[str]) -> list[str]: def _get_smtp_connection(host: str, port: int, timeout: int, with_ssl: bool) -> smtplib.SMTP: """ - Returns an SMTP connection to the specified host and port, with optional SSL encryption. + Return an SMTP connection to the specified host and port, with optional SSL encryption. :param host: The hostname or IP address of the SMTP server. :param port: The port number to connect to on the SMTP server. diff --git a/airflow/utils/empty_set.py b/airflow/utils/empty_set.py index 0871c1ec5a12f..eb084f9f17e23 100644 --- a/airflow/utils/empty_set.py +++ b/airflow/utils/empty_set.py @@ -20,7 +20,7 @@ def _get_empty_set_for_configuration() -> set[tuple[str, str]]: """ - Retrieves an empty_set_for_configuration. + Retrieve an empty_set_for_configuration. This method is only needed because configuration module has a deprecated method called set, and it confuses mypy. This method will be removed when we remove the deprecated method. diff --git a/airflow/utils/event_scheduler.py b/airflow/utils/event_scheduler.py index 33cc2b569086f..f234be4ace35b 100644 --- a/airflow/utils/event_scheduler.py +++ b/airflow/utils/event_scheduler.py @@ -31,7 +31,7 @@ def call_regular_interval( arguments=(), kwargs={}, ): - """Helper to call a function at (roughly) a given interval.""" + """Call a function at (roughly) a given interval.""" def repeat(*args, **kwargs): action(*args, **kwargs) diff --git a/airflow/utils/file.py b/airflow/utils/file.py index 2b230b61290b3..9178fa4af5541 100644 --- a/airflow/utils/file.py +++ b/airflow/utils/file.py @@ -119,7 +119,7 @@ def match(path: Path, rules: list[_IgnoreRule]) -> bool: def TemporaryDirectory(*args, **kwargs): - """This function is deprecated. Please use `tempfile.TemporaryDirectory`.""" + """Use `tempfile.TemporaryDirectory`, this function is deprecated.""" import warnings from tempfile import TemporaryDirectory as TmpDir @@ -134,7 +134,7 @@ def TemporaryDirectory(*args, **kwargs): def mkdirs(path, mode): """ - Creates the directory specified by path, creating intermediate directories as necessary. + Create the directory specified by path, creating intermediate directories as necessary. If directory already exists, this is a no-op. @@ -180,7 +180,7 @@ def correct_maybe_zipped(fileloc: None | str | Path) -> None | str | Path: def open_maybe_zipped(fileloc, mode="r"): """ - Opens the given file. + Open the given file. If the path contains a folder with a .zip suffix, then the folder is treated as a zip archive, opening the file inside the archive. @@ -306,7 +306,7 @@ def list_py_file_paths( def find_dag_file_paths(directory: str | os.PathLike[str], safe_mode: bool) -> list[str]: - """Finds file paths of all DAG files.""" + """Find file paths of all DAG files.""" file_paths = [] for file_path in find_path_from_directory(directory, ".airflowignore"): diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py index d407286204694..e07608030d421 100644 --- a/airflow/utils/helpers.py +++ b/airflow/utils/helpers.py @@ -48,7 +48,7 @@ def validate_key(k: str, max_length: int = 250): - """Validates value used as a key.""" + """Validate value used as a key.""" if not isinstance(k, str): raise TypeError(f"The key has to be a string and is {type(k)}:{k}") if len(k) > max_length: @@ -61,7 +61,7 @@ def validate_key(k: str, max_length: int = 250): def validate_group_key(k: str, max_length: int = 200): - """Validates value used as a group key.""" + """Validate value used as a group key.""" if not isinstance(k, str): raise TypeError(f"The key has to be a string and is {type(k)}:{k}") if len(k) > max_length: @@ -73,7 +73,7 @@ def validate_group_key(k: str, max_length: int = 200): def alchemy_to_dict(obj: Any) -> dict | None: - """Transforms a SQLAlchemy model instance into a dictionary.""" + """Transform a SQLAlchemy model instance into a dictionary.""" if not obj: return None output = {} @@ -86,7 +86,7 @@ def alchemy_to_dict(obj: Any) -> dict | None: def ask_yesno(question: str, default: bool | None = None) -> bool: - """Helper to get a yes or no answer from the user.""" + """Get a yes or no answer from the user.""" yes = {"yes", "y"} no = {"no", "n"} @@ -162,7 +162,7 @@ def as_flattened_list(iterable: Iterable[Iterable[T]]) -> list[T]: def parse_template_string(template_string: str) -> tuple[str | None, jinja2.Template | None]: - """Parses Jinja template string.""" + """Parse Jinja template string.""" import jinja2 if "{{" in template_string: # jinja mode @@ -195,7 +195,7 @@ def render_log_filename(ti: TaskInstance, try_number, filename_template) -> str: def convert_camel_to_snake(camel_str: str) -> str: - """Converts CamelCase to snake_case.""" + """Convert CamelCase to snake_case.""" return CAMELCASE_TO_SNAKE_CASE_REGEX.sub(r"_\1", camel_str).lower() @@ -221,7 +221,7 @@ def partition(pred: Callable[[T], bool], iterable: Iterable[T]) -> tuple[Iterabl def chain(*args, **kwargs): - """This function is deprecated. Please use `airflow.models.baseoperator.chain`.""" + """Use `airflow.models.baseoperator.chain`, this function is deprecated.""" warnings.warn( "This function is deprecated. Please use `airflow.models.baseoperator.chain`.", RemovedInAirflow3Warning, @@ -231,7 +231,7 @@ def chain(*args, **kwargs): def cross_downstream(*args, **kwargs): - """This function is deprecated. Please use `airflow.models.baseoperator.cross_downstream`.""" + """Use `airflow.models.baseoperator.cross_downstream`, this function is deprecated.""" warnings.warn( "This function is deprecated. Please use `airflow.models.baseoperator.cross_downstream`.", RemovedInAirflow3Warning, @@ -296,7 +296,7 @@ def render_template_as_native(template: jinja2.Template, context: Context) -> An def exactly_one(*args) -> bool: """ - Returns True if exactly one of *args is "truthy", and False otherwise. + Return True if exactly one of *args is "truthy", and False otherwise. If user supplies an iterable, we raise ValueError and force them to unpack. """ @@ -309,7 +309,7 @@ def exactly_one(*args) -> bool: def at_most_one(*args) -> bool: """ - Returns True if at most one of *args is "truthy", and False otherwise. + Return True if at most one of *args is "truthy", and False otherwise. NOTSET is treated the same as None. diff --git a/airflow/utils/json.py b/airflow/utils/json.py index 7f05c8778d27b..4d89e340c1cd4 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -118,7 +118,7 @@ def object_hook(self, dct: dict) -> object: @staticmethod def orm_object_hook(dct: dict) -> object: - """Creates a readable representation of a serialized object.""" + """Create a readable representation of a serialized object.""" return deserialize(dct, False) diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py index 2a7bd688b5ae7..1314cd7ff9b85 100644 --- a/airflow/utils/log/file_task_handler.py +++ b/airflow/utils/log/file_task_handler.py @@ -193,7 +193,7 @@ def set_context(self, ti: TaskInstance) -> None | SetContextPropagate: @staticmethod def add_triggerer_suffix(full_path, job_id=None): """ - Helper for deriving trigger log filename from task log filename. + Derive trigger log filename from task log filename. E.g. given /path/to/file.log returns /path/to/file.log.trigger.123.log, where 123 is the triggerer id. We use the triggerer ID instead of trigger ID to distinguish @@ -219,7 +219,7 @@ def close(self): self.handler.close() def _render_filename(self, ti: TaskInstance, try_number: int) -> str: - """Returns the worker log filename.""" + """Return the worker log filename.""" with create_session() as session: dag_run = ti.get_dagrun(session=session) template = dag_run.get_log_template(session=session).filename diff --git a/airflow/utils/log/log_reader.py b/airflow/utils/log/log_reader.py index d93f15bb1a983..9e751e47b1abf 100644 --- a/airflow/utils/log/log_reader.py +++ b/airflow/utils/log/log_reader.py @@ -41,7 +41,7 @@ def read_log_chunks( self, ti: TaskInstance, try_number: int | None, metadata ) -> tuple[list[tuple[tuple[str, str]]], dict[str, str]]: """ - Reads chunks of Task Instance logs. + Read chunks of Task Instance logs. :param ti: The taskInstance :param try_number: If provided, logs for the given try will be returned. @@ -65,7 +65,7 @@ def read_log_chunks( def read_log_stream(self, ti: TaskInstance, try_number: int | None, metadata: dict) -> Iterator[str]: """ - Used to continuously read log to the end. + Continuously read log to the end. :param ti: The Task Instance :param try_number: the task try number @@ -134,7 +134,7 @@ def render_log_filename( session: Session = NEW_SESSION, ) -> str: """ - Renders the log attachment filename. + Render the log attachment filename. :param ti: The task instance :param try_number: The task try number diff --git a/airflow/utils/log/logging_mixin.py b/airflow/utils/log/logging_mixin.py index 97e1a29ff6e2c..9f7d5bc9373cd 100644 --- a/airflow/utils/log/logging_mixin.py +++ b/airflow/utils/log/logging_mixin.py @@ -77,12 +77,12 @@ def _get_log(obj: Any, clazz: type[_T]) -> Logger: @classmethod def logger(cls) -> Logger: - """Returns a logger.""" + """Return a logger.""" return LoggingMixin._get_log(cls, cls) @property def log(self) -> Logger: - """Returns a logger.""" + """Return a logger.""" return LoggingMixin._get_log(self, self.__class__) def _set_context(self, context): @@ -171,7 +171,7 @@ def flush(self): def isatty(self): """ - Returns False to indicate the fd is not connected to a tty(-like) device. + Return False to indicate the fd is not connected to a tty(-like) device. For compatibility reasons. """ @@ -218,7 +218,7 @@ def stream(self): def set_context(logger, value): """ - Walks the tree of loggers and tries to set the context for each handler. + Walk the tree of loggers and try to set the context for each handler. :param logger: logger :param value: value to set diff --git a/airflow/utils/log/secrets_masker.py b/airflow/utils/log/secrets_masker.py index c70e469a3c372..efd612c04da2e 100644 --- a/airflow/utils/log/secrets_masker.py +++ b/airflow/utils/log/secrets_masker.py @@ -86,7 +86,11 @@ def get_sensitive_variables_fields(): def should_hide_value_for_key(name): - """Should the value for this given name (Variable name, or key in conn.extra_dejson) be hidden.""" + """ + Return if the value for this given name should be hidden. + + Name might be a Variable name, or key in conn.extra_dejson, for example. + """ from airflow import settings if isinstance(name, str) and settings.HIDE_SENSITIVE_VAR_CONN_FIELDS: @@ -313,7 +317,7 @@ def _test_mode(self) -> bool: return conf.getboolean("core", "unit_test_mode") def _adaptations(self, secret: str) -> Generator[str, None, None]: - """Yields the secret along with any adaptations to the secret that should be masked.""" + """Yield the secret along with any adaptations to the secret that should be masked.""" yield secret if self._mask_adapter: diff --git a/airflow/utils/operator_helpers.py b/airflow/utils/operator_helpers.py index 20f272f4f36df..ef1f05e3048ee 100644 --- a/airflow/utils/operator_helpers.py +++ b/airflow/utils/operator_helpers.py @@ -199,7 +199,7 @@ def determine_kwargs( def make_kwargs_callable(func: Callable[..., R]) -> Callable[..., R]: """ - Creates a new callable that only forwards necessary arguments from any provided input. + Create a new callable that only forwards necessary arguments from any provided input. Make a new callable that can accept any number of positional or keyword arguments but only forwards those required by the given callable func. diff --git a/airflow/utils/platform.py b/airflow/utils/platform.py index 72906683f63ed..2ee11c548f049 100644 --- a/airflow/utils/platform.py +++ b/airflow/utils/platform.py @@ -53,7 +53,7 @@ def is_terminal_support_colors() -> bool: def get_airflow_git_version(): - """Returns the git commit hash representing the current version of the application.""" + """Return the git commit hash representing the current version of the application.""" git_version = None try: git_version = str(pkgutil.get_data("airflow", "git_version"), encoding="UTF-8") diff --git a/airflow/utils/process_utils.py b/airflow/utils/process_utils.py index 74ad50bf7404e..f3104df918cdf 100644 --- a/airflow/utils/process_utils.py +++ b/airflow/utils/process_utils.py @@ -299,7 +299,7 @@ def patch_environ(new_env_variables: dict[str, str]) -> Generator[None, None, No def check_if_pidfile_process_is_running(pid_file: str, process_name: str): """ - Checks if a pidfile already exists and process is still running. + Check if a pidfile already exists and process is still running. If process is dead then pidfile is removed. diff --git a/airflow/utils/providers_configuration_loader.py b/airflow/utils/providers_configuration_loader.py index 543a25c05e6c9..4aeb93edcb39a 100644 --- a/airflow/utils/providers_configuration_loader.py +++ b/airflow/utils/providers_configuration_loader.py @@ -27,8 +27,7 @@ def providers_configuration_loaded(func: Callable[PS, RT]) -> Callable[PS, RT]: """ - Decorator that makes sure that providers configuration is loaded before actually calling - the decorated function. + Make sure that providers configuration is loaded before actually calling the decorated function. ProvidersManager initialization of configuration is relatively inexpensive - it walks through all providers's entrypoints, retrieve the provider_info and loads config yaml parts of the get_info. diff --git a/airflow/utils/python_virtualenv.py b/airflow/utils/python_virtualenv.py index be8bbe0d22c57..1abf6e1f372a1 100644 --- a/airflow/utils/python_virtualenv.py +++ b/airflow/utils/python_virtualenv.py @@ -80,7 +80,8 @@ def prepare_virtualenv( pip_install_options: list[str] | None = None, index_urls: list[str] | None = None, ) -> str: - """Creates a virtual environment and installs the additional python packages. + """ + Create a virtual environment and install the additional python packages. :param venv_directory: The path for directory where the environment will be created. :param python_bin: Path to the Python executable. @@ -126,7 +127,7 @@ def write_python_script( render_template_as_native_obj: bool = False, ): """ - Renders the python script to a file to execute in the virtual environment. + Render the python script to a file to execute in the virtual environment. :param jinja_context: The jinja context variables to unpack and replace with its placeholders in the template file. diff --git a/airflow/utils/retries.py b/airflow/utils/retries.py index 6adce3eb43779..5b67feec439de 100644 --- a/airflow/utils/retries.py +++ b/airflow/utils/retries.py @@ -59,7 +59,8 @@ def retry_db_transaction(_func: F) -> F: def retry_db_transaction(_func: Callable | None = None, *, retries: int = MAX_DB_RETRIES, **retry_kwargs): - """Decorator to retry functions in case of ``OperationalError`` from DB. + """ + Retry functions in case of ``OperationalError`` from DB. It should not be used with ``@provide_session``. """ diff --git a/airflow/utils/serve_logs.py b/airflow/utils/serve_logs.py index bae8a97da93a9..0e926ccd89205 100644 --- a/airflow/utils/serve_logs.py +++ b/airflow/utils/serve_logs.py @@ -162,7 +162,7 @@ def load(self): def serve_logs(port=None): - """Serves logs generated by Worker.""" + """Serve logs generated by Worker.""" setproctitle("airflow serve-logs") wsgi_app = create_app() diff --git a/airflow/utils/session.py b/airflow/utils/session.py index 5c7e9eef505c9..400cac7375f90 100644 --- a/airflow/utils/session.py +++ b/airflow/utils/session.py @@ -60,7 +60,7 @@ def find_session_idx(func: Callable[PS, RT]) -> int: def provide_session(func: Callable[PS, RT]) -> Callable[PS, RT]: """ - Function decorator that provides a session if it isn't provided. + Provide a session if it isn't provided. If you want to reuse a session or run the function as part of a database transaction, you pass it to the function, if not this wrapper diff --git a/airflow/utils/setup_teardown.py b/airflow/utils/setup_teardown.py index cedd5b7a7a5c2..f1de3b4092a08 100644 --- a/airflow/utils/setup_teardown.py +++ b/airflow/utils/setup_teardown.py @@ -181,8 +181,10 @@ def _push_tasks(cls, operator: AbstractOperator | list[AbstractOperator], setup: @classmethod def _update_teardown_downstream(cls, operator: AbstractOperator | list[AbstractOperator]): - """This recursively goes through the tasks downstream of the setup in the context manager, - if found, updates the _teardown_downstream_of_setup accordingly. + """ + Recursively go through the tasks downstream of the setup in the context manager. + + If found, update the _teardown_downstream_of_setup accordingly. """ operator = operator[0] if isinstance(operator, list) else operator @@ -209,8 +211,10 @@ def _get_teardowns(tasks): @classmethod def _update_setup_upstream(cls, operator: AbstractOperator | list[AbstractOperator]): - """This recursively goes through the tasks upstream of the teardown task in the context manager, - if found, updates the _setup_upstream_of_teardown accordingly. + """ + Recursively go through the tasks upstream of the teardown task in the context manager. + + If found, updates the _setup_upstream_of_teardown accordingly. """ operator = operator[0] if isinstance(operator, list) else operator @@ -253,7 +257,7 @@ def set_setup_task_as_roots(cls, roots): @classmethod def set_work_task_roots_and_leaves(cls): - """Sets the work task roots and leaves.""" + """Set the work task roots and leaves.""" if setup_task := cls._context_managed_setup_task: if isinstance(setup_task, list): setup_task = tuple(setup_task) @@ -293,8 +297,7 @@ def set_work_task_roots_and_leaves(cls): @classmethod def set_setup_teardown_relationships(cls): """ - Here we set relationship between setup to setup and - teardown to teardown. + Set relationship between setup to setup and teardown to teardown. code:: python with setuptask >> teardowntask: diff --git a/airflow/utils/sqlalchemy.py b/airflow/utils/sqlalchemy.py index 5690799ce468b..38716d4eb5f55 100644 --- a/airflow/utils/sqlalchemy.py +++ b/airflow/utils/sqlalchemy.py @@ -86,7 +86,7 @@ def process_bind_param(self, value, dialect): def process_result_value(self, value, dialect): """ - Processes DateTimes from the DB making sure it is always returning UTC. + Process DateTimes from the DB making sure to always return UTC. Not using timezone.convert_to_utc as that converts to configured TIMEZONE while the DB might be running with some other setting. We assume UTC @@ -120,7 +120,7 @@ class ExtendedJSON(TypeDecorator): cache_ok = True def db_supports_json(self): - """Checks if the database supports JSON (i.e. is NOT MSSQL).""" + """Check if the database supports JSON (i.e. is NOT MSSQL).""" return not conf.get("database", "sql_alchemy_conn").startswith("mssql") def load_dialect_impl(self, dialect) -> TypeEngine: @@ -545,7 +545,8 @@ def tuple_in_condition( *, session: Session | None = None, ) -> ColumnOperators: - """Generates a tuple-in-collection operator to use in ``.where()``. + """ + Generate a tuple-in-collection operator to use in ``.where()``. For most SQL backends, this generates a simple ``([col, ...]) IN [condition]`` clause. This however does not work with MSSQL, where we need to expand to @@ -591,7 +592,8 @@ def tuple_not_in_condition( *, session: Session | None = None, ) -> ColumnOperators: - """Generates a tuple-not-in-collection operator to use in ``.where()``. + """ + Generate a tuple-not-in-collection operator to use in ``.where()``. This is similar to ``tuple_in_condition`` except generating ``NOT IN``. diff --git a/airflow/utils/state.py b/airflow/utils/state.py index a0c828ee07477..22fb6e27c8140 100644 --- a/airflow/utils/state.py +++ b/airflow/utils/state.py @@ -134,7 +134,7 @@ class State: @classmethod def color(cls, state): - """Returns color for a state.""" + """Return color for a state.""" return cls.state_color.get(state, "white") @classmethod diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py index deee607dd17ae..fd2d0f727ba14 100644 --- a/airflow/utils/task_group.py +++ b/airflow/utils/task_group.py @@ -269,7 +269,7 @@ def update_relative( self, other: DependencyMixin, upstream: bool = True, edge_modifier: EdgeModifier | None = None ) -> None: """ - Overrides TaskMixin.update_relative. + Override TaskMixin.update_relative. Update upstream_group_ids/downstream_group_ids/upstream_task_ids/downstream_task_ids accordingly so that we can reduce the number of edges when displaying Graph view. @@ -341,7 +341,7 @@ def __exit__(self, _type, _value, _tb): TaskGroupContext.pop_context_managed_task_group() def has_task(self, task: BaseOperator) -> bool: - """Returns True if this TaskGroup or its children TaskGroups contains the given task.""" + """Return True if this TaskGroup or its children TaskGroups contains the given task.""" if task.task_id in self.children: return True @@ -422,7 +422,7 @@ def downstream_join_id(self) -> str: return f"{self.group_id}.downstream_join_id" def get_task_group_dict(self) -> dict[str, TaskGroup]: - """Returns a flat dictionary of group_id: TaskGroup.""" + """Return a flat dictionary of group_id: TaskGroup.""" task_group_map = {} def build_map(task_group): @@ -442,14 +442,14 @@ def get_child_by_label(self, label: str) -> DAGNode: return self.children[self.child_id(label)] def serialize_for_task_group(self) -> tuple[DagAttributeTypes, Any]: - """Required by DAGNode.""" + """Serialize task group; required by DAGNode.""" from airflow.serialization.serialized_objects import TaskGroupSerialization return DagAttributeTypes.TASK_GROUP, TaskGroupSerialization.serialize_task_group(self) def hierarchical_alphabetical_sort(self): """ - Sorts children in hierarchical alphabetical order. + Sort children in hierarchical alphabetical order. - groups in alphabetical order first - tasks in alphabetical order after them. @@ -533,7 +533,7 @@ def iter_mapped_task_groups(self) -> Iterator[MappedTaskGroup]: group = group.task_group def iter_tasks(self) -> Iterator[AbstractOperator]: - """Returns an iterator of the child tasks.""" + """Return an iterator of the child tasks.""" from airflow.models.abstractoperator import AbstractOperator groups_to_visit = [self] @@ -577,7 +577,8 @@ def iter_mapped_dependencies(self) -> Iterator[Operator]: @cache def get_parse_time_mapped_ti_count(self) -> int: - """Number of instances a task in this group should be mapped to, when a DAG run is created. + """ + Return the Number of instances a task in this group should be mapped to, when a DAG run is created. This only considers literal mapped arguments, and would return *None* when any non-literal values are used for mapping. @@ -596,7 +597,8 @@ def get_parse_time_mapped_ti_count(self) -> int: ) def get_mapped_ti_count(self, run_id: str, *, session: Session) -> int: - """Number of instances a task in this group should be mapped to at run time. + """ + Return the number of instances a task in this group should be mapped to at run time. This considers both literal and non-literal mapped arguments, and the result is therefore available when all depended tasks have finished. The diff --git a/airflow/utils/timeout.py b/airflow/utils/timeout.py index 06c4ba1de7694..59330e7c0784a 100644 --- a/airflow/utils/timeout.py +++ b/airflow/utils/timeout.py @@ -39,7 +39,7 @@ def __init__(self, seconds=1, error_message="Timeout"): self.error_message = error_message + ", PID: " + str(os.getpid()) def handle_timeout(self, *args): - """Logs information and raises AirflowTaskTimeout.""" + """Log information and raises AirflowTaskTimeout.""" self.log.error("Process timed out, PID: %s", str(os.getpid())) raise AirflowTaskTimeout(self.error_message) @@ -64,7 +64,7 @@ def __init__(self, seconds=1, error_message="Timeout"): self.error_message = error_message + ", PID: " + str(os.getpid()) def handle_timeout(self, signum, frame): - """Logs information and raises AirflowTaskTimeout.""" + """Log information and raises AirflowTaskTimeout.""" self.log.error("Process timed out, PID: %s", str(os.getpid())) raise AirflowTaskTimeout(self.error_message) diff --git a/airflow/utils/timezone.py b/airflow/utils/timezone.py index 1db15ac612195..f7116cd81f98e 100644 --- a/airflow/utils/timezone.py +++ b/airflow/utils/timezone.py @@ -64,7 +64,7 @@ def utcnow() -> dt.datetime: def utc_epoch() -> dt.datetime: - """Gets the epoch in the users timezone.""" + """Get the epoch in the user's timezone.""" # pendulum utcnow() is not used as that sets a TimezoneInfo object # instead of a Timezone. This is not picklable and also creates issues # when using replace() @@ -85,7 +85,7 @@ def convert_to_utc(value: dt.datetime) -> DateTime: def convert_to_utc(value: dt.datetime | None) -> DateTime | None: - """Creates a datetime with the default timezone added if none is associated. + """Create a datetime with the default timezone added if none is associated. :param value: datetime :return: datetime with tzinfo @@ -182,7 +182,7 @@ def make_naive(value, timezone=None): def datetime(*args, **kwargs): """ - Wrapper around datetime.datetime that adds settings.TIMEZONE if tzinfo not specified. + Wrap around datetime.datetime to add settings.TIMEZONE if tzinfo not specified. :return: datetime.datetime """ diff --git a/airflow/utils/trigger_rule.py b/airflow/utils/trigger_rule.py index dabc9f94bd166..6ec37cba81870 100644 --- a/airflow/utils/trigger_rule.py +++ b/airflow/utils/trigger_rule.py @@ -40,12 +40,12 @@ class TriggerRule(str, Enum): @classmethod def is_valid(cls, trigger_rule: str) -> bool: - """Validates a trigger rule.""" + """Validate a trigger rule.""" return trigger_rule in cls.all_triggers() @classmethod def all_triggers(cls) -> set[str]: - """Returns all trigger rules.""" + """Return all trigger rules.""" return set(cls.__members__.values()) def __str__(self) -> str: diff --git a/airflow/utils/types.py b/airflow/utils/types.py index 0eab9b3b8785b..32a0bda6bbb9e 100644 --- a/airflow/utils/types.py +++ b/airflow/utils/types.py @@ -60,7 +60,7 @@ def generate_run_id(self, logical_date: datetime) -> str: @staticmethod def from_run_id(run_id: str) -> DagRunType: - """Resolved DagRun type from run_id.""" + """Resolve DagRun type from run_id.""" for run_type in DagRunType: if run_id and run_id.startswith(f"{run_type.value}__"): return run_type diff --git a/airflow/utils/weekday.py b/airflow/utils/weekday.py index cb404628c1330..1e6f698f69c94 100644 --- a/airflow/utils/weekday.py +++ b/airflow/utils/weekday.py @@ -50,7 +50,7 @@ def get_weekday_number(cls, week_day_str: str): @classmethod def convert(cls, day: str | WeekDay) -> int: - """Helper function that returns the day number in the week.""" + """Return the day number in the week.""" if isinstance(day, WeekDay): return day return cls.get_weekday_number(week_day_str=day) diff --git a/airflow/utils/weight_rule.py b/airflow/utils/weight_rule.py index 002a81f4d589e..f65f2fa77e1af 100644 --- a/airflow/utils/weight_rule.py +++ b/airflow/utils/weight_rule.py @@ -37,7 +37,7 @@ def is_valid(cls, weight_rule: str) -> bool: @classmethod @cache def all_weight_rules(cls) -> set[str]: - """Returns all weight rules.""" + """Return all weight rules.""" return set(cls.__members__.values()) def __str__(self) -> str: