Skip to content

Commit 5289140

Browse files
authored
chore: Update comments and logging in OpenLineage ExtractorManager (#37622)
1 parent 0c2d2c6 commit 5289140

1 file changed

Lines changed: 10 additions & 2 deletions

File tree

  • airflow/providers/openlineage/extractors

airflow/providers/openlineage/extractors/manager.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,18 +57,26 @@ def __init__(self):
5757
self.extractors: dict[str, type[BaseExtractor]] = {}
5858
self.default_extractor = DefaultExtractor
5959

60-
# Comma-separated extractors in OPENLINEAGE_EXTRACTORS variable.
61-
# Extractors should implement BaseExtractor
60+
# Built-in Extractors like Bash and Python
6261
for extractor in _iter_extractor_types():
6362
for operator_class in extractor.get_operator_classnames():
6463
self.extractors[operator_class] = extractor
6564

65+
# Semicolon-separated extractors in Airflow configuration or OPENLINEAGE_EXTRACTORS variable.
66+
# Extractors should implement BaseExtractor
6667
env_extractors = conf.get("openlineage", "extractors", fallback=os.getenv("OPENLINEAGE_EXTRACTORS"))
6768
# skip either when it's empty string or None
6869
if env_extractors:
6970
for extractor in env_extractors.split(";"):
7071
extractor: type[BaseExtractor] = try_import_from_string(extractor.strip())
7172
for operator_class in extractor.get_operator_classnames():
73+
if operator_class in self.extractors:
74+
self.log.debug(
75+
"Duplicate extractor found for `%s`. `%s` will be used instead of `%s`",
76+
operator_class,
77+
extractor,
78+
self.extractors[operator_class],
79+
)
7280
self.extractors[operator_class] = extractor
7381

7482
def add_extractor(self, operator_class: str, extractor: type[BaseExtractor]):

0 commit comments

Comments
 (0)