Skip to content

Move some base_aws logging from info to debug level #20858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
uranusjr merged 1 commit into from
Jan 21, 2022
Merged

Move some base_aws logging from info to debug level #20858

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 14 additions & 16 deletions airflow/providers/amazon/aws/hooks/base_aws.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ def _create_basic_session(self, session_kwargs: Dict[str, Any]) -> boto3.session
if self.region_name is None and 'region_name' in self.extra_config:
self.log.info("Retrieving region_name from Connection.extra_config['region_name']")
region_name = self.extra_config["region_name"]
self.log.info(
self.log.debug(
"Creating session with aws_access_key_id=%s region_name=%s",
aws_access_key_id,
region_name,
Expand All @@ -105,7 +105,7 @@ def _create_basic_session(self, session_kwargs: Dict[str, Any]) -> boto3.session

def _create_session_with_assume_role(self, session_kwargs: Dict[str, Any]) -> boto3.session.Session:
assume_role_method = self.extra_config.get('assume_role_method', 'assume_role')
self.log.info("assume_role_method=%s", assume_role_method)
self.log.debug("assume_role_method=%s", assume_role_method)
supported_methods = ['assume_role', 'assume_role_with_saml', 'assume_role_with_web_identity']
if assume_role_method not in supported_methods:
raise NotImplementedError(
Expand Down Expand Up @@ -140,7 +140,7 @@ def _create_session_with_assume_role(self, session_kwargs: Dict[str, Any]) -> bo
return boto3.session.Session(botocore_session=session, **session_kwargs)

def _refresh_credentials(self) -> Dict[str, Any]:
self.log.info('Refreshing credentials')
self.log.debug('Refreshing credentials')
assume_role_method = self.extra_config.get('assume_role_method', 'assume_role')
sts_session = self.basic_session

Expand All @@ -162,7 +162,7 @@ def _refresh_credentials(self) -> Dict[str, Any]:
raise RuntimeError(f'sts_response_http_status={sts_response_http_status}')
credentials = sts_response['Credentials']
expiry_time = credentials.get('Expiration').isoformat()
self.log.info(f'New credentials expiry_time:{expiry_time}')
self.log.debug('New credentials expiry_time: %s', expiry_time)
credentials = {
"access_key": credentials.get("AccessKeyId"),
"secret_key": credentials.get("SecretAccessKey"),
Expand All @@ -178,7 +178,7 @@ def _read_role_arn_from_extra_config(self) -> Optional[str]:
if role_arn is None and aws_account_id is not None and aws_iam_role is not None:
self.log.info("Constructing role_arn from aws_account_id and aws_iam_role")
role_arn = f"arn:aws:iam::{aws_account_id}:role/{aws_iam_role}"
self.log.info("role_arn is %s", role_arn)
self.log.debug("role_arn is %s", role_arn)
return role_arn

def _read_credentials_from_connection(self) -> Tuple[Optional[str], Optional[str]]:
Expand All @@ -199,8 +199,6 @@ def _read_credentials_from_connection(self) -> Tuple[Optional[str], Optional[str
self.extra_config.get("profile"),
)
self.log.info("Credentials retrieved from extra_config['s3_config_file']")
else:
self.log.info("No credentials retrieved from Connection")
Comment thread
uranusjr marked this conversation as resolved.
Outdated
return aws_access_key_id, aws_secret_access_key

def _strip_invalid_session_name_characters(self, role_session_name: str) -> str:
Expand All @@ -211,7 +209,7 @@ def _assume_role(self, sts_client: boto3.client) -> Dict:
if "external_id" in self.extra_config: # Backwards compatibility
assume_role_kwargs["ExternalId"] = self.extra_config.get("external_id")
role_session_name = self._strip_invalid_session_name_characters(f"Airflow_{self.conn.conn_id}")
self.log.info(
self.log.debug(
"Doing sts_client.assume_role to role_arn=%s (role_session_name=%s)",
self.role_arn,
role_session_name,
Expand All @@ -233,7 +231,7 @@ def _assume_role_with_saml(self, sts_client: boto3.client) -> Dict[str, Any]:
'Currently only "http_spegno_auth" is supported, and must be specified.'
)

self.log.info("Doing sts_client.assume_role_with_saml to role_arn=%s", self.role_arn)
self.log.debug("Doing sts_client.assume_role_with_saml to role_arn=%s", self.role_arn)
assume_role_kwargs = self.extra_config.get("assume_role_kwargs", {})
return sts_client.assume_role_with_saml(
RoleArn=self.role_arn,
Expand All @@ -246,7 +244,7 @@ def _get_idp_response(
self, saml_config: Dict[str, Any], auth: requests.auth.AuthBase
) -> requests.models.Response:
idp_url = saml_config["idp_url"]
self.log.info("idp_url= %s", idp_url)
self.log.debug("idp_url= %s", idp_url)

session = requests.Session()

Expand Down Expand Up @@ -303,8 +301,8 @@ def _fetch_saml_assertion_using_http_spegno_auth(self, saml_config: Dict[str, An
'The IDP response contains sensitive information, but log_idp_response is ON (%s).',
log_idp_response,
)
self.log.info('idp_response.content= %s', idp_response.content)
self.log.info('xpath= %s', xpath)
self.log.debug('idp_response.content= %s', idp_response.content)
self.log.debug('xpath= %s', xpath)
# Extract SAML Assertion from the returned HTML / XML
xml = etree.fromstring(idp_response.content)
saml_assertion = xml.xpath(xpath)
Expand Down Expand Up @@ -413,7 +411,7 @@ def _get_credentials(self, region_name: Optional[str]) -> Tuple[boto3.session.Se
session = boto3.session.Session(region_name=region_name)
return session, None

self.log.info("Airflow Connection: aws_conn_id=%s", self.aws_conn_id)
self.log.debug("Airflow Connection: aws_conn_id=%s", self.aws_conn_id)

try:
# Fetch the Airflow connection object
Expand All @@ -423,7 +421,7 @@ def _get_credentials(self, region_name: Optional[str]) -> Tuple[boto3.session.Se

# https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html#botocore.config.Config
if "config_kwargs" in extra_config:
self.log.info(
self.log.debug(
"Retrieving config_kwargs from Connection.extra_config['config_kwargs']: %s",
extra_config["config_kwargs"],
)
Expand All @@ -437,10 +435,10 @@ def _get_credentials(self, region_name: Optional[str]) -> Tuple[boto3.session.Se

except AirflowException:
self.log.warning("Unable to use Airflow Connection for credentials.")
self.log.info("Fallback on boto3 credential strategy")
self.log.debug("Fallback on boto3 credential strategy")
# http://boto3.readthedocs.io/en/latest/guide/configuration.html

self.log.info(
self.log.debug(
"Creating session using boto3 credential strategy region_name=%s",
region_name,
)
Expand Down