fix: requests error handling (#82)

- added locally built docker build-image
- fixed retries for 5xx errors
- fixed various leftover build issues
LOG-11814

Author: dkhokhlov

Dockerfile

@@ -0,0 +1,7 @@
+FROM condaforge/miniforge3:4.12.0-0
+
+RUN conda install -y gcc pip poetry=1.1.7 git
+RUN mkdir /workdir && chmod 777 /workdir
+RUN git config --global --add safe.directory /workdir
+WORKDIR /workdir
+

Jenkinsfile

@@ -5,7 +5,12 @@ def DEFAULT_BRANCH = 'master'
 def CURRENT_BRANCH = [env.CHANGE_BRANCH, env.BRANCH_NAME]?.find{branch -> branch != null}
 
 pipeline {
-  agent none
+  agent {
+    node {
+      label 'ec2-fleet'
+      customWorkspace "${PROJECT_NAME}-${BUILD_NUMBER}"
+    }
+  }
 
   options {
     timestamps()
@@ -18,12 +23,6 @@ pipeline {
 
   stages {
     stage('Test') {
-      agent {
-        node {
-          label 'ec2-fleet'
-          customWorkspace "${PROJECT_NAME}-${BUILD_NUMBER}"
-        }
-      }
 
       steps {
         sh 'make install lint test'
@@ -45,12 +44,6 @@ pipeline {
     }
 
     stage('Release') {
-      agent {
-        node {
-          label 'ec2-fleet'
-          customWorkspace "${PROJECT_NAME}-${BUILD_NUMBER}"
-        }
-      }
 
       stages {
         stage('dry run') {

Makefile

@@ -7,7 +7,7 @@ include .config.mk
 DOCKER = docker
 DOCKER_RUN := $(DOCKER) run --rm -i
 WORKDIR :=/workdir
-DOCKER_COMMAND := $(DOCKER_RUN) -v $(PWD):$(WORKDIR):Z -w $(WORKDIR) \
+DOCKER_COMMAND := $(DOCKER_RUN) -u "$(shell id -u)":"$(shell id -g)" -v $(PWD):$(WORKDIR):Z -w $(WORKDIR) \
 	-e XDG_CONFIG_HOME=$(WORKDIR) \
 	-e XDG_CACHE_HOME=$(WORKDIR) \
 	-e POETRY_CACHE_DIR=$(WORKDIR)/.cache \
@@ -21,13 +21,19 @@ DOCKER_COMMAND := $(DOCKER_RUN) -v $(PWD):$(WORKDIR):Z -w $(WORKDIR) \
 	-e GIT_AUTHOR_EMAIL \
 	-e GIT_COMMITTER_NAME \
 	-e GIT_COMMITTER_EMAIL \
-	us.gcr.io/logdna-k8s/python:3.7-ci
+	logdna-poetry:local
+
 
 POETRY_COMMAND := $(DOCKER_COMMAND) poetry
 
 # Exports the variables for shell use
 export
 
+# build image
+.PHONY:build-image
+build-image:
+	DOCKER_BUILDKIT=1 $(DOCKER) build -t logdna-poetry:local .
+
 # This helper function makes debugging much easier.
 .PHONY:debug-%
 debug-%: ## Debug a variable by calling `make debug-VARIABLE`
@@ -39,30 +45,31 @@ help: ## Show this help, includes list of all actions.
 	@awk 'BEGIN {FS = ":.*?## "}; /^.+: .*?## / && !/awk/ {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' ${MAKEFILE_LIST}
 
 .PHONY:run
-run: ## purge build time artifacts
+run: install ## purge build time artifacts
 	$(DOCKER_COMMAND) bash
+
 .PHONY:clean
 clean: ## purge build time artifacts
 	rm -rf dist/ build/ coverage/ pypoetry/ pip/ **/__pycache__/ .pytest_cache/ .cache .coverage
 
 .PHONY:changelog
-changelog: ## print the next version of the change log to stdout
+changelog: install ## print the next version of the change log to stdout
 	$(POETRY_COMMAND) run semantic-release changelog --unreleased
 
 .PHONY:install
-install: ## install development and build time dependencies
-	$(POETRY_COMMAND) install --no-interaction -vvv
+install: build-image ## install development and build time dependencies
+	$(POETRY_COMMAND) install --no-interaction
 
 .PHONY:lint
-lint: ## run lint rules and print error report
+lint: install ## run lint rules and print error report
 	$(POETRY_COMMAND) run task lint
 
 .PHONY:lint-fix
-lint-fix:## attempt to auto fix linting error and report remaining errors
+lint-fix: install ## attempt to auto fix linting error and report remaining errors
 	$(POETRY_COMMAND) run task lint:fix
 
 .PHONY:package
-package: ## Generate a python sdist and wheel
+package: install ## Generate a python sdist and wheel
 	$(POETRY_COMMAND) build
 
 .PHONY:release
@@ -71,7 +78,6 @@ release: clean install fetch-tags ## run semantic release build and publish resu
 
 .PHONY: fetch-tags
 fetch-tags:  ## workaround for jenkins repo cloning behavior
-	git config remote.origin.url "https://logdnabot:${GH_TOKEN}@github.com/logdna/python"
 	git fetch origin --tags
 
 .PHONY:release-dry
@@ -91,6 +97,5 @@ release-major: clean install                    ## run semantic release build an
 	$(POETRY_COMMAND) run semantic-release publish --major
 
 .PHONY:test
-test:                            ## run project test suite
+test: install ## run project test suite
 	$(POETRY_COMMAND) run task test
-

logdna/logdna.py

@@ -177,7 +177,13 @@ def try_request(self):
             self.close_flusher()
             self.exception_flag = True
 
-    def send_request(self, data):
+    def send_request(self, data): # noqa: max-complexity: 13
+        """
+            Send log data to LogDNA server
+        Returns:
+            True  - discard flush buffer
+            False - retry, keep flush buffer
+        """
         try:
             response = requests.post(url=self.url,
                                      json=data,
@@ -190,38 +196,84 @@ def send_request(self, data):
                                          'now': int(time.time() * 1000)
                                      },
                                      stream=True,
+                                     allow_redirects=True,
                                      timeout=self.request_timeout,
                                      headers={'user-agent': self.user_agent})
 
-            response.raise_for_status()
             status_code = response.status_code
-            if status_code in [401, 403]:
-                self.internalLogger.debug(
-                    'Please provide a valid ingestion key.' +
-                    ' Discarding flush buffer')
-                return True
-
+            '''
+                response code:
+                    1XX                       unexpected status
+                    200                       expected status, OK
+                    2XX                       unexpected status
+                    301 302 303               unexpected status,
+                                              per "allow_redirects=True"
+                    3XX                       unexpected status
+                    401, 403                  expected client error,
+                                              invalid ingestion key
+                    4XX                       unexpected client error
+                    500 502 503 507           expected server error, transient
+                    5XX                       unexpected server error
+                handling:
+                    expected status           discard flush buffer
+                    unexpected status         log + discard flush buffer
+                    expected client error     log + discard flush buffer
+                    unexpected client error   log + discard flush buffer
+                    expected server error     log + retry
+                    unexpected server error   log + discard flush buffer
+            '''
             if status_code == 200:
-                return True
+                return True  # discard
 
-            if status_code in [400, 500, 504]:
-                self.internalLogger.debug('The request failed %s. Retrying...',
-                                          response.reason)
-                return True
+            if isinstance(response.reason, bytes):
+                # We attempt to decode utf-8 first because some servers
+                # choose to localize their reason strings. If the string
+                # isn't utf-8, we fall back to iso-8859-1 for all other
+                # encodings. (See PR #3538)
+                try:
+                    reason = response.reason.decode('utf-8')
+                except UnicodeDecodeError:
+                    reason = response.reason.decode('iso-8859-1')
             else:
+                reason = response.reason
+
+            if 200 < status_code <= 399:
+                self.internalLogger.debug('Unexpected response: %s. ' +
+                                          'Discarding flush buffer',
+                                          reason)
+                return True  # discard
+
+            if status_code in [401, 403]:
                 self.internalLogger.debug(
-                    'The request failed: %s. Retrying...', response.reason)
+                    'Please provide a valid ingestion key. ' +
+                    'Discarding flush buffer')
+                return True  # discard
+
+            if 400 <= status_code <= 499:
+                self.internalLogger.debug('Client Error: %s. ' +
+                                          'Discarding flush buffer',
+                                          reason)
+                return True  # discard
+
+            if status_code in [500, 502, 503, 507]:
+                self.internalLogger.debug('Server Error: %s. Retrying...',
+                                          reason)
+                return False  # retry
+
+            self.internalLogger.debug('The request failed: %s.' +
+                                      'Discarding flush buffer',
+                                      reason)
 
         except requests.exceptions.Timeout as timeout:
-            self.internalLogger.debug('Timeout error occurred %s. Retrying...',
+            self.internalLogger.debug('Timeout Error: %s. Retrying...',
                                       timeout)
+            return False  # retry
 
         except requests.exceptions.RequestException as exception:
             self.internalLogger.debug(
                 'Error sending logs %s. Discarding flush buffer', exception)
-            return True
 
-        return False
+        return True  # discard
 
     def emit(self, record):
         msg = self.format(record)