From e0d3c283e3689dab9466294dfef4f877ebe03878 Mon Sep 17 00:00:00 2001 From: jozefbakus Date: Mon, 22 Jan 2024 16:33:56 +0100 Subject: [PATCH 1/3] #818: Regularly execute archive_dag_instances function --- .../resources/db_scripts/db_script_latest.sql | 8 +++ .../db_scripts/housekeeping/cleanUpRuns.sh | 64 +++++++++++++++++++ .../db_scripts/liquibase/db.changelog.yml | 3 + .../v0.5.20.add-housekeeping-lock-table.sql | 23 +++++++ .../v0.5.20.add-housekeeping-lock-table.yml | 25 ++++++++ 5 files changed, 123 insertions(+) create mode 100755 src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh create mode 100644 src/main/resources/db_scripts/liquibase/v0.5.20.add-housekeeping-lock-table.sql create mode 100644 src/main/resources/db_scripts/liquibase/v0.5.20.add-housekeeping-lock-table.yml diff --git a/src/main/resources/db_scripts/db_script_latest.sql b/src/main/resources/db_scripts/db_script_latest.sql index d3cd532f5..4dc4742e5 100644 --- a/src/main/resources/db_scripts/db_script_latest.sql +++ b/src/main/resources/db_scripts/db_script_latest.sql @@ -363,3 +363,11 @@ BEGIN END; $$ LANGUAGE plpgsql; +create table if not exists housekeepinglock +( + locked boolean not null, + started_at timestamp +); + +insert into "housekeepinglock" ("locked", "started_at") +values (false, null); diff --git a/src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh b/src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh new file mode 100755 index 000000000..3be43cb72 --- /dev/null +++ b/src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh @@ -0,0 +1,64 @@ +#! /bin/bash + +RECIPIENTS=$1 +RETENTION_DAYS=$2 +ENV=$3 + +if [[ -n "$RECIPIENTS" && -n "$RETENTION_DAYS" && -n "$ENV" ]]; then + echo "Starting runs clean up with retention days: $RETENTION_DAYS, env: $ENV and recipients: $RECIPIENTS" +else + echo "Incorrect input. Exiting the program." + exit 1 +fi + +config_file="../../application.properties" + +db_url=$(grep '^db.url=' "$config_file" | awk -F'jdbc:postgresql://' '{print $2}' | awk -F'\\?' '{print $1}') +db_user=$(grep '^db.user=' "$config_file" | awk -F'=' '{print $2}') +db_password=$(grep '^db.password=' "$config_file" | awk -F'=' '{print $2}') +connection_url="postgresql://$db_user:$db_password@$db_url" + +obtain_lock_response=$(psql -X $connection_url -c "UPDATE housekeepinglock SET locked = 'true', started_at = now() WHERE locked = false AND started_at is null;" 2>&1) + +if [[ $obtain_lock_response == "UPDATE 1" ]]; then + echo "Exclusive lock for running runs clean up job obtained." + + archive_dag_instances_response=$(psql -X $connection_url -c "CALL archive_dag_instances(i_to_ts => (now() - '$RETENTION_DAYS days'::interval)::timestamp without time zone);" 2>&1) + release_lock_response=$(psql -X $connection_url -c "UPDATE housekeepinglock SET locked = 'false', started_at = null;" 2>&1) + + message="" + subject="" + if [[ $archive_dag_instances_response == *"ERROR"* ]]; then + subject="Hyperdrive Notifications - $ENV - Runs clean up succeeded" + message+="Runs clean up job failed with following output:" + else + subject="Hyperdrive Notifications - $ENV - Runs clean up failed!" + message+="Runs clean up job succeeded with following output:" + fi + message+="\n$archive_dag_instances_response" + + if [[ $release_lock_response == "UPDATE 1" ]]; then + message+="\n\nLock for running runs clean job was successfully released" + else + message+="\n\nLock for running runs clean job was not released!" + fi + echo -e "$message" + echo -e "$message" | mailx -s "$subject" -r "noreply@absa.co.za" "$RECIPIENTS" + +else + obtain_lock_time_response=$(psql -X $connection_url -c "SELECT started_at FROM housekeepinglock;" 2>&1) + + timestamp_string=$(echo "$obtain_lock_time_response" | grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}') + timestamp_unix=$(date -jf "%Y-%m-%d %H:%M:%S" "$timestamp_string" +%s) + current_unix=$(date +%s) + time_difference=$((current_unix - timestamp_unix)) + days_difference=$((time_difference / 86400)) + + if [ "$days_difference" -gt "29" ]; then + message="Runs clean up has been running for more than 30 days." + echo -e "$message" + echo -e "$message" | mailx -s "Hyperdrive Notifications - $ENV - Runs clean up locked" -r "noreply@absa.co.za" "$RECIPIENTS" + else + echo "Runs clean up is running on different machine." + fi +fi diff --git a/src/main/resources/db_scripts/liquibase/db.changelog.yml b/src/main/resources/db_scripts/liquibase/db.changelog.yml index 764544244..d14cb2e75 100644 --- a/src/main/resources/db_scripts/liquibase/db.changelog.yml +++ b/src/main/resources/db_scripts/liquibase/db.changelog.yml @@ -98,3 +98,6 @@ databaseChangeLog: - include: relativeToChangelogFile: true file: v0.5.20.add-diagnostics-field.yml + - include: + relativeToChangelogFile: true + file: v0.5.20.add-housekeeping-lock-table.yml diff --git a/src/main/resources/db_scripts/liquibase/v0.5.20.add-housekeeping-lock-table.sql b/src/main/resources/db_scripts/liquibase/v0.5.20.add-housekeeping-lock-table.sql new file mode 100644 index 000000000..e508d6cd0 --- /dev/null +++ b/src/main/resources/db_scripts/liquibase/v0.5.20.add-housekeeping-lock-table.sql @@ -0,0 +1,23 @@ +/* + * Copyright 2018 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +create table if not exists housekeepinglock +( + locked boolean not null, + started_at timestamp +); + +insert into "housekeepinglock" ("locked", "started_at") +values (false, null); \ No newline at end of file diff --git a/src/main/resources/db_scripts/liquibase/v0.5.20.add-housekeeping-lock-table.yml b/src/main/resources/db_scripts/liquibase/v0.5.20.add-housekeeping-lock-table.yml new file mode 100644 index 000000000..40851a185 --- /dev/null +++ b/src/main/resources/db_scripts/liquibase/v0.5.20.add-housekeeping-lock-table.yml @@ -0,0 +1,25 @@ +# +# Copyright 2018 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +databaseChangeLog: + - changeSet: + id: v0.5.20.add-housekeeping-lock-table + logicalFilePath: v0.5.20.add-housekeeping-lock-table + author: HyperdriveDevTeam@absa.africa + context: default + changes: + - sqlFile: + relativeToChangelogFile: true + path: v0.5.20.add-housekeeping-lock-table.sql From af7ef81d56ba97538919404fcfb7d0a37dc21fa7 Mon Sep 17 00:00:00 2001 From: jozefbakus Date: Mon, 22 Jan 2024 16:47:15 +0100 Subject: [PATCH 2/3] License --- .../db_scripts/housekeeping/cleanUpRuns.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh b/src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh index 3be43cb72..f712c279f 100755 --- a/src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh +++ b/src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh @@ -1,4 +1,18 @@ #! /bin/bash +# +# Copyright 2018 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# RECIPIENTS=$1 RETENTION_DAYS=$2 From 363a255aba0fc581bc2f6d19500f196dd43bcf5a Mon Sep 17 00:00:00 2001 From: jozefbakus Date: Wed, 24 Jan 2024 12:08:58 +0100 Subject: [PATCH 3/3] PR Fixes --- .../db_scripts/housekeeping/cleanUpRuns.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh b/src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh index f712c279f..01d9e4bcf 100755 --- a/src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh +++ b/src/main/resources/db_scripts/housekeeping/cleanUpRuns.sh @@ -17,6 +17,7 @@ RECIPIENTS=$1 RETENTION_DAYS=$2 ENV=$3 +CONFIG_FILE_LOCATION=${4:-"../../application.properties"} if [[ -n "$RECIPIENTS" && -n "$RETENTION_DAYS" && -n "$ENV" ]]; then echo "Starting runs clean up with retention days: $RETENTION_DAYS, env: $ENV and recipients: $RECIPIENTS" @@ -25,11 +26,9 @@ else exit 1 fi -config_file="../../application.properties" - -db_url=$(grep '^db.url=' "$config_file" | awk -F'jdbc:postgresql://' '{print $2}' | awk -F'\\?' '{print $1}') -db_user=$(grep '^db.user=' "$config_file" | awk -F'=' '{print $2}') -db_password=$(grep '^db.password=' "$config_file" | awk -F'=' '{print $2}') +db_url=$(grep '^db.url=' "$CONFIG_FILE_LOCATION" | awk -F'jdbc:postgresql://' '{print $2}' | awk -F'\\?' '{print $1}') +db_user=$(grep '^db.user=' "$CONFIG_FILE_LOCATION" | awk -F'=' '{print $2}') +db_password=$(grep '^db.password=' "$CONFIG_FILE_LOCATION" | awk -F'=' '{print $2}') connection_url="postgresql://$db_user:$db_password@$db_url" obtain_lock_response=$(psql -X $connection_url -c "UPDATE housekeepinglock SET locked = 'true', started_at = now() WHERE locked = false AND started_at is null;" 2>&1) @@ -43,10 +42,10 @@ if [[ $obtain_lock_response == "UPDATE 1" ]]; then message="" subject="" if [[ $archive_dag_instances_response == *"ERROR"* ]]; then - subject="Hyperdrive Notifications - $ENV - Runs clean up succeeded" + subject="Hyperdrive Notifications - $ENV - Runs clean up failed!" message+="Runs clean up job failed with following output:" else - subject="Hyperdrive Notifications - $ENV - Runs clean up failed!" + subject="Hyperdrive Notifications - $ENV - Runs clean up succeeded" message+="Runs clean up job succeeded with following output:" fi message+="\n$archive_dag_instances_response" @@ -68,11 +67,11 @@ else time_difference=$((current_unix - timestamp_unix)) days_difference=$((time_difference / 86400)) - if [ "$days_difference" -gt "29" ]; then - message="Runs clean up has been running for more than 30 days." + if [ "$days_difference" -gt "2" ]; then + message="Runs clean up has been running for more than 3 days." echo -e "$message" echo -e "$message" | mailx -s "Hyperdrive Notifications - $ENV - Runs clean up locked" -r "noreply@absa.co.za" "$RECIPIENTS" else - echo "Runs clean up is running on different machine." + echo "Could not execute runs clean up job because there is another instance currently running for ($((time_difference / 60)) minutes)." fi fi