FINERACT-428 Parallelization of Jobs : Parallelizing and paging of recalculate interest for loans

fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/domain/JobParameter.java

@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.jobs.domain;
+
+import org.apache.fineract.infrastructure.core.domain.AbstractPersistableCustom;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "job_parameters")
+public class JobParameter extends AbstractPersistableCustom<Long> {
+
+    @Column(name = "job_id", nullable = false)
+    private Long jobId;
+
+    @Column(name = "parameter_name",nullable = true)
+    private String parameterName;
+
+    @Column(name = "parameter_value",nullable = true)
+    private String parameterValue;
+
+
+    public JobParameter(final Long jobId, final String parameterName, final String parameterValue) {
+        this.jobId = jobId;
+        this.parameterName = parameterName;
+        this.parameterValue = parameterValue;
+    }
+
+    public static JobParameter getInstance(final Long jobId, final String parameterName, final String parameterValue){
+        return new JobParameter(jobId,parameterName,parameterValue);
+    }
+
+    public Long getJobId() {
+        return jobId;
+    }
+
+    public void setJobId(final Long jobId) {
+        this.jobId = jobId;
+    }
+
+    public String getParameterName() {
+        return parameterName;
+    }
+
+    public void setParameterName(final String parameterName) {
+        this.parameterName = parameterName;
+    }
+
+    public String getParameterValue() {
+        return parameterValue;
+    }
+
+    public void setParameterValue(final String parameterValue) {
+        this.parameterValue = parameterValue;
+    }
+
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!obj.getClass().equals(getClass())) return false;
+        JobParameter jobParameter = (JobParameter) obj;
+        return jobParameter.getJobId().equals(this.getJobId())
+                && jobParameter.getParameterName().equals(this.getParameterName())
+                && jobParameter.getParameterValue().equals(this.getParameterValue());
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/domain/JobParameterRepository.java

@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.jobs.domain;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
+
+import java.util.List;
+
+public interface JobParameterRepository extends JpaRepository<JobParameter, Long>, JpaSpecificationExecutor<JobParameter> {
+
+    @Query("select jobParameter from JobParameter jobParameter where jobParameter.jobId=:jobId")
+    List<JobParameter> findJobParametersByJobId(@Param("jobId") Long jobId);
+}

fineract-provider/src/main/java/org/apache/fineract/infrastructure/jobs/service/JobRegisterServiceImpl.java

@@ -18,12 +18,7 @@
  */
 package org.apache.fineract.infrastructure.jobs.service;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Properties;
-import java.util.TimeZone;
+import java.util.*;
 
 import javax.annotation.PostConstruct;
 
@@ -32,6 +27,8 @@
 import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
 import org.apache.fineract.infrastructure.jobs.annotation.CronMethodParser;
 import org.apache.fineract.infrastructure.jobs.annotation.CronMethodParser.ClassMethodNamesPair;
+import org.apache.fineract.infrastructure.jobs.domain.JobParameter;
+import org.apache.fineract.infrastructure.jobs.domain.JobParameterRepository;
 import org.apache.fineract.infrastructure.jobs.domain.ScheduledJobDetail;
 import org.apache.fineract.infrastructure.jobs.domain.SchedulerDetail;
 import org.apache.fineract.infrastructure.jobs.exception.JobNotFoundException;
@@ -77,6 +74,7 @@ public class JobRegisterServiceImpl implements JobRegisterService, ApplicationLi
     private SchedulerJobListener schedulerJobListener;
     private SchedulerStopListener schedulerStopListener;
     private SchedulerTriggerListener globalSchedulerTriggerListener;
+    private JobParameterRepository jobParameterRepository;
 
     private final HashMap<String, Scheduler> schedulers = new HashMap<>(4);
 
@@ -110,6 +108,11 @@ public void setGlobalTriggerListener(SchedulerTriggerListener globalTriggerListe
         this.globalSchedulerTriggerListener = globalTriggerListener;
     }
 
+    @Autowired
+    public  void setJobParameterRepository(JobParameterRepository jobParameterRepository){
+        this.jobParameterRepository=jobParameterRepository;
+    }
+
     @PostConstruct
     public void loadAllJobs() {
         final List<FineractPlatformTenant> allTenants = this.tenantDetailsService.findAllTenants();
@@ -345,6 +348,7 @@ private JobDetail createJobDetail(final ScheduledJobDetail scheduledJobDetail) t
         final Object targetObject = getBeanObject(Class.forName(jobDetails.className));
         final MethodInvokingJobDetailFactoryBean jobDetailFactoryBean = new MethodInvokingJobDetailFactoryBean();
         jobDetailFactoryBean.setName(scheduledJobDetail.getJobName() + "JobDetail" + tenant.getId());
+        jobDetailFactoryBean.setArguments(new Object[]{getJobParameter(scheduledJobDetail)});
         jobDetailFactoryBean.setTargetObject(targetObject);
         jobDetailFactoryBean.setTargetMethod(jobDetails.methodName);
         jobDetailFactoryBean.setGroup(scheduledJobDetail.getGroupName());
@@ -353,6 +357,14 @@ private JobDetail createJobDetail(final ScheduledJobDetail scheduledJobDetail) t
         return jobDetailFactoryBean.getObject();
     }
 
+    public Map<String,String> getJobParameter(ScheduledJobDetail scheduledJobDetail){
+        List<JobParameter> jobParameterList= jobParameterRepository.findJobParametersByJobId(scheduledJobDetail.getId());
+        Map<String,String> jobParameterMap=new HashMap<>();
+        for (JobParameter jobparameter:jobParameterList) {
+            jobParameterMap.put(jobparameter.getParameterName(),jobparameter.getParameterValue());
+        }
+        return  jobParameterMap;
+    }
     private Object getBeanObject(final Class<?> classType) throws ClassNotFoundException {
         final List<Class<?>> typesList = new ArrayList<>();
         final Class<?>[] interfaceType = classType.getInterfaces();
@@ -418,4 +430,4 @@ private JobKey constructJobKey(final String Key) {
         final JobKey JobKey = new JobKey(keyParams[0], keyParams[1]);
         return JobKey;
     }
-}
+}

fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanApplicationWriteServiceImpl.java

@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.portfolio.loanaccount.service;
+
+import org.apache.fineract.infrastructure.core.service.RoutingDataSourceServiceFactory;
+import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+
+@Service
+public class LoanApplicationWriteServiceImpl implements LoanApplicationWriteService {
+
+    private final static Logger logger = LoggerFactory.getLogger(LoanApplicationWriteServiceImpl.class);
+
+    private final RoutingDataSourceServiceFactory dataSourceServiceFactory;
+
+    @Autowired
+    public LoanApplicationWriteServiceImpl(final RoutingDataSourceServiceFactory dataSourceServiceFactory){
+        this.dataSourceServiceFactory = dataSourceServiceFactory;
+    }
+
+    @Override
+    public void updateLoanSummaryDetails(Integer limit,Integer lastLoanId,String officeHierachy) {
+        final JdbcTemplate jdbcTemplate = new JdbcTemplate(this.dataSourceServiceFactory.determineDataSourceService().retrieveDataSource());
+
+        final StringBuilder updateSqlBuilder = new StringBuilder(900);
+        updateSqlBuilder.append("update m_loan ");
+        updateSqlBuilder.append("join (");
+        updateSqlBuilder.append("SELECT ml.id AS loanId,");
+        updateSqlBuilder.append("SUM(mr.principal_amount) as principal_disbursed_derived, ");
+        updateSqlBuilder.append("SUM(IFNULL(mr.principal_completed_derived,0)) as principal_repaid_derived, ");
+        updateSqlBuilder.append("SUM(IFNULL(mr.principal_writtenoff_derived,0)) as principal_writtenoff_derived,");
+        updateSqlBuilder.append("SUM(IFNULL(mr.interest_amount,0)) as interest_charged_derived,");
+        updateSqlBuilder.append("SUM(IFNULL(mr.interest_completed_derived,0)) as interest_repaid_derived,");
+        updateSqlBuilder.append("SUM(IFNULL(mr.interest_waived_derived,0)) as interest_waived_derived,");
+        updateSqlBuilder.append("SUM(IFNULL(mr.interest_writtenoff_derived,0)) as interest_writtenoff_derived,");
+        updateSqlBuilder
+                .append("SUM(IFNULL(mr.fee_charges_amount,0)) + IFNULL((select SUM(lc.amount) from  m_loan_charge lc where lc.loan_id=ml.id and lc.is_active=1 and lc.charge_time_enum=1),0) as fee_charges_charged_derived,");
+        updateSqlBuilder
+                .append("SUM(IFNULL(mr.fee_charges_completed_derived,0)) + IFNULL((select SUM(lc.amount_paid_derived) from  m_loan_charge lc where lc.loan_id=ml.id and lc.is_active=1 and lc.charge_time_enum=1),0) as fee_charges_repaid_derived,");
+        updateSqlBuilder.append("SUM(IFNULL(mr.fee_charges_waived_derived,0)) as fee_charges_waived_derived,");
+        updateSqlBuilder.append("SUM(IFNULL(mr.fee_charges_writtenoff_derived,0)) as fee_charges_writtenoff_derived,");
+        updateSqlBuilder.append("SUM(IFNULL(mr.penalty_charges_amount,0)) as penalty_charges_charged_derived,");
+        updateSqlBuilder.append("SUM(IFNULL(mr.penalty_charges_completed_derived,0)) as penalty_charges_repaid_derived,");
+        updateSqlBuilder.append("SUM(IFNULL(mr.penalty_charges_waived_derived,0)) as penalty_charges_waived_derived,");
+        updateSqlBuilder.append("SUM(IFNULL(mr.penalty_charges_writtenoff_derived,0)) as penalty_charges_writtenoff_derived ");
+        updateSqlBuilder.append(" FROM m_loan ml ");
+        updateSqlBuilder.append("INNER JOIN m_loan_repayment_schedule mr on mr.loan_id = ml.id ");
+        updateSqlBuilder.append("INNER JOIN m_client mc on mc.id=ml.client_id ");
+        updateSqlBuilder.append("INNER JOIN m_office o on mc.office_id = o.id ");
+        updateSqlBuilder.append("WHERE ml.disbursedon_date is not null and o.hierarchy like ? and ml.id < ? ");
+        updateSqlBuilder.append("GROUP BY ml.id ");
+        updateSqlBuilder.append("limit ? ");
+        updateSqlBuilder.append(") x on x.loanId = m_loan.id ");
+
+        updateSqlBuilder.append("SET m_loan.principal_disbursed_derived = x.principal_disbursed_derived,");
+        updateSqlBuilder.append("m_loan.principal_repaid_derived = x.principal_repaid_derived,");
+        updateSqlBuilder.append("m_loan.principal_writtenoff_derived = x.principal_writtenoff_derived,");
+        updateSqlBuilder
+                .append("m_loan.principal_outstanding_derived = (x.principal_disbursed_derived - (x.principal_repaid_derived + x.principal_writtenoff_derived)),");
+        updateSqlBuilder.append("m_loan.interest_charged_derived = x.interest_charged_derived,");
+        updateSqlBuilder.append("m_loan.interest_repaid_derived = x.interest_repaid_derived,");
+        updateSqlBuilder.append("m_loan.interest_waived_derived = x.interest_waived_derived,");
+        updateSqlBuilder.append("m_loan.interest_writtenoff_derived = x.interest_writtenoff_derived,");
+        updateSqlBuilder
+                .append("m_loan.interest_outstanding_derived = (x.interest_charged_derived - (x.interest_repaid_derived + x.interest_waived_derived + x.interest_writtenoff_derived)),");
+        updateSqlBuilder.append("m_loan.fee_charges_charged_derived = x.fee_charges_charged_derived,");
+        updateSqlBuilder.append("m_loan.fee_charges_repaid_derived = x.fee_charges_repaid_derived,");
+        updateSqlBuilder.append("m_loan.fee_charges_waived_derived = x.fee_charges_waived_derived,");
+        updateSqlBuilder.append("m_loan.fee_charges_writtenoff_derived = x.fee_charges_writtenoff_derived,");
+        updateSqlBuilder
+                .append("m_loan.fee_charges_outstanding_derived = (x.fee_charges_charged_derived - (x.fee_charges_repaid_derived + x.fee_charges_waived_derived + x.fee_charges_writtenoff_derived)),");
+        updateSqlBuilder.append("m_loan.penalty_charges_charged_derived = x.penalty_charges_charged_derived,");
+        updateSqlBuilder.append("m_loan.penalty_charges_repaid_derived = x.penalty_charges_repaid_derived,");
+        updateSqlBuilder.append("m_loan.penalty_charges_waived_derived = x.penalty_charges_waived_derived,");
+        updateSqlBuilder.append("m_loan.penalty_charges_writtenoff_derived = x.penalty_charges_writtenoff_derived,");
+        updateSqlBuilder
+                .append("m_loan.penalty_charges_outstanding_derived = (x.penalty_charges_charged_derived - (x.penalty_charges_repaid_derived + x.penalty_charges_waived_derived + x.penalty_charges_writtenoff_derived)),");
+        updateSqlBuilder
+                .append("m_loan.total_expected_repayment_derived = (x.principal_disbursed_derived + x.interest_charged_derived + x.fee_charges_charged_derived + x.penalty_charges_charged_derived),");
+        updateSqlBuilder
+                .append("m_loan.total_repayment_derived = (x.principal_repaid_derived + x.interest_repaid_derived + x.fee_charges_repaid_derived + x.penalty_charges_repaid_derived),");
+        updateSqlBuilder
+                .append("m_loan.total_expected_costofloan_derived = (x.interest_charged_derived + x.fee_charges_charged_derived + x.penalty_charges_charged_derived),");
+        updateSqlBuilder
+                .append("m_loan.total_costofloan_derived = (x.interest_repaid_derived + x.fee_charges_repaid_derived + x.penalty_charges_repaid_derived),");
+        updateSqlBuilder
+                .append("m_loan.total_waived_derived = (x.interest_waived_derived + x.fee_charges_waived_derived + x.penalty_charges_waived_derived),");
+        updateSqlBuilder
+                .append("m_loan.total_writtenoff_derived = (x.interest_writtenoff_derived +  x.fee_charges_writtenoff_derived + x.penalty_charges_writtenoff_derived),");
+        updateSqlBuilder.append("m_loan.total_outstanding_derived=");
+        updateSqlBuilder.append(" (x.principal_disbursed_derived - (x.principal_repaid_derived + x.principal_writtenoff_derived)) + ");
+        updateSqlBuilder
+                .append(" (x.interest_charged_derived - (x.interest_repaid_derived + x.interest_waived_derived + x.interest_writtenoff_derived)) +");
+        updateSqlBuilder
+                .append(" (x.fee_charges_charged_derived - (x.fee_charges_repaid_derived + x.fee_charges_waived_derived + x.fee_charges_writtenoff_derived)) +");
+        updateSqlBuilder
+                .append(" (x.penalty_charges_charged_derived - (x.penalty_charges_repaid_derived + x.penalty_charges_waived_derived + x.penalty_charges_writtenoff_derived))");
+
+        final int result = jdbcTemplate.update(updateSqlBuilder.toString(), new Object[] {officeHierachy,lastLoanId,limit});
+
+        logger.info(ThreadLocalContextUtil.getTenant().getName() + ": Results affected by update: " + result);
+
+    }
+}

fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanReadPlatformService.java

@@ -21,6 +21,7 @@
 import java.math.BigDecimal;
 import java.util.Collection;
 import java.util.Date;
+import java.util.List;
 
 import org.apache.fineract.infrastructure.core.service.Page;
 import org.apache.fineract.infrastructure.core.service.SearchParameters;
@@ -45,7 +46,7 @@ public interface LoanReadPlatformService {
     LoanAccountData retrieveOne(Long loanId);
 
     LoanScheduleData retrieveRepaymentSchedule(Long loanId, RepaymentScheduleRelatedLoanData repaymentScheduleRelatedData,
-            Collection<DisbursementData> disbursementData, boolean isInterestRecalculationEnabled, BigDecimal totalPaidFeeCharges);
+                                               Collection<DisbursementData> disbursementData, boolean isInterestRecalculationEnabled, BigDecimal totalPaidFeeCharges);
 
     Collection<LoanTransactionData> retrieveLoanTransactions(Long loanId);
 
@@ -84,7 +85,7 @@ LoanScheduleData retrieveRepaymentSchedule(Long loanId, RepaymentScheduleRelated
     /*
      * musoni-specific at present - will find overdue scheduled installments
      * that have a special 'overdue charge' associated with the loan product.
-     * 
+     *
      * The 'overdue-charge' is only ever applied once to an installment and as a
      * result overdue installments with this charge already applied are not
      * returned.
@@ -111,6 +112,8 @@ LoanScheduleData retrieveRepaymentSchedule(Long loanId, RepaymentScheduleRelated
 
     Collection<Long> fetchLoansForInterestRecalculation();
 
+    List<Long> fetchLoansForInterestRecalculation(Integer pageSize, Long maxLoanIdInList, String officeHierarchy);
+
     LoanTransactionData retrieveLoanPrePaymentTemplate(Long loanId, LocalDate onDate);
 
     Collection<LoanTransactionData> retrieveWaiverLoanTransactions(Long loanId);
@@ -124,14 +127,16 @@ LoanScheduleData retrieveRepaymentSchedule(Long loanId, RepaymentScheduleRelated
     PaidInAdvanceData retrieveTotalPaidInAdvance(Long loanId);
 
     LoanTransactionData retrieveRefundByCashTemplate(Long loanId);
-    
+
     Collection<InterestRatePeriodData> retrieveLoanInterestRatePeriodData(LoanAccountData loan);
 
     Collection<Long> retrieveLoanIdsWithPendingIncomePostingTransactions();
 
     LoanTransactionData retrieveLoanForeclosureTemplate(final Long loanId, final LocalDate transactionDate);
 
-	LoanAccountData retrieveLoanByLoanAccount(String loanAccountNumber);
-
-	Long retrieveLoanIdByAccountNumber(String loanAccountNumber);
+    LoanAccountData retrieveLoanByLoanAccount(String loanAccountNumber);
+
+    Long retrieveLoanIdByAccountNumber(String loanAccountNumber);
+
+    Integer retrieveNumberOfActiveLoans();
 }