[salesforce] Order EventLogFile queries by the cursor field (CreatedDate)

packages/salesforce/_dev/deploy/docker/files/config.yml

@@ -10,7 +10,7 @@ rules:
   - path: /services/data/v56.0/query/
     methods: ["GET"]
     query_params:
-      q: ["SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' ORDER BY LogDate ASC NULLS FIRST"]
+      q: ["SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Login' ORDER BY CreatedDate ASC NULLS FIRST"]
     responses:
       - status_code: 200
         body: '{"totalSize":1,"done":true,"records":[{"attributes":{"type":"EventLogFile","url":"/services/data/v56.0/sobjects/EventLogFile/0AT5j00002GVrfnGAD"},"CreatedDate":"2022-09-14T21:43:41.000+0000","LogDate":"2022-09-13T00:00:00.000+0000","LogFile":"/services/data/v56.0/sobjects/EventLogFile/0AT5j00002GVrfnGAD/LogFile"}]}'
@@ -49,7 +49,7 @@ rules:
   - path: /services/data/v56.0/query/
     methods: ["GET"]
     query_params:
-      q: ["SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' ORDER BY LogDate ASC NULLS FIRST"]
+      q: ["SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND EventType = 'Logout' ORDER BY CreatedDate ASC NULLS FIRST"]
     responses:
       - status_code: 200
         body: '{"totalSize":1,"done":true,"records":[{"attributes":{"type":"EventLogFile","url":"/services/data/v56.0/sobjects/EventLogFile/0AT5j00002GWEsRGAX"},"CreatedDate":"2022-09-19T21:03:41.000+0000","LogDate":"2022-09-18T00:00:00.000+0000","LogFile":"/services/data/v56.0/sobjects/EventLogFile/0AT5j00002GWEsRGAX/LogFile"}]}'
@@ -78,7 +78,7 @@ rules:
   - path: /services/data/v56.0/query/
     methods: ["GET"]
     query_params:
-      q: ["SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST"]
+      q: ["SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE Interval = 'Hourly' AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY CreatedDate ASC NULLS FIRST"]
     responses:
       - status_code: 200
         body: '{"done":true,"records":[{"CreatedDate":"2022-10-01T23:22:27.000+0000","Id":"0AT5j00002GWEsRGAY","LogDate":"2022-09-30T00:00:00.000+0000","LogFile":"/services/data/v56.0/sobjects/EventLogFile/0AT5j00002GWEsRGAY/LogFile","attributes":{"type":"EventLogFile","url":"/services/data/v56.0/sobjects/EventLogFile/0AT5j00002GWEsRGAY"}}],"totalSize":1}'

packages/salesforce/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.8.1"
+  changes:
+    - description: Fix EventLogFile cursor not advancing by ordering the Apex, Login, and Logout queries by `CreatedDate` (the cursor field) instead of `LogDate`. Previously the mismatch let the persisted cursor regress, causing repeated re-collection of old data.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/19954
 - version: "1.8.0"
   changes:
     - description: Expose the `Request Timeout` (`resource.timeout`) setting for all data streams so the HTTP client timeout can be increased when log collection (for example, large Apex EventLogFile downloads) times out.

packages/salesforce/data_stream/apex/agent/stream/salesforce.yml.hbs

@@ -29,8 +29,8 @@ event_monitoring_method:
     enabled: true
     interval: {{period}}
     query:
-      default: SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE {{#if initial_interval}}LogDate > [[ (formatTime (now.Add (parseDuration "-{{initial_interval}}")) "2006-01-02T15:04:05.000Z0700") ]] AND {{/if}}{{#if log_file_interval}}Interval = '{{log_file_interval}}' AND {{/if}}(EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST
-      value: SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE {{#if log_file_interval}}Interval = '{{log_file_interval}}' AND {{/if}}CreatedDate > [[ .cursor.event_log_file.last_event_time ]] AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY LogDate ASC NULLS FIRST
+      default: SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE {{#if initial_interval}}LogDate > [[ (formatTime (now.Add (parseDuration "-{{initial_interval}}")) "2006-01-02T15:04:05.000Z0700") ]] AND {{/if}}{{#if log_file_interval}}Interval = '{{log_file_interval}}' AND {{/if}}(EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY CreatedDate ASC NULLS FIRST
+      value: SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE {{#if log_file_interval}}Interval = '{{log_file_interval}}' AND {{/if}}CreatedDate > [[ .cursor.event_log_file.last_event_time ]] AND (EventType = 'ApexCallout' OR EventType = 'ApexExecution' OR EventType = 'ApexRestApi' OR EventType = 'ApexSoap' OR EventType = 'ApexTrigger' OR EventType = 'ExternalCustomApexCallout') ORDER BY CreatedDate ASC NULLS FIRST
     cursor:
       field: "CreatedDate"
 

packages/salesforce/data_stream/login/agent/stream/salesforce.yml.hbs

@@ -25,8 +25,8 @@ event_monitoring_method:
     enabled: {{#if event_log_file}}true{{else}}false{{/if}}
     interval: {{elf_period}}
     query:
-      default: SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE {{#if initial_interval}}LogDate > [[ (formatTime (now.Add (parseDuration "-{{initial_interval}}")) "2006-01-02T15:04:05.000Z0700") ]] AND {{/if}}{{#if log_file_interval}}Interval = '{{log_file_interval}}' AND {{/if}}EventType = 'Login' ORDER BY LogDate ASC NULLS FIRST
-      value: SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE {{#if log_file_interval}}Interval = '{{log_file_interval}}' AND {{/if}}EventType = 'Login' AND CreatedDate > [[ .cursor.event_log_file.last_event_time ]] ORDER BY LogDate ASC NULLS FIRST
+      default: SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE {{#if initial_interval}}LogDate > [[ (formatTime (now.Add (parseDuration "-{{initial_interval}}")) "2006-01-02T15:04:05.000Z0700") ]] AND {{/if}}{{#if log_file_interval}}Interval = '{{log_file_interval}}' AND {{/if}}EventType = 'Login' ORDER BY CreatedDate ASC NULLS FIRST
+      value: SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE {{#if log_file_interval}}Interval = '{{log_file_interval}}' AND {{/if}}EventType = 'Login' AND CreatedDate > [[ .cursor.event_log_file.last_event_time ]] ORDER BY CreatedDate ASC NULLS FIRST
     cursor:
       field: "CreatedDate"
   object:

packages/salesforce/data_stream/logout/agent/stream/salesforce.yml.hbs

@@ -25,8 +25,8 @@ event_monitoring_method:
     enabled: {{#if event_log_file}}true{{else}}false{{/if}}
     interval: {{elf_period}}
     query:
-      default: SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE {{#if initial_interval}}LogDate > [[ (formatTime (now.Add (parseDuration "-{{initial_interval}}")) "2006-01-02T15:04:05.000Z0700") ]] AND {{/if}}{{#if log_file_interval}}Interval = '{{log_file_interval}}' AND {{/if}}EventType = 'Logout' ORDER BY LogDate ASC NULLS FIRST
-      value: SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE {{#if log_file_interval}}Interval = '{{log_file_interval}}' AND {{/if}}EventType = 'Logout' AND CreatedDate > [[ .cursor.event_log_file.last_event_time ]] ORDER BY LogDate ASC NULLS FIRST
+      default: SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE {{#if initial_interval}}LogDate > [[ (formatTime (now.Add (parseDuration "-{{initial_interval}}")) "2006-01-02T15:04:05.000Z0700") ]] AND {{/if}}{{#if log_file_interval}}Interval = '{{log_file_interval}}' AND {{/if}}EventType = 'Logout' ORDER BY CreatedDate ASC NULLS FIRST
+      value: SELECT CreatedDate,LogDate,LogFile FROM EventLogFile WHERE {{#if log_file_interval}}Interval = '{{log_file_interval}}' AND {{/if}}EventType = 'Logout' AND CreatedDate > [[ .cursor.event_log_file.last_event_time ]] ORDER BY CreatedDate ASC NULLS FIRST
     cursor:
       field: "CreatedDate"
   object:

packages/salesforce/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.2
 name: salesforce
 title: Salesforce
-version: "1.8.0"
+version: "1.8.1"
 description: |
   Collect logs from Salesforce instances using the Elastic Agent. This integration enables monitoring and analysis of various Salesforce logs, including Login, Logout, Setup Audit Trail, and Apex execution logs. Gain insights into user activity, security events, and application performance.
 type: integration