[PHP-FPM] Add Integration Package with Process Data Stream

packages/php_fpm/_dev/build/docs/README.md

@@ -8,7 +8,7 @@ PHP-FPM (FastCGI Process Manager) is a web tool used to speed up the performance
 
 The PHP-FPM integration collects metrics data.
 
-Metrics give you insight into the statistics of the PHP-FPM. Metrics data streams collected by the PHP-FPM integration include [pool](https://www.php.net/manual/en/fpm.status.php#:~:text=Basic%20information%20%2D%20Always%20displayed%20on%20the%20status%20page) so that the user could monitor and troubleshoot the performance of the PHP-FPM instances.
+Metrics give you insight into the statistics of the PHP-FPM. Metrics data streams collected by the PHP-FPM integration include [pool](https://www.php.net/manual/en/fpm.status.php#:~:text=Basic%20information%20%2D%20Always%20displayed%20on%20the%20status%20page) and [process](https://www.php.net/manual/en/fpm.status.php#:~:text=Per%2Dprocess%20information%20%2D%20only%20displayed%20in%20full%20output%20mode) so that the user can monitor and troubleshoot the performance of the PHP-FPM instances.
 
 Note:
 - Users can monitor and see the metrics inside the ingested documents for PHP-FPM in the logs-* index pattern from `Discover`.
@@ -40,3 +40,11 @@ This is the `pool` data stream. `pool` data stream collects metrics related to t
 {{event "pool"}}
 
 {{fields "pool"}}
+
+### Process
+
+This is the `process` data stream. `process` data stream collects metrics like request duration, content length, process state, etc.
+
+{{event "process"}}
+
+{{fields "process"}}

packages/php_fpm/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.2.0"
+  changes:
+    - description: PHP-FPM integration package with "Process" data stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/4652
 - version: "0.1.0"
   changes:
     - description: PHP-FPM integration package with "Pool" data stream.

packages/php_fpm/data_stream/process/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,3 @@
+fields:
+  tags:
+    - preserve_original_event

packages/php_fpm/data_stream/process/_dev/test/pipeline/test-process-metrics.log

@@ -0,0 +1 @@
+{"pool":"www","process manager":"dynamic","start time":1667299161,"start since":515234,"accepted conn":16647,"listen queue":0,"max listen queue":0,"listen queue len":0,"idle processes":1,"active processes":1,"total processes":2,"max active processes":1,"max children reached":0,"slow requests":0, "processes":{"pid":183251,"state":"Running","start time":1667299161,"start since":515234,"requests":8324,"request duration":208,"request method":"GET","request uri":"/status?json&full","content length":0,"user":"-","script":"-","last request cpu":0.00,"last request memory":0}}

packages/php_fpm/data_stream/process/_dev/test/pipeline/test-process-metrics.log-expected.json

@@ -0,0 +1,63 @@
+{
+    "expected": [
+        {
+            "ecs": {
+                "version": "8.4.0"
+            },
+            "event": {
+                "category": [
+                    "web",
+                    "configuration",
+                    "process"
+                ],
+                "kind": "event",
+                "module": "php_fpm",
+                "original": "{\"pool\":\"www\",\"process manager\":\"dynamic\",\"start time\":1667299161,\"start since\":515234,\"accepted conn\":16647,\"listen queue\":0,\"max listen queue\":0,\"listen queue len\":0,\"idle processes\":1,\"active processes\":1,\"total processes\":2,\"max active processes\":1,\"max children reached\":0,\"slow requests\":0, \"processes\":{\"pid\":183251,\"state\":\"Running\",\"start time\":1667299161,\"start since\":515234,\"requests\":8324,\"request duration\":208,\"request method\":\"GET\",\"request uri\":\"/status?json\u0026full\",\"content length\":0,\"user\":\"-\",\"script\":\"-\",\"last request cpu\":0.00,\"last request memory\":0}}",
+                "type": [
+                    "info"
+                ]
+            },
+            "http": {
+                "request": {
+                    "body": {
+                        "bytes": 0
+                    },
+                    "method": "GET"
+                }
+            },
+            "php_fpm": {
+                "process": {
+                    "pool": {
+                        "name": "www"
+                    },
+                    "request": {
+                        "count": 8324,
+                        "duration": 208,
+                        "last": {
+                            "cpu": {
+                                "pct": 0.0
+                            },
+                            "memory": 0
+                        }
+                    },
+                    "script": "-",
+                    "start_since": 515234,
+                    "start_time": 1667299161,
+                    "state": "Running"
+                }
+            },
+            "process": {
+                "pid": 183251
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "url": {
+                "original": "/status?json\u0026full"
+            },
+            "user": {
+                "name": "-"
+            }
+        }
+    ]
+}

packages/php_fpm/data_stream/process/_dev/test/system/test-default-config.yml

@@ -0,0 +1,5 @@
+vars:
+  hostname:
+    - http://{{Hostname}}
+  status_path:
+    - /status

packages/php_fpm/data_stream/process/agent/stream/httpjson.yml.hbs

@@ -0,0 +1,24 @@
+config_version: 2
+interval: {{period}}
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+request.method: GET
+request.url: {{hostname}}{{status_path}}?json&full
+response.split:
+  keep_parent: true
+  target: body.processes
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/php_fpm/data_stream/process/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,138 @@
+---
+description: Pipeline for parsing PHP-FPM Process metrics.
+processors:
+  - set:
+      field: ecs.version
+      value: 8.4.0
+      ignore_failure: true
+      ignore_empty_value: true
+  - set:
+      field: event.type
+      value: ["info"]
+      ignore_failure: true
+      ignore_empty_value: true
+  - set:
+      field: event.kind
+      value: event
+      ignore_failure: true
+      ignore_empty_value: true
+  - set:
+      field: event.category
+      value: ["web","configuration","process"]
+      ignore_failure: true
+      ignore_empty_value: true
+  - set:
+      field: event.module
+      value: php_fpm
+      ignore_failure: true
+      ignore_empty_value: true
+  - rename:
+      field: message
+      target_field: event.original
+      ignore_missing: true
+      ignore_failure: true
+  - json:
+      field: event.original
+      target_field: json
+      ignore_failure: true
+  - rename:
+      field: json.processes.state
+      target_field: php_fpm.process.state
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.processes.requests
+      target_field: php_fpm.process.request.count
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.processes.request duration
+      target_field: php_fpm.process.request.duration
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.processes.script
+      target_field: php_fpm.process.script
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.pool
+      target_field: php_fpm.process.pool.name
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.processes.last request cpu
+      target_field: php_fpm.process.request.last.cpu.pct
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.processes.last request memory
+      target_field: php_fpm.process.request.last.memory
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.processes.pid
+      target_field: process.pid
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.processes.request uri
+      target_field: url.original
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.processes.user
+      target_field: user.name
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.processes.request method
+      target_field: http.request.method
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.processes.content length
+      target_field: http.request.body.bytes
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.processes.start since
+      target_field: php_fpm.process.start_since
+      ignore_missing: true
+      ignore_failure: true
+  - rename:
+      field: json.processes.start time
+      target_field: php_fpm.process.start_time
+      ignore_missing: true
+      ignore_failure: true
+  - remove:
+      field: event.original
+      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_missing: true
+      ignore_failure: true
+  - script:
+      description: Drops null/empty values recursively.
+      lang: painless
+      source: |
+        boolean drop(Object o) {
+          if (o == null || o == "" || o == "(null)") {
+            return true;
+          } else if (o instanceof Map) {
+            ((Map) o).values().removeIf(v -> drop(v));
+            return (((Map) o).size() == 0);
+          } else if (o instanceof List) {
+            ((List) o).removeIf(v -> drop(v));
+            return (((List) o).length == 0);
+          }
+          return false;
+        }
+        drop(ctx);
+  - remove:
+      field:
+        - json
+      ignore_missing: true
+      ignore_failure: true
+on_failure:
+  - set:
+      field: error.message
+      value: "{{{_ingest.on_failure_message}}}"

packages/php_fpm/data_stream/process/fields/base-fields.yml

@@ -0,0 +1,15 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: input.type
+  type: keyword
+  description: Type of Filebeat input.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

packages/php_fpm/data_stream/process/fields/ecs.yml

@@ -0,0 +1,28 @@
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: error.message
+- external: ecs
+  name: event.category
+- external: ecs
+  name: event.dataset
+- external: ecs
+  name: event.ingested
+- external: ecs
+  name: event.kind
+- external: ecs
+  name: event.module
+- external: ecs
+  name: event.type
+- external: ecs
+  name: http.request.body.bytes
+- external: ecs
+  name: http.request.method
+- external: ecs
+  name: process.pid
+- external: ecs
+  name: url.original
+- external: ecs
+  name: user.name
+- external: ecs
+  name: tags

packages/php_fpm/data_stream/process/fields/fields.yml

@@ -0,0 +1,47 @@
+- name: php_fpm
+  type: group
+  fields:
+    - name: process
+      type: group
+      fields:
+        - name: pool.name
+          type: keyword
+          description: The name of the FPM process pool.
+        - name: request
+          type: group
+          fields:
+            - name: count
+              type: long
+              description: The total number of requests served.
+              metric_type: counter
+            - name: duration
+              type: long
+              description: The total time in seconds spent serving requests.
+              metric_type: counter
+              unit: s
+            - name: last
+              type: group
+              fields:
+                - name: cpu.pct
+                  type: long
+                  description: The %cpu of the last request. This will be 0 if the process is not Idle because the calculation is done when the request processing is complete.
+                  metric_type: gauge
+                  unit: percent
+                - name: memory
+                  type: long
+                  description: The maximum amount of memory consumed by the last request. This will be 0 if the process is not Idle because the calculation is done when the request processing is complete.
+                  metric_type: gauge
+        - name: script
+          type: keyword
+          description: The full path of the script executed by the last request. This will be '-' if not applicable (eg. status page requests).
+        - name: start_since
+          type: long
+          description: The number of seconds since the process started.
+          metric_type: counter
+          unit: s
+        - name: start_time
+          type: long
+          description: The date/time at which the process started.
+        - name: state
+          type: keyword
+          description: The state of the process.