diff --git a/airflow/www/jest-setup.js b/airflow/www/jest-setup.js index a4679deed25ea..14974f3f65199 100644 --- a/airflow/www/jest-setup.js +++ b/airflow/www/jest-setup.js @@ -23,6 +23,9 @@ import '@testing-library/jest-dom'; import axios from 'axios'; import { setLogger } from 'react-query'; +// eslint-disable-next-line import/no-extraneous-dependencies +import moment from 'moment-timezone'; + axios.defaults.adapter = require('axios/lib/adapters/http'); axios.interceptors.response.use( @@ -51,3 +54,5 @@ global.stateColors = { }; global.defaultDagRunDisplayNumber = 245; + +global.moment = moment; diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx index 8479593715237..1e6bc6664a5b6 100644 --- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx @@ -17,7 +17,9 @@ * under the License. */ -import React, { useRef, useState, useEffect } from 'react'; +import React, { + useRef, useState, useEffect, useMemo, +} from 'react'; import { Text, Box, @@ -26,12 +28,15 @@ import { Code, Button, Checkbox, + Select, } from '@chakra-ui/react'; import { getMetaValue } from '../../../../../utils'; import LogLink from './LogLink'; import useTaskLog from '../../../../api/useTaskLog'; import LinkButton from '../../../../components/LinkButton'; +import { logLevel, parseLogs } from './utils'; +import { useTimezone } from '../../../../context/timezone'; const showExternalLogRedirect = getMetaValue('show_external_log_redirect') === 'True'; const externalLogName = getMetaValue('external_log_name'); @@ -65,6 +70,9 @@ const Logs = ({ const [selectedAttempt, setSelectedAttempt] = useState(1); const [shouldRequestFullContent, setShouldRequestFullContent] = useState(false); const [wrap, setWrap] = useState(false); + const [logLevelFilter, setLogLevelFilter] = useState(''); + const [fileSourceFilter, setFileSourceFilter] = useState(''); + const { timezone } = useTimezone(); const { data, isSuccess } = useTaskLog({ dagId, dagRunId, @@ -86,55 +94,98 @@ const Logs = ({ execution_date: executionDate, }).toString(); + const { parsedLogs, fileSources = [] } = useMemo(() => parseLogs( + data, + timezone, + logLevelFilter, + fileSourceFilter, + ), + [data, fileSourceFilter, logLevelFilter, timezone]); + + useEffect(() => { + // Reset fileSourceFilter and selected attempt when changing to + // a task that do not have those filters anymore. + if (!internalIndexes.includes(selectedAttempt)) { + setSelectedAttempt(internalIndexes[0]); + } + if (fileSourceFilter && !fileSources.includes(fileSourceFilter)) { + setFileSourceFilter(''); + } + }, [data, internalIndexes, fileSourceFilter, fileSources, selectedAttempt]); + return ( <> {tryNumber > 0 && ( <> (by attempts) - - - - {internalIndexes.map((index) => ( - - ))} - - - setWrap((previousState) => !previousState)} - px={4} + + + {internalIndexes.map((index) => ( + + ))} + + + + + + + + - + + setWrap((previousState) => !previousState)} + px={4} + > + Wrap + + setShouldRequestFullContent((previousState) => !previousState)} + px={4} + data-testid="full-content-checkbox" + > + Full Logs + + + + See More + + + { isSuccess && ( - {data} + {parsedLogs}
) diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.jsx b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.jsx index 47af93b1e1597..9cf97cda7276e 100644 --- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.jsx +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.jsx @@ -40,7 +40,7 @@ const mockTaskLog = ` [2022-06-04 00:00:01,921] {standard_task_runner.py:81} INFO - Job 1626: Subtask section_1.get_entry_group [2022-06-04 00:00:01,921] {dagbag.py:507} INFO - Filling up the DagBag from /files/dags/test_ui_grid.py [2022-06-04 00:00:01,964] {task_command.py:377} INFO - Running on host 5d28cfda3219 -[2022-06-04 00:00:02,010] {taskinstance.py:1548} INFO - Exporting the following env vars: +[2022-06-04 00:00:02,010] {taskinstance.py:1548} WARNING - Exporting the following env vars: AIRFLOW_CTX_DAG_OWNER=*** AIRFLOW_CTX_DAG_ID=test_ui_grid `; @@ -67,8 +67,8 @@ describe('Test Logs Component.', () => { tryNumber={tryNumber} />, ); - expect(getByText('[2022-06-04 00:00:01,906] {taskinstance.py:1330} INFO -', { exact: false })).toBeDefined(); - expect(getByText('[2022-06-04 00:00:01,921] {standard_task_runner.py:81} INFO - Job 1626: Subtask section_1.get_entry_group', + expect(getByText('[2022-06-04, 00:00:01 UTC] {taskinstance.py:1329} INFO -', { exact: false })).toBeDefined(); + expect(getByText('[2022-06-04, 00:00:01 UTC] {standard_task_runner.py:81} INFO - Job 1626: Subtask section_1.get_entry_group', { exact: false })).toBeDefined(); expect(getByText('AIRFLOW_CTX_DAG_ID=test_ui_grid', { exact: false })).toBeDefined(); }); diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js new file mode 100644 index 0000000000000..0b6d3de4dd10b --- /dev/null +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js @@ -0,0 +1,70 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* global moment */ + +import { defaultFormatWithTZ } from '../../../../../datetime_utils'; + +export const logLevel = { + DEBUG: 'DEBUG', + INFO: 'INFO', + WARNING: 'WARNING', + ERROR: 'ERROR', + CRITICAL: 'CRITICAL', +}; + +export const parseLogs = (data, timezone, logLevelFilter, fileSourceFilter) => { + const lines = data.split('\n'); + + if (!data) { + return {}; + } + + const parsedLines = []; + const fileSources = new Set(); + + lines.forEach((line) => { + let parsedLine = line; + + // Apply log level filter. + if (logLevelFilter && !line.includes(logLevelFilter)) { + return; + } + + const regExp = /\[(.*?)\] \{(.*?)\}/; + const matches = line.match(regExp); + let logGroup = ''; + if (matches) { + // Replace UTC with the local timezone. + const dateTime = matches[1]; + [logGroup] = matches[2].split(':'); + if (dateTime && timezone) { + const localDateTime = moment.utc(dateTime).tz(timezone).format(defaultFormatWithTZ); + parsedLine = line.replace(dateTime, localDateTime); + } + + fileSources.add(logGroup); + } + if (!fileSourceFilter || fileSourceFilter === logGroup) { + parsedLines.push(parsedLine); + } + }); + + return { parsedLogs: parsedLines.join('\n'), fileSources: Array.from(fileSources).sort() }; +}; diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js new file mode 100644 index 0000000000000..3faa0944a4f5b --- /dev/null +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js @@ -0,0 +1,128 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* global describe, test, expect */ + +import { parseLogs } from './utils'; + +const mockTaskLog = ` +5d28cfda3219 +*** Reading local file: /root/airflow/logs/dag_id=test_ui_grid/run_id=scheduled__2022-06-03T00:00:00+00:00/task_id=section_1.get_entry_group/attempt=1.log +[2022-06-04 00:00:01,901] {taskinstance.py:1132} INFO - Dependencies all met for +[2022-06-04 00:00:01,906] {taskinstance.py:1132} INFO - Dependencies all met for +[2022-06-04 00:00:01,906] {taskinstance.py:1329} INFO - +-------------------------------------------------------------------------------- +[2022-06-04 00:00:01,906] {taskinstance.py:1330} INFO - Starting attempt 1 of 1 +[2022-06-04 00:00:01,906] {taskinstance.py:1331} INFO - +-------------------------------------------------------------------------------- +[2022-06-04 00:00:01,916] {taskinstance.py:1350} INFO - Executing on 2022-06-03 00:00:00+00:00 +[2022-06-04 00:00:01,919] {standard_task_runner.py:52} INFO - Started process 41646 to run task +[2022-06-04 00:00:01,920] {standard_task_runner.py:80} INFO - Running: ['***', 'tasks', 'run', 'test_ui_grid', 'section_1.get_entry_group', 'scheduled__2022-06-03T00:00:00+00:00', '--job-id', '1626', '--raw', '--subdir', 'DAGS_FOLDER/test_ui_grid.py', '--cfg-path', '/tmp/tmpte7k80ur'] +[2022-06-04 00:00:01,921] {standard_task_runner.py:81} INFO - Job 1626: Subtask section_1.get_entry_group +[2022-06-04 00:00:01,921] {dagbag.py:507} INFO - Filling up the DagBag from /files/dags/test_ui_grid.py +[2022-06-04 00:00:01,964] {task_command.py:377} INFO - Running on host 5d28cfda3219 +[2022-06-04 00:00:02,010] {taskinstance.py:1548} WARNING - Exporting the following env vars: +AIRFLOW_CTX_DAG_OWNER=*** +AIRFLOW_CTX_DAG_ID=test_ui_grid +`; + +describe('Test Logs Utils.', () => { + test('parseLogs function replaces datetimes', () => { + const { parsedLogs, fileSources } = parseLogs( + mockTaskLog, + 'UTC', + null, + null, + ); + + expect(parsedLogs).toContain('2022-06-04, 00:00:01 UTC'); + expect(fileSources).toEqual([ + 'dagbag.py', + 'standard_task_runner.py', + 'task_command.py', + 'taskinstance.py', + ]); + const result = parseLogs( + mockTaskLog, + 'America/Los_Angeles', + null, + null, + ); + expect(result.parsedLogs).toContain('2022-06-03, 17:00:01 PDT'); + }); + + test.each([ + { logLevelFilter: 'INFO', expectedNumberOfLines: 11, expectedNumberOfFileSources: 4 }, + { logLevelFilter: 'WARNING', expectedNumberOfLines: 1, expectedNumberOfFileSources: 1 }, + ])('Filtering logs on $logLevelFilter level should return $expectedNumberOfLines lines and $expectedNumberOfFileSources file sources', + ({ + logLevelFilter, + expectedNumberOfLines, expectedNumberOfFileSources, + }) => { + const { parsedLogs, fileSources } = parseLogs( + mockTaskLog, + null, + logLevelFilter, + null, + ); + + expect(fileSources).toHaveLength(expectedNumberOfFileSources); + const lines = parsedLogs.split('\n'); + expect(lines).toHaveLength(expectedNumberOfLines); + lines.forEach((line) => expect(line).toContain(logLevelFilter)); + }); + + test('parseLogs function with file source filter', () => { + const { parsedLogs, fileSources } = parseLogs( + mockTaskLog, + null, + null, + 'taskinstance.py', + ); + + expect(fileSources).toEqual([ + 'dagbag.py', + 'standard_task_runner.py', + 'task_command.py', + 'taskinstance.py', + ]); + const lines = parsedLogs.split('\n'); + expect(lines).toHaveLength(7); + lines.forEach((line) => expect(line).toContain('taskinstance.py')); + }); + + test('parseLogs function with filter on log level and file source', () => { + const { parsedLogs, fileSources } = parseLogs( + mockTaskLog, + null, + 'INFO', + 'taskinstance.py', + ); + + expect(fileSources).toEqual([ + 'dagbag.py', + 'standard_task_runner.py', + 'task_command.py', + 'taskinstance.py', + ]); + const lines = parsedLogs.split('\n'); + expect(lines).toHaveLength(6); + lines.forEach((line) => expect(line).toContain('INFO')); + }); +}); diff --git a/airflow/www/static/js/grid/utils/testUtils.jsx b/airflow/www/static/js/grid/utils/testUtils.jsx index 50ef2305fec20..61cb33b24572a 100644 --- a/airflow/www/static/js/grid/utils/testUtils.jsx +++ b/airflow/www/static/js/grid/utils/testUtils.jsx @@ -21,15 +21,11 @@ import React from 'react'; import { ChakraProvider, Table, Tbody } from '@chakra-ui/react'; import { QueryClient, QueryClientProvider } from 'react-query'; import { MemoryRouter } from 'react-router-dom'; -// eslint-disable-next-line import/no-extraneous-dependencies -import moment from 'moment-timezone'; import { ContainerRefProvider } from '../context/containerRef'; import { TimezoneProvider } from '../context/timezone'; import { AutoRefreshProvider } from '../context/autorefresh'; -global.moment = moment; - export const Wrapper = ({ children }) => { const queryClient = new QueryClient({ defaultOptions: {