From 334e4085d70ec578795d2a00332448511d2fd85b Mon Sep 17 00:00:00 2001 From: pierrejeambrun Date: Sun, 12 Jun 2022 18:19:27 +0200 Subject: [PATCH 1/5] Logs filtering + local datetime --- .../content/taskInstance/Logs/index.jsx | 125 ++++++++++++------ .../content/taskInstance/Logs/utils.js | 73 ++++++++++ 2 files changed, 154 insertions(+), 44 deletions(-) create mode 100644 airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx index 8479593715237..d9cf33e051703 100644 --- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx @@ -17,7 +17,9 @@ * under the License. */ -import React, { useRef, useState, useEffect } from 'react'; +import React, { + useRef, useState, useEffect, useMemo, +} from 'react'; import { Text, Box, @@ -26,12 +28,15 @@ import { Code, Button, Checkbox, + Select, } from '@chakra-ui/react'; import { getMetaValue } from '../../../../../utils'; import LogLink from './LogLink'; import useTaskLog from '../../../../api/useTaskLog'; import LinkButton from '../../../../components/LinkButton'; +import { LogLevel, parseLogs } from './utils'; +import { useTimezone } from '../../../../context/timezone'; const showExternalLogRedirect = getMetaValue('show_external_log_redirect') === 'True'; const externalLogName = getMetaValue('external_log_name'); @@ -65,6 +70,9 @@ const Logs = ({ const [selectedAttempt, setSelectedAttempt] = useState(1); const [shouldRequestFullContent, setShouldRequestFullContent] = useState(false); const [wrap, setWrap] = useState(false); + const [logLevelFilter, setLogLevelFilter] = useState(''); + const [logGroupFilter, setLogGroupFilter] = useState(''); + const { timezone } = useTimezone(); const { data, isSuccess } = useTaskLog({ dagId, dagRunId, @@ -86,55 +94,84 @@ const Logs = ({ execution_date: executionDate, }).toString(); + const { parsedLogs, logGroups = [] } = useMemo(() => parseLogs( + data, + timezone, + logLevelFilter, + logGroupFilter, + ), + [data, logGroupFilter, logLevelFilter, timezone]); + return ( <> {tryNumber > 0 && ( <> (by attempts) - - - - {internalIndexes.map((index) => ( - - ))} - - - setWrap((previousState) => !previousState)} - px={4} - > - Wrap - - setShouldRequestFullContent((previousState) => !previousState)} - px={4} - data-testid="full-content-checkbox" - > - Full Logs - - - + + {internalIndexes.map((index) => ( + + ))} - + + + + + + setWrap((previousState) => !previousState)} + px={4} + > + Wrap + + setShouldRequestFullContent((previousState) => !previousState)} + px={4} + data-testid="full-content-checkbox" + > + Full Logs + + + + See More + + + { isSuccess && ( - {data} + {parsedLogs}
) diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js new file mode 100644 index 0000000000000..b07e62aedda6e --- /dev/null +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js @@ -0,0 +1,73 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { defaultFormatWithTZ } from '../../../../../datetime_utils'; + +/* global moment */ + +export const LogLevel = { + DEBUG: 'DEBUG', + INFO: 'INFO', + WARNING: 'WARNING', + ERROR: 'ERROR', + CRITICAL: 'CRITICAL', +}; + +export const parseLogs = (data, timezone, logLevelFilter, logGroupFilter) => { + const lines = data.split('\n'); + + if (!data) { + return {}; + } + + const parsedLines = []; + const logGroups = new Set(); + + lines.forEach((line) => { + let parsedLine = line; + + // Apply log level filter. + if (logLevelFilter && !line.includes(logLevelFilter)) { + return; + } + + const regExp = /\[(.*?)\] \{(.*?)\}/; + const matches = line.match(regExp); + if (matches) { + // Replace UTC with the local timezone. + const dateTime = matches[1]; + const logGroup = matches[2].split(':')[0]; + if (dateTime && timezone) { + const localDateTime = moment(dateTime).tz(timezone).format(defaultFormatWithTZ); + parsedLine = line.replace(dateTime, localDateTime); + } + + logGroups.add(logGroup); + + // Apply the log group filter. + if (logGroupFilter && logGroup !== logGroupFilter) { + return; + } + } + + parsedLines.push(parsedLine); + }); + + return { parsedLogs: parsedLines.join('\n'), logGroups: Array.from(logGroups).sort() }; +}; From a65c4e97bf7b6cd2cdc4cc216ebb82b07bb937f8 Mon Sep 17 00:00:00 2001 From: pierrejeambrun Date: Sun, 12 Jun 2022 19:25:57 +0200 Subject: [PATCH 2/5] Add tests --- airflow/www/jest-setup.js | 5 + .../content/taskInstance/Logs/index.test.jsx | 6 +- .../content/taskInstance/Logs/utils.js | 19 ++- .../content/taskInstance/Logs/utils.test.js | 110 ++++++++++++++++++ .../www/static/js/grid/utils/testUtils.jsx | 4 - 5 files changed, 126 insertions(+), 18 deletions(-) create mode 100644 airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js diff --git a/airflow/www/jest-setup.js b/airflow/www/jest-setup.js index a4679deed25ea..14974f3f65199 100644 --- a/airflow/www/jest-setup.js +++ b/airflow/www/jest-setup.js @@ -23,6 +23,9 @@ import '@testing-library/jest-dom'; import axios from 'axios'; import { setLogger } from 'react-query'; +// eslint-disable-next-line import/no-extraneous-dependencies +import moment from 'moment-timezone'; + axios.defaults.adapter = require('axios/lib/adapters/http'); axios.interceptors.response.use( @@ -51,3 +54,5 @@ global.stateColors = { }; global.defaultDagRunDisplayNumber = 245; + +global.moment = moment; diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.jsx b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.jsx index 47af93b1e1597..9cf97cda7276e 100644 --- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.jsx +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.test.jsx @@ -40,7 +40,7 @@ const mockTaskLog = ` [2022-06-04 00:00:01,921] {standard_task_runner.py:81} INFO - Job 1626: Subtask section_1.get_entry_group [2022-06-04 00:00:01,921] {dagbag.py:507} INFO - Filling up the DagBag from /files/dags/test_ui_grid.py [2022-06-04 00:00:01,964] {task_command.py:377} INFO - Running on host 5d28cfda3219 -[2022-06-04 00:00:02,010] {taskinstance.py:1548} INFO - Exporting the following env vars: +[2022-06-04 00:00:02,010] {taskinstance.py:1548} WARNING - Exporting the following env vars: AIRFLOW_CTX_DAG_OWNER=*** AIRFLOW_CTX_DAG_ID=test_ui_grid `; @@ -67,8 +67,8 @@ describe('Test Logs Component.', () => { tryNumber={tryNumber} />, ); - expect(getByText('[2022-06-04 00:00:01,906] {taskinstance.py:1330} INFO -', { exact: false })).toBeDefined(); - expect(getByText('[2022-06-04 00:00:01,921] {standard_task_runner.py:81} INFO - Job 1626: Subtask section_1.get_entry_group', + expect(getByText('[2022-06-04, 00:00:01 UTC] {taskinstance.py:1329} INFO -', { exact: false })).toBeDefined(); + expect(getByText('[2022-06-04, 00:00:01 UTC] {standard_task_runner.py:81} INFO - Job 1626: Subtask section_1.get_entry_group', { exact: false })).toBeDefined(); expect(getByText('AIRFLOW_CTX_DAG_ID=test_ui_grid', { exact: false })).toBeDefined(); }); diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js index b07e62aedda6e..8f80b5f10fef0 100644 --- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js @@ -17,10 +17,10 @@ * under the License. */ -import { defaultFormatWithTZ } from '../../../../../datetime_utils'; - /* global moment */ +import { defaultFormatWithTZ } from '../../../../../datetime_utils'; + export const LogLevel = { DEBUG: 'DEBUG', INFO: 'INFO', @@ -49,24 +49,21 @@ export const parseLogs = (data, timezone, logLevelFilter, logGroupFilter) => { const regExp = /\[(.*?)\] \{(.*?)\}/; const matches = line.match(regExp); + let logGroup = ''; if (matches) { // Replace UTC with the local timezone. const dateTime = matches[1]; - const logGroup = matches[2].split(':')[0]; + [logGroup] = matches[2].split(':'); if (dateTime && timezone) { - const localDateTime = moment(dateTime).tz(timezone).format(defaultFormatWithTZ); + const localDateTime = moment.utc(dateTime).tz(timezone).format(defaultFormatWithTZ); parsedLine = line.replace(dateTime, localDateTime); } logGroups.add(logGroup); - - // Apply the log group filter. - if (logGroupFilter && logGroup !== logGroupFilter) { - return; - } } - - parsedLines.push(parsedLine); + if (!logGroupFilter || logGroupFilter === logGroup) { + parsedLines.push(parsedLine); + } }); return { parsedLogs: parsedLines.join('\n'), logGroups: Array.from(logGroups).sort() }; diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js new file mode 100644 index 0000000000000..7699f6738582e --- /dev/null +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js @@ -0,0 +1,110 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* global describe, test, expect */ + +import { parseLogs } from './utils'; + +const mockTaskLog = ` +5d28cfda3219 +*** Reading local file: /root/airflow/logs/dag_id=test_ui_grid/run_id=scheduled__2022-06-03T00:00:00+00:00/task_id=section_1.get_entry_group/attempt=1.log +[2022-06-04 00:00:01,901] {taskinstance.py:1132} INFO - Dependencies all met for +[2022-06-04 00:00:01,906] {taskinstance.py:1132} INFO - Dependencies all met for +[2022-06-04 00:00:01,906] {taskinstance.py:1329} INFO - +-------------------------------------------------------------------------------- +[2022-06-04 00:00:01,906] {taskinstance.py:1330} INFO - Starting attempt 1 of 1 +[2022-06-04 00:00:01,906] {taskinstance.py:1331} INFO - +-------------------------------------------------------------------------------- +[2022-06-04 00:00:01,916] {taskinstance.py:1350} INFO - Executing on 2022-06-03 00:00:00+00:00 +[2022-06-04 00:00:01,919] {standard_task_runner.py:52} INFO - Started process 41646 to run task +[2022-06-04 00:00:01,920] {standard_task_runner.py:80} INFO - Running: ['***', 'tasks', 'run', 'test_ui_grid', 'section_1.get_entry_group', 'scheduled__2022-06-03T00:00:00+00:00', '--job-id', '1626', '--raw', '--subdir', 'DAGS_FOLDER/test_ui_grid.py', '--cfg-path', '/tmp/tmpte7k80ur'] +[2022-06-04 00:00:01,921] {standard_task_runner.py:81} INFO - Job 1626: Subtask section_1.get_entry_group +[2022-06-04 00:00:01,921] {dagbag.py:507} INFO - Filling up the DagBag from /files/dags/test_ui_grid.py +[2022-06-04 00:00:01,964] {task_command.py:377} INFO - Running on host 5d28cfda3219 +[2022-06-04 00:00:02,010] {taskinstance.py:1548} WARNING - Exporting the following env vars: +AIRFLOW_CTX_DAG_OWNER=*** +AIRFLOW_CTX_DAG_ID=test_ui_grid +`; + +describe('Test Logs Utils.', () => { + test('parseLogs function replaces datetimes', () => { + const { parsedLogs, logGroups } = parseLogs( + mockTaskLog, + 'UTC', + null, + null, + ); + expect(parsedLogs).toContain('2022-06-04, 00:00:01 UTC'); + expect(logGroups).toEqual([ + 'dagbag.py', + 'standard_task_runner.py', + 'task_command.py', + 'taskinstance.py', + ]); + + const result = parseLogs( + mockTaskLog, + 'America/Los_Angeles', + null, + null, + ); + expect(result.parsedLogs).toContain('2022-06-03, 17:00:01 PDT'); + }); + + test.each([ + { logLevelFilter: 'INFO', expectedNumberOfLines: 11, expectedNumberOfLogGroups: 4 }, + { logLevelFilter: 'WARNING', expectedNumberOfLines: 1, expectedNumberOfLogGroups: 1 }, + ])('Filtering logs on $logLevelFilter level should return $expectedNumberOfLines lines and $expectedNumberOfLogGroups log groups', + ({ + logLevelFilter, + expectedNumberOfLines, expectedNumberOfLogGroups, + }) => { + const { parsedLogs, logGroups } = parseLogs( + mockTaskLog, + null, + logLevelFilter, + null, + ); + + expect(logGroups).toHaveLength(expectedNumberOfLogGroups); + + const lines = parsedLogs.split('\n'); + expect(lines).toHaveLength(expectedNumberOfLines); + lines.forEach((line) => expect(line).toContain(logLevelFilter)); + }); + + test('parseLogs function group level filter', () => { + const { parsedLogs, logGroups } = parseLogs( + mockTaskLog, + null, + null, + 'taskinstance.py', + ); + expect(logGroups).toEqual([ + 'dagbag.py', + 'standard_task_runner.py', + 'task_command.py', + 'taskinstance.py', + ]); + + const lines = parsedLogs.split('\n'); + expect(lines).toHaveLength(7); + lines.forEach((line) => expect(line).toContain('taskinstance.py')); + }); +}); diff --git a/airflow/www/static/js/grid/utils/testUtils.jsx b/airflow/www/static/js/grid/utils/testUtils.jsx index 50ef2305fec20..61cb33b24572a 100644 --- a/airflow/www/static/js/grid/utils/testUtils.jsx +++ b/airflow/www/static/js/grid/utils/testUtils.jsx @@ -21,15 +21,11 @@ import React from 'react'; import { ChakraProvider, Table, Tbody } from '@chakra-ui/react'; import { QueryClient, QueryClientProvider } from 'react-query'; import { MemoryRouter } from 'react-router-dom'; -// eslint-disable-next-line import/no-extraneous-dependencies -import moment from 'moment-timezone'; import { ContainerRefProvider } from '../context/containerRef'; import { TimezoneProvider } from '../context/timezone'; import { AutoRefreshProvider } from '../context/autorefresh'; -global.moment = moment; - export const Wrapper = ({ children }) => { const queryClient = new QueryClient({ defaultOptions: { From c53799333fe6a3d2c97f29ecd6151af4427792b6 Mon Sep 17 00:00:00 2001 From: pierrejeambrun Date: Sun, 12 Jun 2022 22:19:24 +0200 Subject: [PATCH 3/5] Reset fields on task change if not availables. --- .../grid/details/content/taskInstance/Logs/index.jsx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx index d9cf33e051703..3ea1565c6d82b 100644 --- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx @@ -102,6 +102,17 @@ const Logs = ({ ), [data, logGroupFilter, logLevelFilter, timezone]); + useEffect(() => { + // Reset filters and selected attempt when changing to + // a task that do not have those filters anymore. + if (!internalIndexes.includes(selectedAttempt)) { + setSelectedAttempt(internalIndexes[0]); + } + if (logGroupFilter && !logGroups.includes(logGroupFilter)) { + setLogGroupFilter(''); + } + }, [data, internalIndexes, logGroupFilter, logGroups, selectedAttempt]); + return ( <> {tryNumber > 0 && ( From e6d7cbdf3c4790140a7d4448faeb039c61be5def Mon Sep 17 00:00:00 2001 From: pierrejeambrun Date: Mon, 13 Jun 2022 18:57:56 +0200 Subject: [PATCH 4/5] Update following code review. --- .../content/taskInstance/Logs/index.jsx | 28 +++++++++---------- .../content/taskInstance/Logs/utils.js | 12 ++++---- .../content/taskInstance/Logs/utils.test.js | 22 +++++++-------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx index 3ea1565c6d82b..73122cdb6d113 100644 --- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx @@ -35,7 +35,7 @@ import { getMetaValue } from '../../../../../utils'; import LogLink from './LogLink'; import useTaskLog from '../../../../api/useTaskLog'; import LinkButton from '../../../../components/LinkButton'; -import { LogLevel, parseLogs } from './utils'; +import { logLevel, parseLogs } from './utils'; import { useTimezone } from '../../../../context/timezone'; const showExternalLogRedirect = getMetaValue('show_external_log_redirect') === 'True'; @@ -71,7 +71,7 @@ const Logs = ({ const [shouldRequestFullContent, setShouldRequestFullContent] = useState(false); const [wrap, setWrap] = useState(false); const [logLevelFilter, setLogLevelFilter] = useState(''); - const [logGroupFilter, setLogGroupFilter] = useState(''); + const [fileSourceFilter, setFileSourceFilter] = useState(''); const { timezone } = useTimezone(); const { data, isSuccess } = useTaskLog({ dagId, @@ -94,24 +94,24 @@ const Logs = ({ execution_date: executionDate, }).toString(); - const { parsedLogs, logGroups = [] } = useMemo(() => parseLogs( + const { parsedLogs, fileSources = [] } = useMemo(() => parseLogs( data, timezone, logLevelFilter, - logGroupFilter, + fileSourceFilter, ), - [data, logGroupFilter, logLevelFilter, timezone]); + [data, fileSourceFilter, logLevelFilter, timezone]); useEffect(() => { - // Reset filters and selected attempt when changing to + // Reset fileSourceFilter and selected attempt when changing to // a task that do not have those filters anymore. if (!internalIndexes.includes(selectedAttempt)) { setSelectedAttempt(internalIndexes[0]); } - if (logGroupFilter && !logGroups.includes(logGroupFilter)) { - setLogGroupFilter(''); + if (fileSourceFilter && !fileSources.includes(fileSourceFilter)) { + setFileSourceFilter(''); } - }, [data, internalIndexes, logGroupFilter, logGroups, selectedAttempt]); + }, [data, internalIndexes, fileSourceFilter, fileSources, selectedAttempt]); return ( <> @@ -140,17 +140,17 @@ const Logs = ({ mr={2} > - {Object.values(LogLevel).map((value) => ( + {Object.values(logLevel).map((value) => ( ))} diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js index 8f80b5f10fef0..0b6d3de4dd10b 100644 --- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.js @@ -21,7 +21,7 @@ import { defaultFormatWithTZ } from '../../../../../datetime_utils'; -export const LogLevel = { +export const logLevel = { DEBUG: 'DEBUG', INFO: 'INFO', WARNING: 'WARNING', @@ -29,7 +29,7 @@ export const LogLevel = { CRITICAL: 'CRITICAL', }; -export const parseLogs = (data, timezone, logLevelFilter, logGroupFilter) => { +export const parseLogs = (data, timezone, logLevelFilter, fileSourceFilter) => { const lines = data.split('\n'); if (!data) { @@ -37,7 +37,7 @@ export const parseLogs = (data, timezone, logLevelFilter, logGroupFilter) => { } const parsedLines = []; - const logGroups = new Set(); + const fileSources = new Set(); lines.forEach((line) => { let parsedLine = line; @@ -59,12 +59,12 @@ export const parseLogs = (data, timezone, logLevelFilter, logGroupFilter) => { parsedLine = line.replace(dateTime, localDateTime); } - logGroups.add(logGroup); + fileSources.add(logGroup); } - if (!logGroupFilter || logGroupFilter === logGroup) { + if (!fileSourceFilter || fileSourceFilter === logGroup) { parsedLines.push(parsedLine); } }); - return { parsedLogs: parsedLines.join('\n'), logGroups: Array.from(logGroups).sort() }; + return { parsedLogs: parsedLines.join('\n'), fileSources: Array.from(fileSources).sort() }; }; diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js index 7699f6738582e..99469e7d61fd6 100644 --- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/utils.test.js @@ -44,14 +44,14 @@ AIRFLOW_CTX_DAG_ID=test_ui_grid describe('Test Logs Utils.', () => { test('parseLogs function replaces datetimes', () => { - const { parsedLogs, logGroups } = parseLogs( + const { parsedLogs, fileSources } = parseLogs( mockTaskLog, 'UTC', null, null, ); expect(parsedLogs).toContain('2022-06-04, 00:00:01 UTC'); - expect(logGroups).toEqual([ + expect(fileSources).toEqual([ 'dagbag.py', 'standard_task_runner.py', 'task_command.py', @@ -68,35 +68,35 @@ describe('Test Logs Utils.', () => { }); test.each([ - { logLevelFilter: 'INFO', expectedNumberOfLines: 11, expectedNumberOfLogGroups: 4 }, - { logLevelFilter: 'WARNING', expectedNumberOfLines: 1, expectedNumberOfLogGroups: 1 }, - ])('Filtering logs on $logLevelFilter level should return $expectedNumberOfLines lines and $expectedNumberOfLogGroups log groups', + { logLevelFilter: 'INFO', expectedNumberOfLines: 11, expectedNumberOfFileSources: 4 }, + { logLevelFilter: 'WARNING', expectedNumberOfLines: 1, expectedNumberOfFileSources: 1 }, + ])('Filtering logs on $logLevelFilter level should return $expectedNumberOfLines lines and $expectedNumberOfFileSources file sources', ({ logLevelFilter, - expectedNumberOfLines, expectedNumberOfLogGroups, + expectedNumberOfLines, expectedNumberOfFileSources, }) => { - const { parsedLogs, logGroups } = parseLogs( + const { parsedLogs, fileSources } = parseLogs( mockTaskLog, null, logLevelFilter, null, ); - expect(logGroups).toHaveLength(expectedNumberOfLogGroups); + expect(fileSources).toHaveLength(expectedNumberOfFileSources); const lines = parsedLogs.split('\n'); expect(lines).toHaveLength(expectedNumberOfLines); lines.forEach((line) => expect(line).toContain(logLevelFilter)); }); - test('parseLogs function group level filter', () => { - const { parsedLogs, logGroups } = parseLogs( + test('parseLogs function with file source filter', () => { + const { parsedLogs, fileSources } = parseLogs( mockTaskLog, null, null, 'taskinstance.py', ); - expect(logGroups).toEqual([ + expect(fileSources).toEqual([ 'dagbag.py', 'standard_task_runner.py', 'task_command.py', From 4f04586f0cb163c2073e14d9ec7ee3751637e31a Mon Sep 17 00:00:00 2001 From: pierrejeambrun Date: Mon, 13 Jun 2022 23:41:24 +0200 Subject: [PATCH 5/5] Fix select width --- .../content/taskInstance/Logs/index.jsx | 45 ++++++++++--------- .../content/taskInstance/Logs/utils.test.js | 24 ++++++++-- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx index 73122cdb6d113..1e6bc6664a5b6 100644 --- a/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx +++ b/airflow/www/static/js/grid/details/content/taskInstance/Logs/index.jsx @@ -133,27 +133,30 @@ const Logs = ({ ))} - - + + + + + + { null, null, ); + expect(parsedLogs).toContain('2022-06-04, 00:00:01 UTC'); expect(fileSources).toEqual([ 'dagbag.py', @@ -57,7 +58,6 @@ describe('Test Logs Utils.', () => { 'task_command.py', 'taskinstance.py', ]); - const result = parseLogs( mockTaskLog, 'America/Los_Angeles', @@ -83,7 +83,6 @@ describe('Test Logs Utils.', () => { ); expect(fileSources).toHaveLength(expectedNumberOfFileSources); - const lines = parsedLogs.split('\n'); expect(lines).toHaveLength(expectedNumberOfLines); lines.forEach((line) => expect(line).toContain(logLevelFilter)); @@ -96,15 +95,34 @@ describe('Test Logs Utils.', () => { null, 'taskinstance.py', ); + expect(fileSources).toEqual([ 'dagbag.py', 'standard_task_runner.py', 'task_command.py', 'taskinstance.py', ]); - const lines = parsedLogs.split('\n'); expect(lines).toHaveLength(7); lines.forEach((line) => expect(line).toContain('taskinstance.py')); }); + + test('parseLogs function with filter on log level and file source', () => { + const { parsedLogs, fileSources } = parseLogs( + mockTaskLog, + null, + 'INFO', + 'taskinstance.py', + ); + + expect(fileSources).toEqual([ + 'dagbag.py', + 'standard_task_runner.py', + 'task_command.py', + 'taskinstance.py', + ]); + const lines = parsedLogs.split('\n'); + expect(lines).toHaveLength(6); + lines.forEach((line) => expect(line).toContain('INFO')); + }); });