From 0f3625524e298660aec079f3b1c950ba08c17b64 Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Thu, 7 Sep 2023 11:20:50 +0200 Subject: [PATCH 01/18] core: Fix fork mode check for "minimize_crash" parameter libFuzzer spawns new sub-processes, if the "minimize_crash" parameter is specified. --- packages/core/options.test.ts | 16 ++++++++++++++++ packages/core/options.ts | 21 +++++++++++++-------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/core/options.test.ts b/packages/core/options.test.ts index cfaa82f4..8261c726 100644 --- a/packages/core/options.test.ts +++ b/packages/core/options.test.ts @@ -22,6 +22,7 @@ import { Options, ParameterResolverIndex, setParameterResolverValue, + spawnsSubprocess, } from "./options"; const commandLineArguments = ParameterResolverIndex.CommandLineArguments; @@ -151,6 +152,21 @@ describe("KeyFormatSource", () => { }); }); +describe("buildLibFuzzerOptions", () => { + describe("spawnsSubprocess", () => { + it("checks if subprocess libFuzzer flags are present", () => { + expect(spawnsSubprocess(["-fork=1"])).toBeTruthy(); + expect(spawnsSubprocess(["-fork=0"])).toBeFalsy(); + expect( + spawnsSubprocess(["abc", "-foo=0", "-fork=0", "-jobs=1"]), + ).toBeTruthy(); + expect(spawnsSubprocess(["-foo=0"])).toBeFalsy(); + expect(spawnsSubprocess(["abc"])).toBeFalsy(); + expect(spawnsSubprocess(["123"])).toBeFalsy(); + }); + }); +}); + function expectDefaultsExceptKeys(options: Options, ...ignore: string[]) { Object.keys(defaultOptions).forEach((key: string) => { if (ignore.includes(key)) return; diff --git a/packages/core/options.ts b/packages/core/options.ts index 7b8e6329..cd12a4cf 100644 --- a/packages/core/options.ts +++ b/packages/core/options.ts @@ -296,14 +296,7 @@ function prepareLibFuzzerArg0(fuzzerOptions: string[]): string { // When we run in a libFuzzer mode that spawns subprocesses, we create a wrapper script // that can be used as libFuzzer's argv[0]. In the fork mode, the main libFuzzer process // uses argv[0] to spawn further processes that perform the actual fuzzing. - const libFuzzerSpawnsProcess = fuzzerOptions.some( - (flag) => - (flag.startsWith("-fork=") && !flag.startsWith("-fork=0")) || - (flag.startsWith("-jobs=") && !flag.startsWith("-jobs=0")) || - (flag.startsWith("-merge=") && !flag.startsWith("-merge=0")), - ); - - if (!libFuzzerSpawnsProcess) { + if (!spawnsSubprocess(fuzzerOptions)) { // Return a fake argv[0] to start the fuzzer if libFuzzer does not spawn new processes. return "unused_arg0_report_a_bug_if_you_see_this"; } else { @@ -312,6 +305,18 @@ function prepareLibFuzzerArg0(fuzzerOptions: string[]): string { } } +// These flags cause libFuzzer to spawn subprocesses. +const SUBPROCESS_FLAGS = ["fork", "jobs", "merge", "minimize_crash"]; + +export function spawnsSubprocess(fuzzerOptions: string[]): boolean { + return fuzzerOptions.some((option) => + SUBPROCESS_FLAGS.some((flag) => { + const name = `-${flag}=`; + return option.startsWith(name) && !option.startsWith("0", name.length); + }), + ); +} + function createWrapperScript(fuzzerOptions: string[]) { const jazzerArgs = process.argv.filter( (arg) => arg !== "--" && fuzzerOptions.indexOf(arg) === -1, From cb0ace861531493af43fd68ebd63dbd3e77bd375 Mon Sep 17 00:00:00 2001 From: Peter Samarin Date: Thu, 7 Sep 2023 12:02:32 +0200 Subject: [PATCH 02/18] test: Speedup fuzztests --- fuzztests/.jazzerjsrc | 2 +- fuzztests/fuzzer.fuzz.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fuzztests/.jazzerjsrc b/fuzztests/.jazzerjsrc index ba9a2f50..8c712681 100644 --- a/fuzztests/.jazzerjsrc +++ b/fuzztests/.jazzerjsrc @@ -1,5 +1,5 @@ { - "fuzzerOptions": ["-use_value_profile=1", "-runs=100000"], + "fuzzerOptions": ["-use_value_profile=1", "-runs=20000"], "includes": ["jazzer.js"], "timeout": 1000, "disableBugDetectors": ['path-traversal'] diff --git a/fuzztests/fuzzer.fuzz.js b/fuzztests/fuzzer.fuzz.js index a4813256..42e206ab 100644 --- a/fuzztests/fuzzer.fuzz.js +++ b/fuzztests/fuzzer.fuzz.js @@ -28,7 +28,7 @@ describe("fuzzer", () => { it.fuzz("use never zero policy", (data) => { const provider = new FuzzedDataProvider(data); - const iterations = provider.consumeIntegralInRange(1, 1 << 16); + const iterations = provider.consumeIntegralInRange(1, 1 << 8); for (let i = 0; i < iterations; i++) { fuzzer.coverageTracker.incrementCounter(0); } From 829a3be7a2b71dd32b2c3b35afc3c2bf4038c801 Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Thu, 14 Sep 2023 18:05:12 +0200 Subject: [PATCH 03/18] test: Set proper excludes in selffuzz tests --- fuzztests/.jazzerjsrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fuzztests/.jazzerjsrc b/fuzztests/.jazzerjsrc index 8c712681..e063f506 100644 --- a/fuzztests/.jazzerjsrc +++ b/fuzztests/.jazzerjsrc @@ -1,6 +1,7 @@ { "fuzzerOptions": ["-use_value_profile=1", "-runs=20000"], "includes": ["jazzer.js"], - "timeout": 1000, + "excludes": ["node_modules"], + "timeout": 3000, "disableBugDetectors": ['path-traversal'] } From 6503ed1ffcba25d77fcef5c011226a4351eec7ae Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Thu, 14 Sep 2023 13:12:51 +0200 Subject: [PATCH 04/18] test: Fix Jest haste map warnings --- end-to-end/package.json | 3 +-- tests/signal_handlers/SIGINT/package.json | 2 +- tests/signal_handlers/SIGSEGV/package.json | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/end-to-end/package.json b/end-to-end/package.json index 5cba6f74..9ea3475a 100644 --- a/end-to-end/package.json +++ b/end-to-end/package.json @@ -1,7 +1,6 @@ { - "name": "jest_typescript_integration", + "name": "end-to-end", "version": "1.0.0", - "description": "An example showing how Jazzer.js integrates with Jest and TypeScript", "scripts": { "build": "tsc", "dryRun": "jest", diff --git a/tests/signal_handlers/SIGINT/package.json b/tests/signal_handlers/SIGINT/package.json index 80715365..5c5cc066 100644 --- a/tests/signal_handlers/SIGINT/package.json +++ b/tests/signal_handlers/SIGINT/package.json @@ -1,5 +1,5 @@ { - "name": "jazzerjs-signal-handler-tests", + "name": "jazzerjs-signal-handler-tests-sigint", "version": "1.0.0", "description": "Tests for the SIGINT signal handler", "scripts": { diff --git a/tests/signal_handlers/SIGSEGV/package.json b/tests/signal_handlers/SIGSEGV/package.json index 80715365..07baf8b7 100644 --- a/tests/signal_handlers/SIGSEGV/package.json +++ b/tests/signal_handlers/SIGSEGV/package.json @@ -1,5 +1,5 @@ { - "name": "jazzerjs-signal-handler-tests", + "name": "jazzerjs-signal-handler-tests-sigsegv", "version": "1.0.0", "description": "Tests for the SIGINT signal handler", "scripts": { From 4a6669afa61048e32d36e38035d89f8d78142473 Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Fri, 8 Sep 2023 11:22:00 +0200 Subject: [PATCH 05/18] test: Extend test ignore pattern --- .npmignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.npmignore b/.npmignore index 64bca1b8..0adab715 100644 --- a/.npmignore +++ b/.npmignore @@ -18,7 +18,7 @@ docs example # Exclude all tests, test helpers and such -*test* +*.test.* # Exclude all TypeScript source files *.ts From 063330148430d0575bcd67de1ebe31db233e9e55 Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Tue, 12 Sep 2023 09:49:15 +0200 Subject: [PATCH 06/18] e2e: Update local run description --- end-to-end/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/end-to-end/README.md b/end-to-end/README.md index 7065e667..a8a97840 100644 --- a/end-to-end/README.md +++ b/end-to-end/README.md @@ -12,11 +12,14 @@ jazzer.js than the other examples. ## Running Locally ```bash +rm -rf node_modules package-lock.json *.tgz ./package-jazzer-js.sh -npm install --save-dev *.tgz +npm install --prefer-online --save-dev *.tgz npx jest ``` _Note_: running just `npm install` may result in caching issues where the contents of the tarballs in this directory are ignored and older versions from -somewhere are used instead. +the cache are used instead. `--prefer-online` forces npm to check for updated +files, which could cause hash mismatches compared to `package-lock.json`. Hence, +remove `package-lock.json` and other dependencies before running the tests. From cf07ce4e73ab87157d128d47d226180981149a20 Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Fri, 8 Sep 2023 11:03:31 +0200 Subject: [PATCH 07/18] core: Provide global Jazzer.js object --- packages/core/api.ts | 1 + packages/core/globals.ts | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 packages/core/globals.ts diff --git a/packages/core/api.ts b/packages/core/api.ts index e4dd8a45..3b28fde6 100644 --- a/packages/core/api.ts +++ b/packages/core/api.ts @@ -29,6 +29,7 @@ export { } from "./callback"; export { addDictionary } from "./dictionary"; export { reportFinding } from "./finding"; +export { getJazzerJsGlobal, setJazzerJsGlobal } from "./globals"; export const guideTowardsEquality = fuzzer.tracer.guideTowardsEquality; export const guideTowardsContainment = fuzzer.tracer.guideTowardsContainment; diff --git a/packages/core/globals.ts b/packages/core/globals.ts new file mode 100644 index 00000000..97f4db1f --- /dev/null +++ b/packages/core/globals.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function setJazzerJsGlobal(name: string, value: unknown) { + // @ts-ignore + if (globalThis.JazzerJS === undefined) { + Object.defineProperty(globalThis, "JazzerJS", { + value: new Map(), + enumerable: true, + configurable: false, + writable: false, + }); + } + // @ts-ignore + globalThis.JazzerJS.set(name, value); +} + +export function getJazzerJsGlobal(name: string): unknown { + // @ts-ignore + return globalThis.JazzerJS?.get(name); +} From eb6a750009839007c50c585a6ee80b538b46bfbd Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Fri, 8 Sep 2023 11:17:53 +0200 Subject: [PATCH 08/18] core: Return instrumentor from init function --- packages/core/core.ts | 47 ++++++++++++++---------- packages/instrumentor/instrument.test.ts | 5 ++- packages/instrumentor/instrument.ts | 25 ++++++------- 3 files changed, 43 insertions(+), 34 deletions(-) diff --git a/packages/core/core.ts b/packages/core/core.ts index 08d60d36..8e291cf0 100644 --- a/packages/core/core.ts +++ b/packages/core/core.ts @@ -57,21 +57,23 @@ declare global { var options: Options; } -export async function initFuzzing(options: Options): Promise { - registerGlobals(options); - - registerInstrumentor( - new Instrumentor( - options.includes, - options.excludes, - options.customHooks, - options.coverage, - options.dryRun, - options.idSyncFile - ? new FileSyncIdStrategy(options.idSyncFile) - : new MemorySyncIdStrategy(), - ), +export async function initFuzzing( + options: Options, + globals?: unknown[], +): Promise { + registerGlobals(options, globals); + + const instrumentor = new Instrumentor( + options.includes, + options.excludes, + options.customHooks, + options.coverage, + options.dryRun, + options.idSyncFile + ? new FileSyncIdStrategy(options.idSyncFile) + : new MemorySyncIdStrategy(), ); + registerInstrumentor(instrumentor); // Dynamic import works only with javascript files, so we have to manually specify the directory with the // transpiled bug detector files. @@ -96,12 +98,17 @@ export async function initFuzzing(options: Options): Promise { await Promise.all(options.customHooks.map(ensureFilepath).map(importModule)); await hooking.hookManager.finalizeHooks(); + + return instrumentor; } -function registerGlobals(options: Options) { - globalThis.Fuzzer = fuzzer.fuzzer; - globalThis.HookManager = hooking.hookManager; - globalThis.options = options; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function registerGlobals(options: Options, globals: any[] = [globalThis]) { + globals.forEach((global) => { + global.Fuzzer = fuzzer.fuzzer; + global.HookManager = hooking.hookManager; + global.options = options; + }); } // Filters out disabled bug detectors and prepares all the others for dynamic import. @@ -194,7 +201,7 @@ export async function startFuzzingNoInit( if (options.sync) { return Promise.resolve().then(() => - Fuzzer.startFuzzing( + fuzzer.fuzzer.startFuzzing( fuzzFn, fuzzerOptions, // In synchronous mode, we cannot use the SIGINT/SIGSEGV handler in Node, @@ -207,7 +214,7 @@ export async function startFuzzingNoInit( } else { process.on("SIGINT", () => signalHandler(0)); process.on("SIGSEGV", () => signalHandler(SIGSEGV)); - return Fuzzer.startFuzzingAsync(fuzzFn, fuzzerOptions); + return fuzzer.fuzzer.startFuzzingAsync(fuzzFn, fuzzerOptions); } } diff --git a/packages/instrumentor/instrument.test.ts b/packages/instrumentor/instrument.test.ts index 9883247d..2a77315e 100644 --- a/packages/instrumentor/instrument.test.ts +++ b/packages/instrumentor/instrument.test.ts @@ -151,7 +151,10 @@ function evalWithInstrumentor( fileName: string, ) { const result = instrumentor.instrument(content, fileName); - const fn = eval(result); + if (!result?.code) { + throw new Error("Instrumentation failed."); + } + const fn = eval(result.code); fn(); } diff --git a/packages/instrumentor/instrument.ts b/packages/instrumentor/instrument.ts index df432f7d..818099e5 100644 --- a/packages/instrumentor/instrument.ts +++ b/packages/instrumentor/instrument.ts @@ -30,6 +30,7 @@ import { functionHooks } from "./plugins/functionHooks"; import { EdgeIdStrategy, MemorySyncIdStrategy } from "./edgeIdStrategy"; import { extractInlineSourceMap, + SourceMap, SourceMapRegistry, toRawSourceMap, } from "./SourceMapRegistry"; @@ -41,6 +42,7 @@ export { FileSyncIdStrategy, MemorySyncIdStrategy, } from "./edgeIdStrategy"; +export { SourceMap } from "./SourceMapRegistry"; export class Instrumentor { constructor( @@ -68,10 +70,10 @@ export class Instrumentor { return this.sourceMapRegistry.installSourceMapSupport(); } - instrument(code: string, filename: string): string { + instrument(code: string, filename: string, sourceMap?: SourceMap) { // Extract inline source map from code string and use it as input source map // in further transformations. - const inputSourceMap = extractInlineSourceMap(code); + const inputSourceMap = sourceMap ?? extractInlineSourceMap(code); const transformations: PluginItem[] = []; const shouldInstrumentFile = this.shouldInstrumentForFuzzing(filename); @@ -100,19 +102,16 @@ export class Instrumentor { this.idStrategy.startForSourceFile(filename); } - const transformedCode = - this.transform( - filename, - code, - transformations, - this.asInputSourceOption(inputSourceMap), - )?.code || code; - + const result = this.transform( + filename, + code, + transformations, + this.asInputSourceOption(inputSourceMap), + ); if (shouldInstrumentFile) { this.idStrategy.commitIdCount(filename); } - - return transformedCode; + return result; } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -204,7 +203,7 @@ export function registerInstrumentor(instrumentor: Instrumentor) { hookRequire( () => true, (code: string, opts: TransformerOptions): string => { - return instrumentor.instrument(code, opts.filename); + return instrumentor.instrument(code, opts.filename)?.code || code; }, // required to allow jest to run typescript files // jest's typescript integration will transform the typescript into javascript before giving it to the From feaf2962baba28e25af663ab7f04f809e6408ad2 Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Fri, 8 Sep 2023 11:22:29 +0200 Subject: [PATCH 09/18] jest: Rewrite Jest integration --- end-to-end/jest.config.ts | 2 +- examples/jest_integration/.jazzerjsrc.json | 8 +- examples/jest_integration/package.json | 2 +- examples/jest_integration/target.js | 8 +- examples/jest_integration/worker.fuzz.js | 5 +- .../My_other_nested_describe/Nested_test/one | 0 .../Hooks/My_other_regression_test/two | 0 .../worker.fuzz/Hooks/My_regression_test/two | 0 .../jest.config.ts | 2 +- fuzztests/package.json | 2 +- packages/bug-detectors/DEVELOPMENT.md | 21 + .../internal/prototype-pollution.ts | 78 ++- packages/jest-runner/config.test.ts | 4 +- packages/jest-runner/config.ts | 8 + packages/jest-runner/errorUtils.test.ts | 6 +- packages/jest-runner/errorUtils.ts | 10 +- packages/jest-runner/fuzz.test.ts | 187 ++++--- packages/jest-runner/fuzz.ts | 236 +++++---- .../jest-runner/globalsInterceptor.test.ts | 76 +++ packages/jest-runner/globalsInterceptor.ts | 65 +++ packages/jest-runner/index.ts | 125 ++--- packages/jest-runner/readme.md | 2 +- .../jest-runner/testStateInterceptor.test.ts | 114 +++++ packages/jest-runner/testStateInterceptor.ts | 104 ++++ .../transformerInterceptor.test.ts | 259 ++++++++++ .../jest-runner/transformerInterceptor.ts | 193 +++++++ packages/jest-runner/worker.ts | 483 ------------------ .../command-injection/package.json | 2 +- tests/bug-detectors/general/package.json | 2 +- .../bug-detectors/prototype-pollution.test.js | 4 +- .../prototype-pollution/package.json | 2 +- tests/code_coverage/coverage.test.js | 13 +- tests/code_coverage/package.json | 3 +- .../fuzz+lib+codeCoverage-fuzz.json | 10 +- .../fuzz+lib+otherCodeCoverage-fuzz.json | 10 +- .../sample_fuzz_test/package.json | 2 +- tests/helpers.js | 23 +- tests/jest_integration/integration.test.js | 301 +++++++++-- .../jest_project/integration.fuzz.js | 58 +++ .../jest_project/package.json | 7 +- .../jest_project/run-mode-only.fuzz.js | 25 + tests/jest_integration/jest_project/target.js | 27 +- .../jest_project_with_single_test/.gitignore | 3 + .../integration.fuzz.js | 23 + .../package.json | 29 ++ tests/signal_handlers/SIGINT/fuzz.js | 4 +- tests/signal_handlers/SIGINT/package.json | 2 +- tests/signal_handlers/SIGSEGV/fuzz.js | 6 +- tests/signal_handlers/SIGSEGV/package.json | 2 +- tests/signal_handlers/signal_handlers.test.js | 4 +- 50 files changed, 1661 insertions(+), 901 deletions(-) delete mode 100644 examples/jest_integration/worker.fuzz/Hooks/My_nested_describe/My_other_nested_describe/Nested_test/one delete mode 100644 examples/jest_integration/worker.fuzz/Hooks/My_other_regression_test/two delete mode 100644 examples/jest_integration/worker.fuzz/Hooks/My_regression_test/two create mode 100644 packages/jest-runner/globalsInterceptor.test.ts create mode 100644 packages/jest-runner/globalsInterceptor.ts create mode 100644 packages/jest-runner/testStateInterceptor.test.ts create mode 100644 packages/jest-runner/testStateInterceptor.ts create mode 100644 packages/jest-runner/transformerInterceptor.test.ts create mode 100644 packages/jest-runner/transformerInterceptor.ts delete mode 100644 packages/jest-runner/worker.ts create mode 100644 tests/jest_integration/jest_project/run-mode-only.fuzz.js create mode 100644 tests/jest_integration/jest_project_with_single_test/.gitignore create mode 100644 tests/jest_integration/jest_project_with_single_test/integration.fuzz.js create mode 100644 tests/jest_integration/jest_project_with_single_test/package.json diff --git a/end-to-end/jest.config.ts b/end-to-end/jest.config.ts index 59c643db..861b211a 100644 --- a/end-to-end/jest.config.ts +++ b/end-to-end/jest.config.ts @@ -28,7 +28,7 @@ const config: Config = { color: "cyan", }, preset: "ts-jest", - runner: "@jazzer.js/jest-runner", + testRunner: "@jazzer.js/jest-runner", testEnvironment: "node", testMatch: ["/*.fuzz.[jt]s"], }, diff --git a/examples/jest_integration/.jazzerjsrc.json b/examples/jest_integration/.jazzerjsrc.json index 1c0e4ace..6ede3850 100644 --- a/examples/jest_integration/.jazzerjsrc.json +++ b/examples/jest_integration/.jazzerjsrc.json @@ -1,5 +1,9 @@ { "includes": ["target", "integration.fuzz", "worker.fuzz"], - "excludes": ["node_modules"], - "fuzzerOptions": ["-rss_limit_mb=16000", "-runs=100000"] + "excludes": ["@babel"], + "fuzzerOptions": [ + "-rss_limit_mb=16000", + "-use_value_profile=1", + "-runs=1000000" + ] } diff --git a/examples/jest_integration/package.json b/examples/jest_integration/package.json index 19e13759..2d6a7170 100644 --- a/examples/jest_integration/package.json +++ b/examples/jest_integration/package.json @@ -20,7 +20,7 @@ "displayName": "test" }, { - "runner": "@jazzer.js/jest-runner", + "testRunner": "@jazzer.js/jest-runner", "displayName": { "name": "Jazzer.js", "color": "cyan" diff --git a/examples/jest_integration/target.js b/examples/jest_integration/target.js index 3413eaf8..fb957f65 100644 --- a/examples/jest_integration/target.js +++ b/examples/jest_integration/target.js @@ -19,14 +19,10 @@ */ const fuzzMe = function (data) { const s = data.toString(); - if (s.length !== 16) { + if (s.length !== 7) { return; } - if ( - s.slice(0, 8) === "Awesome " && - s.slice(8, 15) === "Fuzzing" && - s[15] === "!" - ) { + if (s.slice(0, 7) === "Awesome") { throw Error("Welcome to Awesome Fuzzing!"); } }; diff --git a/examples/jest_integration/worker.fuzz.js b/examples/jest_integration/worker.fuzz.js index e95c2f42..db5b7f53 100644 --- a/examples/jest_integration/worker.fuzz.js +++ b/examples/jest_integration/worker.fuzz.js @@ -24,7 +24,7 @@ const addCallLog = (uniqueId) => { beforeAll(() => { return new Promise((resolve) => { - setTimeout(() => { + setImmediate(() => { addCallLog("Top-level beforeAll"); resolve(undefined); }, 100); @@ -33,8 +33,7 @@ beforeAll(() => { describe("Hooks", () => { beforeAll((done) => { - const callLog = "My describe: beforeAll"; - addCallLog(callLog); + addCallLog("My describe: beforeAll"); done(); }); diff --git a/examples/jest_integration/worker.fuzz/Hooks/My_nested_describe/My_other_nested_describe/Nested_test/one b/examples/jest_integration/worker.fuzz/Hooks/My_nested_describe/My_other_nested_describe/Nested_test/one deleted file mode 100644 index e69de29b..00000000 diff --git a/examples/jest_integration/worker.fuzz/Hooks/My_other_regression_test/two b/examples/jest_integration/worker.fuzz/Hooks/My_other_regression_test/two deleted file mode 100644 index e69de29b..00000000 diff --git a/examples/jest_integration/worker.fuzz/Hooks/My_regression_test/two b/examples/jest_integration/worker.fuzz/Hooks/My_regression_test/two deleted file mode 100644 index e69de29b..00000000 diff --git a/examples/jest_typescript_integration/jest.config.ts b/examples/jest_typescript_integration/jest.config.ts index e8b6272b..4a49ce44 100644 --- a/examples/jest_typescript_integration/jest.config.ts +++ b/examples/jest_typescript_integration/jest.config.ts @@ -29,7 +29,7 @@ const config: Config = { color: "cyan", }, preset: "ts-jest", - runner: "@jazzer.js/jest-runner", + testRunner: "@jazzer.js/jest-runner", testEnvironment: "node", testMatch: ["/*.fuzz.[jt]s"], }, diff --git a/fuzztests/package.json b/fuzztests/package.json index d4695f80..d918638c 100644 --- a/fuzztests/package.json +++ b/fuzztests/package.json @@ -13,7 +13,7 @@ "jest": { "projects": [ { - "runner": "@jazzer.js/jest-runner", + "testRunner": "@jazzer.js/jest-runner", "displayName": { "name": "Jazzer.js", "color": "cyan" diff --git a/packages/bug-detectors/DEVELOPMENT.md b/packages/bug-detectors/DEVELOPMENT.md index 05946627..0b5419fa 100644 --- a/packages/bug-detectors/DEVELOPMENT.md +++ b/packages/bug-detectors/DEVELOPMENT.md @@ -160,3 +160,24 @@ bugDetectorConfigurations.set("", config); See the `PrototypePollutionConfig` in [Prototype Pollution](internal/prototype-pollution.ts) bug detector for an example. + +## Accessing the global context in Jest tests + +If the bug detectors need to access objects in the global context, they have to +take special care for Jest tests. Internally Jest runs the tests using +`vm.runInContext()`. Thus, the global objects (e.g. `Object`, `Array`, +`Function`, etc.) used in that context are not the same (as in "by reference") +as the ones used by the bug detectors. To deal with this, the bug detectors can +access the `vmContext` object, from which global objects can be accessed as +follows: + +```typescript +import * as vm from "vm"; +const vmContext = getJazzerJsGlobal("vmContext") as vm.Context; +``` + +Objects from the VM context can be extracted as follows: + +```typescript +BASIC_OBJECTS = vm.runInContext('[{},[],"",42,true,()=>{}]', vmContext); +``` diff --git a/packages/bug-detectors/internal/prototype-pollution.ts b/packages/bug-detectors/internal/prototype-pollution.ts index 672d1200..187ce961 100644 --- a/packages/bug-detectors/internal/prototype-pollution.ts +++ b/packages/bug-detectors/internal/prototype-pollution.ts @@ -22,9 +22,11 @@ import { addDictionary, registerInstrumentationPlugin, instrumentationGuard, + getJazzerJsGlobal, } from "@jazzer.js/core"; import { bugDetectorConfigurations } from "../configuration"; +import * as vm from "vm"; // Allow the user to configure this bug detector in the custom-hooks file (if any). class PrototypePollutionConfig { @@ -233,7 +235,7 @@ registerInstrumentationPlugin((): PluginTarget => { // These objects will be used to detect prototype pollution. // Using global arrays for performance reasons. -const BASIC_OBJECTS = [ +let BASIC_OBJECTS = [ {}, [], "", @@ -265,10 +267,29 @@ type ProtoSnapshot = { // Compute prototype snapshots of each selected basic object before any fuzz tests are run. // These snapshots are used to detect prototype pollution after each fuzz test. -const BASIC_PROTO_SNAPSHOTS = computeBasicPrototypeSnapshots(); +let BASIC_PROTO_SNAPSHOTS = computeBasicPrototypeSnapshots(BASIC_OBJECTS); + +if (getJazzerJsGlobal("vmContext")) { + const vmContext = getJazzerJsGlobal("vmContext") as vm.Context; + Object.defineProperty(vmContext, "PrototypePollution", { + value: PrototypePollution, + writable: false, + enumerable: true, + configurable: false, + }); + // Get the basic objects from the vm context. + // TODO: This should be updated every time Jest sacks the current vm context. + BASIC_OBJECTS = vm.runInContext('[{},[],"",42,true,()=>{}]', vmContext); + BASIC_PROTO_SNAPSHOTS = computeBasicPrototypeSnapshots(BASIC_OBJECTS); +} -function computeBasicPrototypeSnapshots(): BasicProtoSnapshots { - return BASIC_OBJECTS.map(getProtoSnapshot); +export function computeBasicPrototypeSnapshots( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + objects: any[], +): BasicProtoSnapshots { + // These objects will be used to detect prototype pollution. + // Using global arrays for performance reasons. + return objects.map(getProtoSnapshot); } /** @@ -299,30 +320,35 @@ function getProtoSnapshot(obj: any): ProtoSnapshot { }; } -registerAfterEachCallback( - function detectPrototypePollutionOfBasicObjects(): void { - const currentProtoSnapshots = computeBasicPrototypeSnapshots(); - // Compare the current prototype snapshots of basic objects to the original ones. - for (let i = 0; i < BASIC_PROTO_SNAPSHOTS.length; i++) { - if (!currentProtoSnapshots[i]) { - reportFinding( - `Prototype Pollution: Prototype of ${BASIC_OBJECT_NAMES[i]} changed.`, - ); - return; - } - const equalityResult = protoSnapshotsEqual( - BASIC_PROTO_SNAPSHOTS[i], - currentProtoSnapshots[i], +registerAfterEachCallback(function detectPrototypePollution() { + detectPrototypePollutionOfBasicObjects(BASIC_PROTO_SNAPSHOTS, BASIC_OBJECTS); +}); + +function detectPrototypePollutionOfBasicObjects( + initialSnapshots: ProtoSnapshot[], + objects: unknown[], +): void { + const currentProtoSnapshots = computeBasicPrototypeSnapshots(objects); + // Compare the current prototype snapshots of basic objects to the original ones. + for (let i = 0; i < initialSnapshots.length; i++) { + if (!currentProtoSnapshots[i]) { + reportFinding( + `Prototype Pollution: Prototype of ${BASIC_OBJECT_NAMES[i]} changed.`, ); - if (equalityResult) { - reportFinding( - `Prototype Pollution: Prototype of ${BASIC_OBJECT_NAMES[i]} changed. ${equalityResult}`, - ); - return; - } + return; } - }, -); + const equalityResult = protoSnapshotsEqual( + initialSnapshots[i], + currentProtoSnapshots[i], + ); + if (equalityResult) { + reportFinding( + `Prototype Pollution: Prototype of ${BASIC_OBJECT_NAMES[i]} changed. ${equalityResult}`, + ); + return; + } + } +} // There are two main ways to pollute a prototype of an object: // 1. Changing a prototype's property using __proto__ diff --git a/packages/jest-runner/config.test.ts b/packages/jest-runner/config.test.ts index 411c11e7..2b45d435 100644 --- a/packages/jest-runner/config.test.ts +++ b/packages/jest-runner/config.test.ts @@ -15,12 +15,12 @@ */ import { defaultOptions } from "@jazzer.js/core"; -import { loadConfig } from "./config"; +import { loadConfig, TIMEOUT_PLACEHOLDER } from "./config"; describe("Config", () => { describe("loadConfig", () => { it("return default configuration if nothing found", () => { - const defaults = { ...defaultOptions }; + const defaults = { ...defaultOptions, timeout: TIMEOUT_PLACEHOLDER }; defaults.mode = "regression"; expect(loadConfig()).toEqual(defaults); }); diff --git a/packages/jest-runner/config.ts b/packages/jest-runner/config.ts index 1f5e32d9..08ae7b8a 100644 --- a/packages/jest-runner/config.ts +++ b/packages/jest-runner/config.ts @@ -22,6 +22,8 @@ import { ParameterResolverIndex, } from "@jazzer.js/core"; +export const TIMEOUT_PLACEHOLDER = Number.MIN_SAFE_INTEGER; + // Lookup `Options` via the `.jazzerjsrc` configuration files. export function loadConfig( options: Partial = {}, @@ -29,6 +31,12 @@ export function loadConfig( ): Options { const result = cosmiconfigSync(optionsKey).search(); const config = result?.config ?? {}; + // If no timeout is specified, use a placeholder value so that no + // default timeout is used. Afterwards remove the placeholder value, + // if not already overwritten by the user. + if (config.timeout === undefined) { + config.timeout = TIMEOUT_PLACEHOLDER; + } // Jazzer.js normally runs in "fuzzing" mode, but, // if not specified otherwise, Jest uses "regression" mode. if (!config.mode) { diff --git a/packages/jest-runner/errorUtils.test.ts b/packages/jest-runner/errorUtils.test.ts index a78ebfe6..e8db960f 100644 --- a/packages/jest-runner/errorUtils.test.ts +++ b/packages/jest-runner/errorUtils.test.ts @@ -42,9 +42,13 @@ describe("ErrorUtils", () => { describe("clean up Jest runner frames", () => { it("in errors", () => { expect(cleanupJestError(undefined)).toBeUndefined(); - expect(cleanupJestError(error)?.stack).toMatch(`Error: + const result = cleanupJestError(error); + expect(result instanceof Error).toBeTruthy(); + if (result instanceof Error) { + expect(result.stack).toMatch(`Error: at /jest_integration/integration.fuzz.js:27:3 `); + } }); it("in stacks", () => { diff --git a/packages/jest-runner/errorUtils.ts b/packages/jest-runner/errorUtils.ts index cb7a5b6a..273600dd 100644 --- a/packages/jest-runner/errorUtils.ts +++ b/packages/jest-runner/errorUtils.ts @@ -14,13 +14,13 @@ * limitations under the License. */ -export const cleanupJestError = ( - error: Error | undefined, -): Error | undefined => { +export const cleanupJestError = (error: Error | unknown): Error | unknown => { if (error == undefined) { - return error; + return undefined; + } + if (error instanceof Error) { + error.stack = cleanupJestRunnerStack(error.stack); } - error.stack = cleanupJestRunnerStack(error.stack); return error; }; diff --git a/packages/jest-runner/fuzz.test.ts b/packages/jest-runner/fuzz.test.ts index 09135170..49af4526 100644 --- a/packages/jest-runner/fuzz.test.ts +++ b/packages/jest-runner/fuzz.test.ts @@ -15,6 +15,19 @@ */ // Mock Corpus class so that no local directories are created during test. +import fs from "fs"; +import * as tmp from "tmp"; +import { Circus, Global } from "@jest/types"; +import { Corpus } from "./corpus"; +import { + fuzz, + FuzzerError, + FuzzTest, + JestTestMode, + runInRegressionMode, +} from "./fuzz"; +import { Options, startFuzzingNoInit } from "@jazzer.js/core"; // Cleanup created files on exit + const inputsPathsMock = jest.fn(); jest.mock("./corpus", () => { return { @@ -25,29 +38,17 @@ jest.mock("./corpus", () => { }); // Mock core package to intercept calls to startFuzzing. -const startFuzzingMock = jest.fn(); const skipMock = jest.fn(); jest.mock("@jazzer.js/core", () => { return { - startFuzzingNoInit: startFuzzingMock, + startFuzzingNoInit: jest.fn(), + wrapFuzzFunctionForBugDetection: (fn: object) => fn, }; }); // Mock console error logs const consoleErrorMock = jest.spyOn(console, "error").mockImplementation(); -import fs from "fs"; -import * as tmp from "tmp"; -import { Global } from "@jest/types"; -import { Corpus } from "./corpus"; -import { - FuzzerError, - FuzzerStartError, - runInFuzzingMode, - runInRegressionMode, -} from "./fuzz"; -import { Options } from "@jazzer.js/core"; - // Cleanup created files on exit tmp.setGracefulCleanup(); @@ -57,26 +58,24 @@ describe("fuzz", () => { }); describe("runInFuzzingMode", () => { - it("execute only one fuzz target function", async () => { - const testFn = jest.fn(); - const corpus = new Corpus("", []); - const options = { - fuzzerOptions: ["--runs=1"], - } as Options; - - // First call should start the fuzzer + it("execute test matching original test name pattern", async () => { await withMockTest(() => { - runInFuzzingMode("first", testFn, corpus, options); + const originalTestNamePattern = jest + .fn() + .mockReturnValue(/^myFuzzTest$/); + invokeFuzz({ originalTestNamePattern })("myFuzzTest", jest.fn()); }); - expect(startFuzzingMock).toBeCalledTimes(1); + expect(startFuzzingNoInit).toBeCalledTimes(1); + }); - // Should fail to start the fuzzer a second time - await expect( - withMockTest(() => { - runInFuzzingMode("second", testFn, corpus, options); - }), - ).rejects.toThrow(FuzzerStartError); - expect(startFuzzingMock).toBeCalledTimes(1); + it("skip test not matching original test name pattern", async () => { + await withMockTest(() => { + const originalTestNamePattern = jest + .fn() + .mockReturnValue(/^not_existing$/); + invokeFuzz({ originalTestNamePattern })("myFuzzTest", jest.fn()); + }); + expect(startFuzzingNoInit).toBeCalledTimes(0); }); }); @@ -86,7 +85,14 @@ describe("fuzz", () => { const corpus = new Corpus("", []); const testFn = jest.fn(); await withMockTest(() => { - runInRegressionMode("fuzz", testFn, corpus, 1000); + runInRegressionMode( + "fuzz", + testFn, + corpus, + {} as Options, + globalThis as Global.Global, + "standard", + ); }); inputPaths.forEach(([name]) => { expect(testFn).toHaveBeenCalledWith(Buffer.from(name)); @@ -103,7 +109,9 @@ describe("fuzz", () => { done(); }, mockDefaultCorpus(), - 1000, + {} as Options, + globalThis as Global.Global, + "standard", ); }); expect(called).toBeTruthy(); @@ -123,81 +131,75 @@ describe("fuzz", () => { }); }, mockDefaultCorpus(), - 1000, + {} as Options, + globalThis as Global.Global, + "standard", ); }); expect(called).toBeTruthy(); }); - it("fail on timeout", async () => { - const rejects = await expect( + it("fail on done callback with async result", async () => { + const rejects = expect( withMockTest(() => { runInRegressionMode( "fuzz", - () => { + // Parameters needed to pass in done callback. + (ignored: Buffer, ignored2: (e?: Error) => void) => { return new Promise(() => { - // do nothing to trigger timeout + // promise is ignored due to done callback }); }, mockDefaultCorpus(), - 100, + {} as Options, + globalThis as Global.Global, + "standard", ); }), ).rejects; await rejects.toThrow(FuzzerError); - await rejects.toThrowError(new RegExp(".*Timeout.*")); + await rejects.toThrowError(new RegExp(".*async or done.*")); }); - it("fail on done callback with async result", async () => { - const rejects = await expect( + // This test is disabled as it prints an additional error message to the console, + // which breaks the CI pipeline. + it.skip("print error on multiple calls to done callback", async () => { + await new Promise((resolve, reject) => { withMockTest(() => { runInRegressionMode( "fuzz", - // Parameters needed to pass in done callback. - (ignored: Buffer, ignored2: (e?: Error) => void) => { - return new Promise(() => { - // promise is ignored due to done callback - }); + (ignored: Buffer, done: (e?: Error) => void) => { + done(); + done(); + // Use another promise to stop test from finishing too fast. + resolve("done called multiple times"); }, mockDefaultCorpus(), - 100, + {} as Options, + globalThis as Global.Global, + "standard", ); - }), - ).rejects; - await rejects.toThrow(FuzzerError); - await rejects.toThrowError(new RegExp(".*async or done.*")); - }); - - it("print error on multiple calls to done callback", async () => { - await new Promise((resolve) => { - expect( - withMockTest(() => { - runInRegressionMode( - "fuzz", - (ignored: Buffer, done: (e?: Error) => void) => { - done(); - done(); - // Use another promise to stop test from finishing too fast. - resolve("done called multiple times"); - }, - mockDefaultCorpus(), - 100, - ); - }), - ); + }).then(resolve, reject); }); expect(consoleErrorMock).toHaveBeenCalledTimes(1); }); - it("skips tests without seed files", async () => { + it("always call tests with empty input", async () => { mockInputPaths(); const corpus: Corpus = new Corpus("", []); const testFn = jest.fn(); await withMockTest(() => { - runInRegressionMode("fuzz", testFn, corpus, 1000); + runInRegressionMode( + "fuzz", + testFn, + corpus, + {} as Options, + globalThis as Global.Global, + "standard", + ); }); - expect(testFn).not.toBeCalled(); - expect(skipMock).toHaveBeenCalled(); + expect(testFn).toHaveBeenCalledWith(Buffer.from("")); + expect(skipMock).not.toHaveBeenCalled(); }); }); }); @@ -256,3 +258,38 @@ const mockDefaultCorpus = () => { mockInputPaths("seed"); return new Corpus("", []); }; + +function invokeFuzz( + params: Partial<{ + globals: Global.Global; + testFile: string; + fuzzingConfig: Options; + currentTestState: () => Circus.DescribeBlock | undefined; + currentTestTimeout: () => number | undefined; + originalTestNamePattern: () => RegExp | undefined; + mode: JestTestMode; + }>, +): FuzzTest { + const paramsWithDefaults = { + globals: globalThis as Global.Global, + testFile: "testfile", + fuzzingConfig: { + fuzzerOptions: [""], + mode: "fuzzing", + } as Options, + currentTestState: jest.fn().mockReturnValue({}), + currentTestTimeout: jest.fn().mockReturnValue(undefined), + originalTestNamePattern: jest.fn().mockReturnValue(undefined), + mode: "standard" as JestTestMode, + ...params, + }; + return fuzz( + paramsWithDefaults.globals, + paramsWithDefaults.testFile, + paramsWithDefaults.fuzzingConfig, + paramsWithDefaults.currentTestState, + paramsWithDefaults.currentTestTimeout, + paramsWithDefaults.originalTestNamePattern, + paramsWithDefaults.mode, + ); +} diff --git a/packages/jest-runner/fuzz.ts b/packages/jest-runner/fuzz.ts index 8098522e..5710cace 100644 --- a/packages/jest-runner/fuzz.ts +++ b/packages/jest-runner/fuzz.ts @@ -14,106 +14,119 @@ * limitations under the License. */ -import { Global } from "@jest/types"; +import { Circus, Global } from "@jest/types"; import { FuzzTarget, FuzzTargetAsyncOrValue, FuzzTargetCallback, } from "@jazzer.js/fuzzer"; -import { loadConfig } from "./config"; -import { JazzerWorker } from "./worker"; +import { TIMEOUT_PLACEHOLDER } from "./config"; import { Corpus } from "./corpus"; -import * as circus from "jest-circus"; import * as fs from "fs"; import { removeTopFramesFromError } from "./errorUtils"; import { + defaultOptions, Options, startFuzzingNoInit, wrapFuzzFunctionForBugDetection, } from "@jazzer.js/core"; -// Globally track when the fuzzer is started in fuzzing mode. -let fuzzerStarted = false; - // Indicate that something went wrong executing the fuzzer. export class FuzzerError extends Error {} -// Error indicating that the fuzzer was already started. -export class FuzzerStartError extends FuzzerError {} - -// Use Jests global object definition. -const g = globalThis as unknown as Global.Global; - export type FuzzTest = ( name: Global.TestNameLike, fn: FuzzTarget, timeout?: number, ) => void; -export const skip: FuzzTest = (name) => { - g.test.skip(toTestName(name), () => { - return; - }); -}; +export const skip: (globals: Global.Global) => FuzzTest = + (globals: Global.Global) => (name) => { + globals.test.skip(toTestName(name), () => { + return; + }); + }; -export const fuzz: FuzzTest = (name, fn, timeout) => { - const testName = toTestName(name); +export type JestTestMode = "skip" | "only" | "standard"; - // Request the current test file path from the worker to create appropriate - // corpus directory hierarchies. It is set by the worker that imports the - // actual test file and changes during execution of multiple test files. - const testFile = JazzerWorker.currentTestPath; +export function fuzz( + globals: Global.Global, + testFile: string, + fuzzingConfig: Options, + currentTestState: () => Circus.DescribeBlock | undefined, + currentTestTimeout: () => number | undefined, + originalTestNamePattern: () => RegExp | undefined, + mode: JestTestMode, +): FuzzTest { + return (name, fn, timeout) => { + // Deep clone the fuzzing config, so that each test can modify it without + // affecting other tests, e.g. set a test specific timeout. + const localConfig = JSON.parse(JSON.stringify(fuzzingConfig)); - // Build up the names of test block elements (describe, test, it) pointing - // to the currently executed fuzz function, based on the circus runner state. - // The used state changes during test file import but, at this point, - // points to the element containing the fuzz function. - const testStatePath = currentTestStatePath(testName); + const state = currentTestState(); + if (!state) { + throw new Error("No test state found"); + } - const corpus = new Corpus(testFile, testStatePath); + // Add tests that don't match the test name pattern as skipped, so that + // only the requested tests are executed. + const testStatePath = currentTestStatePath(toTestName(name), state); + const testNamePattern = originalTestNamePattern(); + const skip = + testStatePath !== undefined && + testNamePattern != undefined && + !testNamePattern.test(testStatePath.join(" ")); + if (skip) { + globals.test.skip(name, () => { + // Ignore + }); + return; + } - const fuzzingConfig = loadConfig(); + const corpus = new Corpus(testFile, testStatePath); - // Timeout priority is: test timeout > config timeout > default timeout. - if (!timeout) { - timeout = fuzzingConfig.timeout; - } else { - fuzzingConfig.timeout = timeout; - } + // Timeout priority is: + // 1. Use timeout directly defined in test function + // 2. Use timeout defined in fuzzing config + // 3. Use jest timeout + if (timeout != undefined) { + localConfig.timeout = timeout; + } else { + const jestTimeout = currentTestTimeout(); + if (jestTimeout != undefined && localConfig.timeout == undefined) { + localConfig.timeout = jestTimeout; + } else if (localConfig.timeout === TIMEOUT_PLACEHOLDER) { + localConfig.timeout = defaultOptions.timeout; + } + } - const wrappedFn = wrapFuzzFunctionForBugDetection(fn); + const wrappedFn = wrapFuzzFunctionForBugDetection(fn); - if (fuzzingConfig.mode === "regression") { - runInRegressionMode(name, wrappedFn, corpus, timeout); - } else if (fuzzingConfig.mode === "fuzzing") { - runInFuzzingMode(name, wrappedFn, corpus, fuzzingConfig); - } else { - throw new Error(`Unknown mode ${fuzzingConfig.mode}`); - } -}; + if (localConfig.mode === "regression") { + runInRegressionMode(name, wrappedFn, corpus, localConfig, globals, mode); + } else if (localConfig.mode === "fuzzing") { + runInFuzzingMode(name, wrappedFn, corpus, localConfig, globals, mode); + } else { + throw new Error(`Unknown mode ${localConfig.mode}`); + } + }; +} export const runInFuzzingMode = ( name: Global.TestNameLike, fn: FuzzTarget, corpus: Corpus, - config: Options, + options: Options, + globals: Global.Global, + mode: JestTestMode, ) => { - config.fuzzerOptions.unshift(corpus.seedInputsDirectory); - config.fuzzerOptions.unshift(corpus.generatedInputsDirectory); - config.fuzzerOptions.push("-artifact_prefix=" + corpus.seedInputsDirectory); - g.test(name, () => { - // Fuzzing is only allowed to start once in a single nodejs instance. - if (fuzzerStarted) { - const message = `Fuzzer already started. Please provide single fuzz test using --testNamePattern. Skipping test "${toTestName( - name, - )}"`; - const error = new FuzzerStartError(message); - // Remove stack trace as it is shown in the CLI / IDE and points to internal code. - error.stack = undefined; - throw error; - } - fuzzerStarted = true; - return startFuzzingNoInit(fn, config); + handleMode(mode, globals.test)(name, () => { + options.fuzzerOptions.unshift(corpus.seedInputsDirectory); + options.fuzzerOptions.unshift(corpus.generatedInputsDirectory); + options.fuzzerOptions.push( + "-artifact_prefix=" + corpus.seedInputsDirectory, + ); + return startFuzzingNoInit(fn, options); }); }; @@ -121,55 +134,39 @@ export const runInRegressionMode = ( name: Global.TestNameLike, fn: FuzzTarget, corpus: Corpus, - timeout: number, + options: Options, + globals: Global.Global, + mode: JestTestMode, ) => { - g.describe(name, () => { - const inputsPaths = corpus.inputsPaths(); - - // Mark fuzz tests with empty inputs as skipped to suppress Jest error. - if (inputsPaths.length === 0) { - g.test.skip(name, () => { - return; + handleMode(mode, globals.describe)(name, () => { + function executeTarget(content: Buffer) { + return new Promise((resolve, reject) => { + // Fuzz test expects a done callback, if more than one parameter is specified. + if (fn.length > 1) { + doneCallbackPromise(fn, content, resolve, reject); + } else { + // Support sync and async fuzz tests. + Promise.resolve() + .then(() => (fn as FuzzTargetAsyncOrValue)(content)) + .then(resolve, reject); + } }); - return; } + // Always execute target function with an empty buffer. + globals.test( + "", + async () => executeTarget(Buffer.from("")), + options.timeout, + ); + // Execute the fuzz test with each input file as no libFuzzer is required. - // Custom hooks are already registered via the jest-runner. - inputsPaths.forEach(([seed, path]) => { - g.test(seed, async () => { - const content = await fs.promises.readFile(path); - let timeoutID: NodeJS.Timeout; - return new Promise((resolve, reject) => { - // Register a timeout for every fuzz test function invocation. - timeoutID = setTimeout(() => { - reject(new FuzzerError(`Timeout reached ${timeout}`)); - }, timeout); - - // Fuzz test expects a done callback, if more than one parameter is specified. - if (fn.length > 1) { - return doneCallbackPromise(fn, content, resolve, reject); - } else { - // Support sync and async fuzz tests. - return Promise.resolve() - .then(() => (fn as FuzzTargetAsyncOrValue)(content)) - .then(resolve, reject); - } - }).then( - (value: unknown) => { - // Remove timeout to enable clean shutdown. - timeoutID?.unref?.(); - clearTimeout(timeoutID); - return value; - }, - (error: unknown) => { - // Remove timeout to enable clean shutdown. - timeoutID?.unref?.(); - clearTimeout(timeoutID); - throw error; - }, - ); - }); + corpus.inputsPaths().forEach(([seed, path]) => { + globals.test( + seed, + async () => executeTarget(await fs.promises.readFile(path)), + options.timeout, + ); }); }); }; @@ -204,6 +201,7 @@ const doneCallbackPromise = ( // Expecting a done callback, but returning a promise, is invalid. This is // already prevented by TypeScript, but we should still check for this // situation due to untyped JavaScript fuzz tests. + // Ignore other return values, as they are not relevant for the fuzz test. // @ts-ignore if (result && typeof result.then === "function") { reject( @@ -217,6 +215,19 @@ const doneCallbackPromise = ( } }; +function handleMode( + mode: JestTestMode, + test: Global.ItConcurrent | Global.Describe, +) { + switch (mode) { + case "skip": + return test.skip; + case "only": + return test.only; + } + return test; +} + const toTestName = (name: Global.TestNameLike): string => { switch (typeof name) { case "string": @@ -231,10 +242,13 @@ const toTestName = (name: Global.TestNameLike): string => { throw new FuzzerError(`Invalid test name "${name}"`); }; -const currentTestStatePath = (testName: string): string[] => { - const elements = [testName]; - let describeBlock = circus.getState().currentDescribeBlock; - while (describeBlock !== circus.getState().rootDescribeBlock) { +const currentTestStatePath = ( + name: string, + state: Circus.DescribeBlock, +): string[] => { + const elements = [name]; + let describeBlock = state; + while (describeBlock.parent) { elements.unshift(describeBlock.name); if (describeBlock.parent) { describeBlock = describeBlock.parent; diff --git a/packages/jest-runner/globalsInterceptor.test.ts b/packages/jest-runner/globalsInterceptor.test.ts new file mode 100644 index 00000000..ff4abc83 --- /dev/null +++ b/packages/jest-runner/globalsInterceptor.test.ts @@ -0,0 +1,76 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Disable any checks for this file, since it makes mocking much easier. +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { interceptGlobals } from "./globalsInterceptor"; +import { fuzz } from "./fuzz"; + +const internalFuzz = jest.fn(); +jest.mock("./fuzz", () => ({ + fuzz: jest.fn().mockImplementation(() => { + return internalFuzz; + }), +})); + +describe("Globals interceptor", () => { + it("extend Jest global test with fuzz function", () => { + const originalSetGlobalsForRuntime = jest.fn(); + const runtime = { + setGlobalsForRuntime: originalSetGlobalsForRuntime, + } as any; + const testPath = "testPath"; + const jazzerConfig = {} as any; + const testState = { + currentTestState: jest.fn(), + currentTestTimeout: jest.fn(), + originalTestNamePattern: jest.fn(), + }; + + const globals = { + it: { + skip: {}, + only: {}, + }, + } as any; + + interceptGlobals(runtime, testPath, jazzerConfig, testState); + + runtime.setGlobalsForRuntime(globals); + + expect(Object.keys(globals.it)).toHaveLength(3); + expect(globals.it.fuzz).toBe(internalFuzz); + expect(globals.it.skip.fuzz).toBe(internalFuzz); + expect(globals.it.only.fuzz).toBe(internalFuzz); + + expect(originalSetGlobalsForRuntime).toHaveBeenCalledWith(globals); + + const fuzzMock = fuzz as jest.Mock; + expect(fuzzMock).toHaveBeenCalledTimes(3); + expect(fuzzMock).toHaveBeenCalledWith( + globals, + testPath, + jazzerConfig, + testState.currentTestState, + testState.currentTestTimeout, + testState.originalTestNamePattern, + "standard", + ); + expect(fuzzMock.mock.calls[1][6]).toBe("skip"); + expect(fuzzMock.mock.calls[2][6]).toBe("only"); + }); +}); diff --git a/packages/jest-runner/globalsInterceptor.ts b/packages/jest-runner/globalsInterceptor.ts new file mode 100644 index 00000000..a51ed8eb --- /dev/null +++ b/packages/jest-runner/globalsInterceptor.ts @@ -0,0 +1,65 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Runtime from "jest-runtime"; +import { Options } from "@jazzer.js/core"; +import { fuzz } from "./fuzz"; +import { InterceptedTestState } from "./testStateInterceptor"; + +export function interceptGlobals( + runtime: Runtime, + testPath: string, + jazzerConfig: Options, + { + currentTestState, + currentTestTimeout, + originalTestNamePattern, + }: InterceptedTestState, +) { + const originalSetGlobalsForRuntime = + runtime.setGlobalsForRuntime.bind(runtime); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + runtime.setGlobalsForRuntime = (globals: any) => { + globals.it.fuzz = fuzz( + globals, + testPath, + jazzerConfig, + currentTestState, + currentTestTimeout, + originalTestNamePattern, + "standard", + ); + globals.it.skip.fuzz = fuzz( + globals, + testPath, + jazzerConfig, + currentTestState, + currentTestTimeout, + originalTestNamePattern, + "skip", + ); + globals.it.only.fuzz = fuzz( + globals, + testPath, + jazzerConfig, + currentTestState, + currentTestTimeout, + originalTestNamePattern, + "only", + ); + originalSetGlobalsForRuntime(globals); + }; +} diff --git a/packages/jest-runner/index.ts b/packages/jest-runner/index.ts index 557d8f60..610cd445 100644 --- a/packages/jest-runner/index.ts +++ b/packages/jest-runner/index.ts @@ -14,90 +14,63 @@ * limitations under the License. */ -import { loadConfig } from "./config"; -import { cleanupJestRunnerStack } from "./errorUtils"; -import { FuzzTest } from "./fuzz"; -import { JazzerWorker } from "./worker"; -import { initFuzzing } from "@jazzer.js/core"; -import { - CallbackTestRunner, - OnTestFailure, - OnTestStart, - OnTestSuccess, - Test, - TestRunnerContext, - TestRunnerOptions, - TestWatcher, -} from "jest-runner"; -import { Config } from "@jest/types"; import * as reports from "istanbul-reports"; +import Runtime from "jest-runtime"; +import { TestResult } from "@jest/test-result"; +import { Config } from "@jest/types"; +import type { JestEnvironment } from "@jest/environment"; -class FuzzRunner extends CallbackTestRunner { - shouldCollectCoverage: boolean; - coverageReporters: Config.CoverageReporters; - constructor(globalConfig: Config.GlobalConfig, context: TestRunnerContext) { - super(globalConfig, context); - this.shouldCollectCoverage = globalConfig.collectCoverage; - this.coverageReporters = globalConfig.coverageReporters; - } +import { initFuzzing, setJazzerJsGlobal } from "@jazzer.js/core"; - async runTests( - tests: Array, - watcher: TestWatcher, - onStart: OnTestStart, - onResult: OnTestSuccess, - onFailure: OnTestFailure, - options: TestRunnerOptions, - ): Promise { - // Prefer Jest coverage configuration. - const config = loadConfig({ - coverage: this.shouldCollectCoverage, - coverageReporters: this.coverageReporters as reports.ReportType[], - }); +import { loadConfig } from "./config"; +import { cleanupJestError } from "./errorUtils"; +import { FuzzTest } from "./fuzz"; +import { interceptScriptTransformerCalls } from "./transformerInterceptor"; +import { interceptTestState } from "./testStateInterceptor"; +import { interceptGlobals } from "./globalsInterceptor"; - await initFuzzing(config); - return this.#runTestsInBand(tests, watcher, onStart, onResult, onFailure); - } +export default async function jazzerTestRunner( + globalConfig: Config.GlobalConfig, + config: Config.ProjectConfig, + environment: JestEnvironment, + runtime: Runtime, + testPath: string, + sendMessageToJest?: boolean, +): Promise { + const vmContext = environment.getVmContext(); + if (vmContext === null) throw new Error("vmContext is undefined"); + setJazzerJsGlobal("vmContext", vmContext); - async #runTestsInBand( - tests: Array, - watcher: TestWatcher, - onStart: OnTestStart, - onResult: OnTestSuccess, - onFailure: OnTestFailure, - ) { - process.env.JEST_WORKER_ID = "1"; - return tests.reduce((promise, test) => { - return promise.then(async () => { - if (watcher.isInterrupted()) { - throw new CancelRun(); - } + const jazzerConfig = loadConfig({ + coverage: globalConfig.collectCoverage, + coverageReporters: globalConfig.coverageReporters as reports.ReportType[], + }); + const globalEnvironments = [environment.getVmContext(), globalThis]; + const instrumentor = await initFuzzing(jazzerConfig, globalEnvironments); + interceptScriptTransformerCalls(runtime, instrumentor); - // Execute every test in a dedicated worker instance. - // Currently, this is only in band but the structure supports parallel - // execution in the future. - await onStart(test); - const worker = new JazzerWorker(); - return worker.run(test, this._globalConfig).then( - (result) => onResult(test, result), - (error) => { - error.stack = cleanupJestRunnerStack(error.stack); - onFailure(test, error); - }, - ); - }); - }, Promise.resolve()); - } -} + const testState = interceptTestState(environment, jazzerConfig); + interceptGlobals(runtime, testPath, jazzerConfig, testState); -class CancelRun extends Error { - constructor(message?: string) { - super(message); - this.name = "CancelRun"; - } -} + const circusRunner = + await runtime["_scriptTransformer"].requireAndTranspileModule( + "jest-circus/runner", + ); -export default FuzzRunner; + return circusRunner( + globalConfig, + config, + environment, + runtime, + testPath, + sendMessageToJest, + ).then((result: TestResult) => { + result.testResults.forEach((testResult) => { + testResult.failureDetails?.forEach(cleanupJestError); + }); + return result; + }); +} // Global definition of the Jest fuzz test extension function. // This is required to allow the Typescript compiler to recognize it. diff --git a/packages/jest-runner/readme.md b/packages/jest-runner/readme.md index f07af0cc..45b06afb 100644 --- a/packages/jest-runner/readme.md +++ b/packages/jest-runner/readme.md @@ -14,7 +14,7 @@ import "@jazzer.js/jest-runner"; import * as target from "./target"; describe("Target", () => { - it.fuzz("executes a method", (data: Buffer) => { + test.fuzz("executes a method", (data: Buffer) => { target.fuzzMe(data); }); }); diff --git a/packages/jest-runner/testStateInterceptor.test.ts b/packages/jest-runner/testStateInterceptor.test.ts new file mode 100644 index 00000000..dc7099db --- /dev/null +++ b/packages/jest-runner/testStateInterceptor.test.ts @@ -0,0 +1,114 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defaultOptions, Options } from "@jazzer.js/core"; +import { interceptTestState } from "./testStateInterceptor"; + +describe("Test state interceptor", () => { + it("hand back the latest describe block", () => { + const { env, config } = mockEnvironment(); + + const { currentTestState } = interceptTestState(env, config); + + env.publishEvent({}, { currentDescribeBlock: "state1" }); + expect(currentTestState()).toBe("state1"); + + env.publishEvent({}, { currentDescribeBlock: "state2" }); + expect(currentTestState()).toBe("state2"); + }); + + it("adjust test name pattern in regression mode", () => { + const { env, config } = mockEnvironment({ mode: "regression" }); + + const { originalTestNamePattern } = interceptTestState(env, config); + + const state = { testNamePattern: /test$/ }; + env.publishEvent({ name: "setup" }, state); + expect(state.testNamePattern).toEqual(/test/); + expect(originalTestNamePattern()).toEqual(/test$/); + }); + + it("do not adjust test name pattern in fuzzing mode", () => { + const { env, config } = mockEnvironment({ mode: "fuzzing" }); + + const interceptedTestState = interceptTestState(env, config); + + const state = { testNamePattern: /test$/ }; + env.publishEvent({ name: "setup" }, state); + expect(state.testNamePattern).toEqual(/test$/); + expect(interceptedTestState.originalTestNamePattern()).toBeUndefined(); + }); + + it("mark all but the first fuzz test as skipped", () => { + function eventWithTestName(name: string) { + return { + name: "test_start", + test: { + name: name, + mode: "run", + parent: { + name: "ROOT DESCRIBE BLOCK", + }, + }, + }; + } + + const { env, config, originalHandleTestEvent } = mockEnvironment({ + mode: "fuzzing", + }); + + interceptTestState(env, config); + + const state = { testNamePattern: /test$/ }; + env.publishEvent(eventWithTestName("1. test"), state); + env.publishEvent(eventWithTestName("2. test"), state); + env.publishEvent(eventWithTestName("3. test"), state); + + expect(originalHandleTestEvent).toHaveBeenCalledTimes(3); + const firstEvent = originalHandleTestEvent.mock.calls[0][0]; + expect(firstEvent.test.mode).toBe("run"); + const secondEvent = originalHandleTestEvent.mock.calls[1][0]; + expect(secondEvent.test.mode).toBe("skip"); + const thirdEvent = originalHandleTestEvent.mock.calls[2][0]; + expect(thirdEvent.test.mode).toBe("skip"); + }); + + it("deactivate Jest timeout in fuzzing mode", () => { + const { env, config } = mockEnvironment({ mode: "fuzzing" }); + + const { currentTestTimeout } = interceptTestState(env, config); + + env.publishEvent({ name: "test_fn_start" }, { testTimeout: 5000 }); + expect(currentTestTimeout()).toBeGreaterThan(5000); + }); +}); + +function mockEnvironment(options: Partial = {}) { + const originalHandleTestEvent = jest.fn(); + const env = { + handleTestEvent: originalHandleTestEvent, + publishEvent: function (event: unknown, state: unknown) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (this.handleTestEvent as any)(event, state); + }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + const config = { + ...defaultOptions, + ...options, + }; + return { env, config, originalHandleTestEvent }; +} diff --git a/packages/jest-runner/testStateInterceptor.ts b/packages/jest-runner/testStateInterceptor.ts new file mode 100644 index 00000000..7978f71f --- /dev/null +++ b/packages/jest-runner/testStateInterceptor.ts @@ -0,0 +1,104 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JestEnvironment } from "@jest/environment"; +import { Options } from "@jazzer.js/core"; +import { Circus } from "@jest/types"; + +// Arbitrary high value to disable Jest timeout. +const JEST_TIMEOUT_DISABLED = 1000 * 60 * 24 * 365; + +export type InterceptedTestState = { + currentTestState: () => Circus.DescribeBlock | undefined; + currentTestTimeout: () => number | undefined; + originalTestNamePattern: () => RegExp | undefined; +}; + +export function interceptTestState( + environment: JestEnvironment, + jazzerConfig: Options, +): InterceptedTestState { + const originalHandleTestEvent = + environment.handleTestEvent?.bind(environment); + let testState: Circus.DescribeBlock | undefined; + let testTimeout: number | undefined; + let firstFuzzTestEncountered: boolean | undefined; + let originalTestNamePattern: RegExp | undefined; + + environment.handleTestEvent = (event: Circus.Event, state: Circus.State) => { + testState = state.currentDescribeBlock; + // First event, created once on start up. + if (event.name === "setup") { + // In regression mode, fuzz tests are added as describe block with every seed file as dedicated + // test inside. This breaks test name pattern matching, so remove "$" from the end of the pattern, + // and skip tests not matching the original pattern in the fuzz function. + if ( + jazzerConfig.mode == "regression" && + state.testNamePattern?.source?.endsWith("$") + ) { + originalTestNamePattern = state.testNamePattern; + state.testNamePattern = new RegExp( + state.testNamePattern.source.slice(0, -1), + ); + } + // Created for every test function, before lifecycle hooks. + } else if (event.name === "test_start") { + // In fuzzing mode, only execute the first encountered (not skipped) fuzz test + // and mark all others as skipped. + if (jazzerConfig.mode === "fuzzing") { + if ( + !firstFuzzTestEncountered && + (!state.testNamePattern || + state.testNamePattern.test(testName(event.test))) + ) { + firstFuzzTestEncountered = true; + } else { + event.test.mode = "skip"; + } + } + // Created for every test function, before the actual function invocation. + } else if (event.name === "test_fn_start") { + // Disable Jest timeout in fuzzing mode by setting it to a high value, + // otherwise Jest will kill the fuzz test after it's timeout (default 5 seconds). + if (jazzerConfig.mode === "fuzzing") { + state.testTimeout = JEST_TIMEOUT_DISABLED; + } + // Use configured timeout as fuzzing timeout as well. Every invocation + // of the fuzz test has to be faster than this. + testTimeout = state.testTimeout; + } + if (originalHandleTestEvent) { + return originalHandleTestEvent(event as Circus.AsyncEvent, state); + } + }; + + // Return closures to access latest received state. + return { + currentTestState: () => testState, + currentTestTimeout: () => testTimeout, + originalTestNamePattern: () => originalTestNamePattern, + }; +} + +function testName(test: Circus.TestEntry): string { + const titles = []; + let parent: Circus.TestEntry | Circus.DescribeBlock | undefined = test; + do { + titles.unshift(parent.name); + } while ((parent = parent.parent)); + titles.shift(); // Remove root describe block. + return titles.join(" "); +} diff --git a/packages/jest-runner/transformerInterceptor.test.ts b/packages/jest-runner/transformerInterceptor.test.ts new file mode 100644 index 00000000..78af5c77 --- /dev/null +++ b/packages/jest-runner/transformerInterceptor.test.ts @@ -0,0 +1,259 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Instrumentor } from "@jazzer.js/instrumentor"; +import { interceptScriptTransformerCalls } from "./transformerInterceptor"; +import Runtime from "jest-runtime"; +import { SourceMap } from "@jazzer.js/instrumentor/dist/SourceMapRegistry"; +import { TransformResult } from "@jest/transform"; +import fs from "fs"; +import tmp from "tmp"; + +tmp.setGracefulCleanup(); + +describe("TransformerInterceptor", () => { + describe("_buildTransformResult", () => { + it("instrument sources", () => { + const instrumentedReturn = { + code: "instrumented code", + map: "instrumented map", + }; + const { instrumentor, runtime, scriptTransformer } = + mockInstrumentorAndRuntime(instrumentedReturn); + + const originalResult = { + code: "original code", + sourceMapPath: "filename", + originalCode: "original code", + }; + const originalBuildTransformResult = + scriptTransformer._buildTransformResult; + originalBuildTransformResult.mockImplementationOnce(() => originalResult); + + const filename = "filename"; + const processed = { + code: "some code", + map: buildSourceMap({ file: filename }), + }; + + interceptScriptTransformerCalls(runtime, instrumentor); + const result = scriptTransformer._buildTransformResult( + filename, + "cacheFilePath", + "content", + undefined, + false, + {}, + processed, + null, + ); + + expect(result.code).toBe(instrumentedReturn.code); + expect(result.originalCode).toBe(originalResult.originalCode); + expect(result.sourceMapPath).toBeDefined(); + const sourceMapContent = JSON.parse( + fs.readFileSync(result.sourceMapPath, "utf8"), + ); + expect(sourceMapContent).toBe(instrumentedReturn.map); + + expect(instrumentor.instrument).toHaveBeenCalledWith( + originalResult.code, + filename, + undefined, + ); + }); + + it("does not intercepts sources if already instrumented", () => { + const { instrumentor, runtime, scriptTransformer } = + mockInstrumentorAndRuntime(); + const originalResult = { + code: "some code; Fuzzer.coverageTracker.incrementCounter(); some other code;", + sourceMapPath: "filename", + originalCode: "original code", + }; + const originalBuildTransformResult = + scriptTransformer._buildTransformResult; + originalBuildTransformResult.mockImplementationOnce(() => originalResult); + + const filename = "filename"; + const processed = { + code: "some code", + map: buildSourceMap({ file: filename }), + }; + + interceptScriptTransformerCalls(runtime, instrumentor); + const result = scriptTransformer._buildTransformResult( + filename, + "cacheFilePath", + "content", + undefined, + false, + {}, + processed, + null, + ); + + expect(result).toBe(originalResult); + expect(instrumentor.instrument).not.toHaveBeenCalled(); + }); + }); + + describe("_transformAndBuildScript", () => { + it("does not instrument result if already instrumented", () => { + const { instrumentor, runtime, scriptTransformer } = + mockInstrumentorAndRuntime(); + const originalResult: TransformResult = { + code: "some code; Fuzzer.coverageTracker.incrementCounter(); some other code;", + originalCode: "originalCode", + sourceMapPath: "sourceMapPath", + }; + const originalTransformAndBuildScript = + scriptTransformer._transformAndBuildScript; + originalTransformAndBuildScript.mockImplementationOnce( + () => originalResult, + ); + + interceptScriptTransformerCalls(runtime, instrumentor); + const result = scriptTransformer._transformAndBuildScript( + "filename", + {}, + {}, + "fileSource", + ); + + expect(result).toBe(originalResult); + expect(originalTransformAndBuildScript).toHaveBeenCalledWith( + "filename", + {}, + {}, + "fileSource", + ); + expect(instrumentor.instrument).not.toHaveBeenCalled(); + }); + + it("writes source map file if instrumented", () => { + const { instrumentor, runtime, scriptTransformer } = + mockInstrumentorAndRuntime({ + code: "instrumentedCode", + map: { fakeSourceMap: "fakeSourceMap" }, + }); + + const originalTransformAndBuildScript = + scriptTransformer._transformAndBuildScript; + originalTransformAndBuildScript.mockImplementationOnce(() => ({ + code: "some code", + originalCode: "originalCode", + sourceMapPath: null, + })); + + interceptScriptTransformerCalls(runtime, instrumentor); + const result = scriptTransformer._transformAndBuildScript( + "filename", + {}, + {}, + "fileSource", + ); + + expect(originalTransformAndBuildScript).toHaveBeenCalledWith( + "filename", + {}, + {}, + "fileSource", + ); + expect(result.code).toBe("instrumentedCode"); + expect(result.originalCode).toBe("originalCode"); + expect(result.sourceMapPath).toBeDefined(); + const sourceMapContent = fs.readFileSync( + result.sourceMapPath as string, + "utf8", + ); + expect(sourceMapContent).toBe('{"fakeSourceMap":"fakeSourceMap"}'); + }); + + it("applies existing source map in instrumentation", () => { + const { instrumentor, runtime, scriptTransformer } = + mockInstrumentorAndRuntime({ + code: "instrumentedCode", + map: { fakeSourceMap: "fakeSourceMap" }, + }); + + const sourceMap = buildSourceMap(); + const sourceMapFile = tmp.fileSync({ prefix: "jazzerjs-test" }); + fs.writeFileSync(sourceMapFile.name, JSON.stringify(sourceMap)); + + const originalTransformAndBuildScript = + scriptTransformer._transformAndBuildScript; + originalTransformAndBuildScript.mockImplementationOnce(() => ({ + code: "some code", + originalCode: "originalCode", + sourceMapPath: sourceMapFile.name, + })); + + interceptScriptTransformerCalls(runtime, instrumentor); + scriptTransformer._transformAndBuildScript( + "filename", + {}, + {}, + "fileSource", + ); + + expect(instrumentor.instrument).toHaveBeenCalledWith( + "some code", + "filename", + sourceMap, + ); + }); + }); +}); + +function mockInstrumentorAndRuntime(instrumentorReturnValue?: unknown): { + instrumentor: Instrumentor; + runtime: Runtime; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + scriptTransformer: any; +} { + const instrumentor = new Instrumentor(); + instrumentor.instrument = jest.fn().mockReturnValue(instrumentorReturnValue); + + // Mocking runtime and its deeply nested structures + // turns out to be quite complex. Using any for now. + const runtime = { + _scriptTransformer: { + _buildTransformResult: jest.fn(), + _transformAndBuildScript: jest.fn(), + _transformAndBuildScriptAsync: jest.fn(), + }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + return { + instrumentor, + runtime, + scriptTransformer: runtime["_scriptTransformer"], + }; +} + +function buildSourceMap(map?: Partial) { + return { + version: 3, + sources: ["source1", "source2"], + names: ["name1", "name2"], + sourceRoot: "", + sourcesContent: ["sourceContent1", "sourceContent2"], + mappings: "mappings", + file: "filename", + ...map, + }; +} diff --git a/packages/jest-runner/transformerInterceptor.ts b/packages/jest-runner/transformerInterceptor.ts new file mode 100644 index 00000000..b370f349 --- /dev/null +++ b/packages/jest-runner/transformerInterceptor.ts @@ -0,0 +1,193 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Disable ban-types to use Function as type in interceptions. +/* eslint-disable @typescript-eslint/ban-types */ + +import fs from "fs"; +import tmp from "tmp"; +import { + CallerTransformOptions, + TransformedSource, + TransformResult, +} from "@jest/transform"; +import Runtime from "jest-runtime"; +import { Instrumentor, SourceMap } from "@jazzer.js/instrumentor"; + +tmp.setGracefulCleanup(); + +// Code containing coverage instrumentation calls is considered to be instrumented. +const INSTRUMENTATION_MARKER = "Fuzzer.coverageTracker.incrementCounter"; + +export function interceptScriptTransformerCalls( + runtime: Runtime, + instrumentor: Instrumentor, +) { + const scriptTransformer = runtime["_scriptTransformer"]; + + // _buildTransformResult is used in transformSource and transformSourceAsync + // and creates a cache file for the transformed code. We instrument and hence change + // the result, so that the cache file does not match anymore and transformation happens + // every time. This prevents loading wrongly (not) instrumented versions from previous + // runs with different configurations. + intercept( + scriptTransformer, + "_buildTransformResult", + (original: Function) => + ( + filename: string, + cacheFilePath: string, + content: string, + transformer: Transformer | undefined, + shouldCallTransform: boolean, + options: CallerTransformOptions, + processed: TransformedSource | null, + sourceMapPath: string | null, + ): TransformResult => { + const result: TransformResult = original( + filename, + cacheFilePath, + content, + transformer, + shouldCallTransform, + options, + processed, + sourceMapPath, + ); + if (!result || isInstrumented(result.code)) { + return result; + } + const instrumented = instrumentor.instrument( + result.code, + filename, + sourceMapContent(sourceMapPath), + ); + if (instrumented?.map) { + sourceMapPath = writeSourceMap(instrumented.map); + } + return { + code: instrumented?.code ?? result.code, + originalCode: result.originalCode, + sourceMapPath: sourceMapPath, + }; + }, + ); + + // _transformAndBuildScript can call transformSource, which requires checks to + // prevent double instrumentation. + // As the original result could already apply transformations the result includes + // a source map path, which the instrumentor needs to take into account for its + // instrumentation. The result is not saved in a cache file and can be changed + // directly to point to a dumped source map file. + intercept( + scriptTransformer, + "_transformAndBuildScript", + (original: Function) => + ( + filename: string, + options: unknown, + transformOptions: unknown, + fileSource?: string, + ): TransformResult => { + const originalResult: TransformResult = original( + filename, + options, + transformOptions, + fileSource, + ); + return processTransformResult(originalResult, filename, instrumentor); + }, + ); + + // Similar to _transformAndBuildScript, but async. Is used to load ESM modules. + intercept( + scriptTransformer, + "_transformAndBuildScriptAsync", + (original: Function) => + async ( + filename: string, + options: unknown, + transformOptions: unknown, + fileSource?: string, + ): Promise => { + const originalResult: TransformResult = await original( + filename, + options, + transformOptions, + fileSource, + ); + return processTransformResult(originalResult, filename, instrumentor); + }, + ); +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function intercept(obj: any, name: string, interceptor: Function) { + obj[name] = interceptor(obj[name].bind(obj)); +} + +function isInstrumented(code: string): boolean { + return code.includes(INSTRUMENTATION_MARKER); +} + +function processTransformResult( + originalResult: TransformResult, + filename: string, + instrumentor: Instrumentor, +): TransformResult { + // If already instrumented by previous calls or internal invocation of + // transformSource simply return the original result. + if (isInstrumented(originalResult.code)) { + return originalResult; + } + + const sourceMap = sourceMapContent(originalResult.sourceMapPath); + const instrumented = instrumentor.instrument( + originalResult.code, + filename, + sourceMap, + ); + if (!instrumented) { + return originalResult; + } + // Source map path is only set if a transformation happened, in that case the + // code should be instrumented via the other intercepted method. + let sourceMapPath = originalResult.sourceMapPath; + if (instrumented?.map) { + sourceMapPath = writeSourceMap(instrumented.map); + } + return { + code: instrumented.code ?? originalResult.code, + sourceMapPath: sourceMapPath, + originalCode: originalResult.originalCode, + }; +} + +function writeSourceMap(sourceMap: Object) { + const sourceMapPath = tmp.fileSync({ prefix: "jazzerjs-map" }).name; + fs.writeFileSync(sourceMapPath, JSON.stringify(sourceMap)); + return sourceMapPath; +} + +function sourceMapContent(sourceMapPath: string | null): SourceMap | undefined { + if (sourceMapPath) { + try { + return JSON.parse(fs.readFileSync(sourceMapPath).toString()); + } catch (e) { + // Ignore missing source map + } + } +} diff --git a/packages/jest-runner/worker.ts b/packages/jest-runner/worker.ts deleted file mode 100644 index aab989a8..00000000 --- a/packages/jest-runner/worker.ts +++ /dev/null @@ -1,483 +0,0 @@ -/* - * Copyright 2023 Code Intelligence GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Test } from "jest-runner"; -import { Circus, Config } from "@jest/types"; -import { TestResult } from "@jest/test-result"; -import { performance } from "perf_hooks"; -import { jestExpect as expect } from "@jest/expect"; -import * as circus from "jest-circus"; -import { formatResultsErrors } from "jest-message-util"; -import { inspect } from "util"; -import { fuzz, FuzzerStartError, skip } from "./fuzz"; -import { cleanupJestRunnerStack, removeTopFramesFromError } from "./errorUtils"; -import { createScriptTransformer } from "@jest/transform"; - -function isGeneratorFunction(obj?: unknown): boolean { - return ( - !!obj && - typeof (obj as Generator).next === "function" && - typeof (obj as Generator)[Symbol.iterator] === "function" - ); -} - -type JazzerTestStatus = { - failures: number; - passes: number; - pending: number; - start: number; - end: number; -}; - -type JazzerTestResult = { - ancestors: string[]; - title: string; - skipped: boolean; - errors: Error[]; - duration?: number; -}; - -export class JazzerWorker { - static #workerInitialized = false; - static #currentTestPath = ""; - readonly defaultTimeout = 5000; // Default Jest timeout - - #testSummary: JazzerTestStatus; - #testResults: JazzerTestResult[]; - - constructor() { - this.#testSummary = { - passes: 0, - failures: 0, - pending: 0, - start: 0, - end: 0, - }; - this.#testResults = []; - } - - static get currentTestPath(): string { - return this.#currentTestPath; - } - - private async initialize(test: Test) { - if (JazzerWorker.#workerInitialized) { - return; - } - JazzerWorker.#workerInitialized = true; - - for (const file of test.context.config.setupFiles) { - const { default: setup } = await this.importFile(file); - setup(); - } - - JazzerWorker.setupGlobal(); - - for (const file of test.context.config.setupFilesAfterEnv) { - const { default: setup } = await this.importFile(file); - setup(); - } - } - - private static setupGlobal() { - // @ts-ignore - globalThis.expect = expect; - // @ts-ignore - globalThis.test = circus.test; - // @ts-ignore - globalThis.test.fuzz = fuzz; - // @ts-ignore - globalThis.test.skip.fuzz = skip; - // @ts-ignore - globalThis.it = circus.it; - // @ts-ignore - globalThis.it.fuzz = fuzz; - // @ts-ignore - globalThis.it.skip.fuzz = skip; - // @ts-ignore - globalThis.describe = circus.describe; - // @ts-ignore - globalThis.beforeAll = circus.beforeAll; - // @ts-ignore - globalThis.afterAll = circus.afterAll; - // @ts-ignore - globalThis.beforeEach = circus.beforeEach; - // @ts-ignore - globalThis.afterEach = circus.afterEach; - } - - async run(test: Test, config: Config.GlobalConfig) { - JazzerWorker.#currentTestPath = test.path; - await this.initialize(test); - - const state = await this.loadTests(test); - - this.#testSummary.start = performance.now(); - await this.runDescribeBlock( - state.rootDescribeBlock, - state.hasFocusedTests, - config.testNamePattern ?? "", - ); - this.#testSummary.end = performance.now(); - - const result = this.testResult(test); - result.failureMessage = formatResultsErrors( - result.testResults, - test.context.config, - config, - test.path, - ); - return result; - } - - private async loadTests(test: Test): Promise { - circus.resetState(); - - // Don't cache transformed files, as that will interfere with the internal - // transformations. Config is read only, so a copy is needed. - const config = { ...test.context.config }; - config.cache = false; - const transformer = await createScriptTransformer(config); - await transformer.requireAndTranspileModule(test.path); - - return circus.getState(); - } - - private async runDescribeBlock( - block: Circus.DescribeBlock, - hasFocusedTests: boolean, - testNamePattern: string, - ancestors: string[] = [], - ) { - const adjustedPattern = this.adjustTestPattern(ancestors, testNamePattern); - - await this.runHooks("beforeAll", block); - - for (const child of block.children) { - const nextAncestors = ancestors.concat(child.name); - if ( - child.mode === "skip" || - (child.type === "test" && - this.shouldSkipTest(nextAncestors, adjustedPattern)) - ) { - this.#testSummary.pending++; - this.#testResults.push({ - ancestors, - title: child.name, - errors: [], - skipped: true, - }); - } else if (child.type === "describeBlock") { - await this.runHooks("beforeEach", block, true); - await this.runDescribeBlock( - child, - hasFocusedTests, - testNamePattern, - nextAncestors, - ); - await this.runHooks("afterEach", block, true); - } else if (child.type === "test") { - await this.runHooks("beforeEach", block, true); - await this.runTestEntry(child, ancestors); - await this.runHooks("afterEach", block, true); - } - } - - await this.runHooks("afterAll", block); - } - - private async runTestEntry( - testEntry: Circus.TestEntry, - ancestors: string[] = [], - ) { - expect.setState({ - suppressedErrors: [], - currentTestName: this.fullTestPath(ancestors.concat(testEntry.name)), - }); - - let skipTest = false; - let errors = []; - await Promise.resolve() - // @ts-ignore - .then(testEntry.fn) - .catch((error) => { - // Mark fuzzer tests as skipped and not as error. - if (error instanceof FuzzerStartError) { - skipTest = true; - } - errors.push(error); - }); - - // Get suppressed errors from ``jest-matchers`` that weren't thrown during - // test execution and add them to the test result, potentially failing - // a passing test. - const state = expect.getState(); - if (state.suppressedErrors.length > 0) { - errors.unshift(...state.suppressedErrors); - } - - errors = errors.map((e) => { - if (e && e.stack) { - e.stack = cleanupJestRunnerStack(e.stack); - } - return e; - }); - - if (skipTest) { - this.#testSummary.pending++; - } else if (errors.length > 0) { - this.#testSummary.failures++; - } else { - this.#testSummary.passes++; - } - this.#testResults.push({ - ancestors, - title: testEntry.name, - skipped: skipTest, - errors, - }); - } - - private async runHooks( - hookType: string, - block: Circus.DescribeBlock, - shouldRunInAncestors = false, - ) { - const hooks = - shouldRunInAncestors && block.parent ? block.parent.hooks : block.hooks; - for (const hook of hooks.filter((hook) => hook.type === hookType)) { - const timeout = hook.timeout ?? this.defaultTimeout; - await this.runHook(block, hook, timeout); - } - } - - private async runHook( - block: Circus.DescribeBlock, - hook: Circus.Hook, - timeout: number, - ) { - let timeoutID: NodeJS.Timeout; - return new Promise((resolve, reject) => { - timeoutID = setTimeout(() => { - reject( - removeTopFramesFromError( - new Error( - `Exceeded timeout of ${timeout} ms for "${hook.type}" of "${block.name}".\nIncrease the timeout value, if this is a long-running test.`, - ), - 1, - ), - ); - }, timeout); - this.executeHook(block, hook, resolve, reject); - }).then( - (value: unknown) => { - // Remove timeout to enable clean shutdown. - timeoutID?.unref?.(); - clearTimeout(timeoutID); - return value; - }, - (error: unknown) => { - // Remove timeout to enable clean shutdown. - timeoutID?.unref?.(); - clearTimeout(timeoutID); - throw error; - }, - ); - } - - private executeHook( - block: Circus.DescribeBlock, - hook: Circus.Hook, - resolve: (value: unknown) => void, - reject: (reason?: unknown) => void, - ) { - let result; - if (hook.fn.length > 0) { - result = new Promise((resolve, reject) => { - let doneCalled = false; - const done = (doneMsg?: string | Error) => { - if (doneCalled) { - // As the promise was already resolved in the last invocation, and - // there could be quite some time until this one, there is not much we - // can do besides printing an error message. - console.error( - `ERROR: Expected done to be called once, but it was called multiple times in "${hook.type}" of "${block.name}".`, - ); - } - doneCalled = true; - if (typeof doneMsg === "string") { - reject( - removeTopFramesFromError(new Error(`Failed: ${doneMsg}`), 1), - ); - } else if (doneMsg) { - reject(doneMsg); - } else { - resolve(undefined); - } - }; - const hookResult = hook.fn(done); - // These checks are executed before the callback, hence rejecting - // the promise is still possible. - if (hookResult instanceof Promise) { - reject( - removeTopFramesFromError( - new Error( - `Using done callback in async "${hook.type}" hook of "${block.name}" is not allowed.`, - ), - 1, - ), - ); - } else if (isGeneratorFunction(hookResult)) { - reject( - removeTopFramesFromError( - new Error( - `Generators are currently not supported by Jazzer.js but used in "${hook.type}" of "${block.name}".`, - ), - 1, - ), - ); - } - }); - } else { - // @ts-ignore - result = hook.fn(); - } - - if (result instanceof Promise) { - result.then(resolve, reject); - } else if (isGeneratorFunction(result)) { - reject( - removeTopFramesFromError( - new Error( - `Generators are currently not supported by Jazzer.js but used in "${hook.type}" of "${block.name}".`, - ), - 1, - ), - ); - } else { - resolve(result); - } - } - - private testResult(test: Test): TestResult { - const runtime = this.#testSummary.end - this.#testSummary.start; - - return { - coverage: globalThis.__coverage__, - console: undefined, - failureMessage: this.#testResults - .filter((t) => t.errors.length > 0) - .map(this.failureToString) - .join("\n"), - leaks: false, - numFailingTests: this.#testSummary.failures, - numPassingTests: this.#testSummary.passes, - numPendingTests: this.#testSummary.pending, - numTodoTests: 0, - openHandles: [], - perfStats: { - start: this.#testSummary.start, - end: this.#testSummary.end, - runtime: Math.round(runtime), // ms precision - slow: runtime / 1000 > test.context.config.slowTestThreshold, - }, - skipped: this.#testResults.every((t) => t.skipped), - snapshot: { - added: 0, - fileDeleted: false, - matched: 0, - unchecked: 0, - uncheckedKeys: [], - unmatched: 0, - updated: 0, - }, - testExecError: undefined, - testFilePath: test.path, - testResults: this.#testResults.map((testResult) => { - return { - ancestorTitles: testResult.ancestors, - duration: testResult.duration ? testResult.duration / 1000 : null, - failureDetails: testResult.errors, - failureMessages: testResult.errors.length - ? [this.failureToString(testResult)] - : [], - fullName: testResult.title, - numPassingAsserts: testResult.errors.length > 0 ? 1 : 0, - status: testResult.skipped - ? "pending" - : testResult.errors.length > 0 - ? "failed" - : "passed", - title: testResult.title, - }; - }), - }; - } - - private failureToString(result: JazzerTestResult) { - return ( - result.errors - .map((error) => inspect(error).replace(/^/gm, " ")) - .join("\n") + "\n" - ); - } - - /** - * If we always remove the dollar sign, then the runner will run all tests matching to a test name. - * For that reason, we only remove the dollar sign if the test name matches exactly. - */ - private adjustTestPattern( - ancestors: string[], - testNamePattern: string, - ): string { - // IntelliJ interprets our fuzz extension as a test and thus appends a dollar sign - // to the fuzz test pattern when started from the IDE. This is fine for the fuzzing mode - // where we register a normal test. However, in the regression mode, we register a - // describe-block. This results in the child tests being skipped. - if ( - testNamePattern.endsWith("$") && - this.doesMatch(ancestors, testNamePattern) - ) { - return testNamePattern.slice(0, -1); - } - return testNamePattern; - } - - private shouldSkipTest(ancestors: string[], testNamePattern: string) { - return !this.doesMatch(ancestors, testNamePattern); - } - - private fullTestPath(elements: string[]): string { - return elements.join(" "); - } - - private doesMatch(ancestors: string[], testNamePattern: string) { - const testPath = this.fullTestPath(ancestors); - if (testNamePattern === "") { - return true; - } - const testNamePatternRE = new RegExp(testNamePattern, "i"); - return testNamePatternRE.test(testPath); - } - - private async importFile(file: string) { - // file: schema is required on Windows - if (!file.startsWith("file://")) { - file = "file://" + file; - } - return await import(file); - } -} diff --git a/tests/bug-detectors/command-injection/package.json b/tests/bug-detectors/command-injection/package.json index 17e7652a..5a2d828d 100644 --- a/tests/bug-detectors/command-injection/package.json +++ b/tests/bug-detectors/command-injection/package.json @@ -13,7 +13,7 @@ "jest": { "projects": [ { - "runner": "@jazzer.js/jest-runner", + "testRunner": "@jazzer.js/jest-runner", "displayName": { "name": "Jazzer.js", "color": "cyan" diff --git a/tests/bug-detectors/general/package.json b/tests/bug-detectors/general/package.json index 229ce0ef..12df04d9 100644 --- a/tests/bug-detectors/general/package.json +++ b/tests/bug-detectors/general/package.json @@ -13,7 +13,7 @@ "jest": { "projects": [ { - "runner": "@jazzer.js/jest-runner", + "testRunner": "@jazzer.js/jest-runner", "displayName": { "name": "Jazzer.js", "color": "cyan" diff --git a/tests/bug-detectors/prototype-pollution.test.js b/tests/bug-detectors/prototype-pollution.test.js index 12fdfebc..0f299bd1 100644 --- a/tests/bug-detectors/prototype-pollution.test.js +++ b/tests/bug-detectors/prototype-pollution.test.js @@ -325,9 +325,9 @@ describe("Prototype Pollution Jest tests", () => { const fuzzTest = new FuzzTestBuilder() .runs(0) .dir(bugDetectorDirectory) - .dryRun(true) + .dryRun(false) .jestTestFile("tests.fuzz.js") - .jestTestName("Pollution of Object") + .jestTestName("Prototype Pollution Jest tests Pollution of Object") .build(); expect(() => { fuzzTest.execute(); diff --git a/tests/bug-detectors/prototype-pollution/package.json b/tests/bug-detectors/prototype-pollution/package.json index 9719be17..35b8bac9 100644 --- a/tests/bug-detectors/prototype-pollution/package.json +++ b/tests/bug-detectors/prototype-pollution/package.json @@ -13,7 +13,7 @@ "jest": { "projects": [ { - "runner": "@jazzer.js/jest-runner", + "testRunner": "@jazzer.js/jest-runner", "displayName": { "name": "Jazzer.js", "color": "cyan" diff --git a/tests/code_coverage/coverage.test.js b/tests/code_coverage/coverage.test.js index 98d8137f..ffe8d5eb 100644 --- a/tests/code_coverage/coverage.test.js +++ b/tests/code_coverage/coverage.test.js @@ -94,12 +94,9 @@ describe("Source code coverage reports", () => { }); describe("for our custom Jest runner", () => { - it("Expect no coverage reports", () => { - executeJestRunner("**.fuzz.js", false, false, true); - expect(defaultCoverageDirectory).toBeCreated(); - const coverageJson = readCoverageJson(defaultCoverageDirectory); - // Jest generates an empty coverage report (unlike our non-jest fuzzer) - expect(coverageJson).toStrictEqual({}); + it("expect no coverage reports", () => { + executeJestRunner("**.fuzz.js", false, false, false); + expect(defaultCoverageDirectory).not.toBeCreated(); }); it("want coverage, no custom hooks", () => { @@ -193,8 +190,10 @@ function executeJestRunner( const config = { includes: includes, excludes: excludePattern, - customHooks: useCustomHooks, }; + if (useCustomHooks) { + config.customHooks = useCustomHooks; + } // write the config file, overwriting any existing one fs.writeFileSync( path.join(testDirectory, ".jazzerjsrc.json"), diff --git a/tests/code_coverage/package.json b/tests/code_coverage/package.json index 5b7613aa..b77d5df9 100644 --- a/tests/code_coverage/package.json +++ b/tests/code_coverage/package.json @@ -3,7 +3,8 @@ "version": "1.0.0", "description": "Unit tests for source code coverage.", "scripts": { - "fuzz": "jest" + "fuzz": "jest", + "test": "jest" }, "devDependencies": { "@jazzer.js/core": "file:../../packages/core" diff --git a/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+codeCoverage-fuzz.json b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+codeCoverage-fuzz.json index 5095d3e7..f868e98d 100644 --- a/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+codeCoverage-fuzz.json +++ b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+codeCoverage-fuzz.json @@ -45,8 +45,8 @@ } }, "branchMap": {}, - "s": { "0": 1, "1": 1, "2": 1, "3": 2 }, - "f": { "0": 1, "1": 2 }, + "s": { "0": 1, "1": 1, "2": 1, "3": 3 }, + "f": { "0": 1, "1": 3 }, "b": {}, "_coverageSchema": "1a1c01bbd47fc00a2c39e90264f33305004495a9", "hash": "25c01ffa3552de53ae353b8e557c73461d766e51" @@ -109,9 +109,9 @@ "line": 24 } }, - "s": { "0": 1, "1": 1, "2": 2, "3": 2, "4": 1, "5": 1 }, - "f": { "0": 2 }, - "b": { "0": [1, 1] }, + "s": { "0": 1, "1": 1, "2": 3, "3": 3, "4": 2, "5": 1 }, + "f": { "0": 3 }, + "b": { "0": [2, 1] }, "_coverageSchema": "1a1c01bbd47fc00a2c39e90264f33305004495a9", "hash": "d5b411e8de7efcd2798a7cd78efd7bd8347f647a" }, diff --git a/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+otherCodeCoverage-fuzz.json b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+otherCodeCoverage-fuzz.json index c898e932..5c25053d 100644 --- a/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+otherCodeCoverage-fuzz.json +++ b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+otherCodeCoverage-fuzz.json @@ -52,11 +52,11 @@ "1": 1, "2": 1, "3": 1, - "4": 2 + "4": 3 }, "f": { "0": 1, - "1": 2 + "1": 3 }, "b": {} }, @@ -118,9 +118,9 @@ "line": 24 } }, - "s": { "0": 1, "1": 1, "2": 2, "3": 2, "4": 1, "5": 1 }, - "f": { "0": 2 }, - "b": { "0": [1, 1] }, + "s": { "0": 1, "1": 1, "2": 3, "3": 3, "4": 2, "5": 1 }, + "f": { "0": 3 }, + "b": { "0": [2, 1] }, "_coverageSchema": "1a1c01bbd47fc00a2c39e90264f33305004495a9", "hash": "d5b411e8de7efcd2798a7cd78efd7bd8347f647a" }, diff --git a/tests/code_coverage/sample_fuzz_test/package.json b/tests/code_coverage/sample_fuzz_test/package.json index 774878fd..afa1d2e9 100644 --- a/tests/code_coverage/sample_fuzz_test/package.json +++ b/tests/code_coverage/sample_fuzz_test/package.json @@ -19,7 +19,7 @@ "name": "Jazzer.js", "color": "cyan" }, - "runner": "@jazzer.js/jest-runner", + "testRunner": "@jazzer.js/jest-runner", "preset": "ts-jest", "testEnvironment": "node" } diff --git a/tests/helpers.js b/tests/helpers.js index 7b3b5116..3667d0e3 100644 --- a/tests/helpers.js +++ b/tests/helpers.js @@ -23,6 +23,7 @@ const assert = require("assert"); // This is used to distinguish an error thrown during fuzzing from other errors, // such as wrong `fuzzEntryPoint`, which would return a "1". const FuzzingExitCode = "77"; +const TimeoutExitCode = "70"; const JestRegressionExitCode = "1"; const WindowsExitCode = "1"; @@ -75,6 +76,7 @@ class FuzzTest { } else { this.executeWithCli(); } + return this; } executeWithCli() { @@ -230,7 +232,7 @@ class FuzzTestBuilder { * default. */ verbose(verbose) { - this._verbose = verbose; + this._verbose = verbose === undefined ? true : verbose; return this; } @@ -450,17 +452,18 @@ function callWithTimeout(fn, timeout) { /** * Returns a Jest describe function that is skipped if the current platform is not the given one. - * @param platform - * @returns describe(.skip) function */ function describeSkipOnPlatform(platform) { return process.platform === platform ? global.describe.skip : global.describe; } -module.exports.FuzzTestBuilder = FuzzTestBuilder; -module.exports.FuzzingExitCode = FuzzingExitCode; -module.exports.JestRegressionExitCode = JestRegressionExitCode; -module.exports.WindowsExitCode = WindowsExitCode; -module.exports.makeFnCalledOnce = makeFnCalledOnce; -module.exports.callWithTimeout = callWithTimeout; -module.exports.describeSkipOnPlatform = describeSkipOnPlatform; +module.exports = { + FuzzTestBuilder, + FuzzingExitCode, + TimeoutExitCode, + JestRegressionExitCode, + WindowsExitCode, + makeFnCalledOnce, + callWithTimeout, + describeSkipOnPlatform, +}; diff --git a/tests/jest_integration/integration.test.js b/tests/jest_integration/integration.test.js index b00d61f8..cdcc54a9 100644 --- a/tests/jest_integration/integration.test.js +++ b/tests/jest_integration/integration.test.js @@ -17,7 +17,9 @@ const { FuzzTestBuilder, FuzzingExitCode, + TimeoutExitCode, WindowsExitCode, + JestRegressionExitCode, } = require("../helpers.js"); const path = require("path"); const fs = require("fs"); @@ -43,46 +45,141 @@ describe("Jest integration", () => { describe("Fuzzing mode", () => { const fuzzingExitCode = process.platform === "win32" ? WindowsExitCode : FuzzingExitCode; - const fuzzTestBuilder = new FuzzTestBuilder() - .dir(projectDir) - .runs(1_000_000) - .jestRunInFuzzingMode(true) - .jestTestFile(jestTestFile + ".js"); + let fuzzTestBuilder; - it("executes sync test", () => { - const fuzzTest = fuzzTestBuilder - .jestTestName("execute sync test") - .build(); - expect(() => { - fuzzTest.execute(); - }).toThrow(fuzzingExitCode); + beforeEach(() => { + fuzzTestBuilder = new FuzzTestBuilder() + .dir(projectDir) + .runs(1_000_000) + .jestRunInFuzzingMode(true) + .jestTestFile(jestTestFile + ".js"); }); - it("execute async test", () => { - const fuzzTest = fuzzTestBuilder - .jestTestName("execute async test") - .build(); - expect(() => { - fuzzTest.execute(); - }).toThrow(fuzzingExitCode); + describe("execute", () => { + it("execute sync test", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute sync test") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(fuzzingExitCode); + }); + + it("execute async test", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute async test") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(fuzzingExitCode); + }); + + it("execute async test returning a promise", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute async test returning a promise") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(fuzzingExitCode); + }); + + it("execute async test using a callback", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute async test using a callback") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(fuzzingExitCode); + }); + + it("single fuzz test without name pattern", () => { + const fuzzTest = fuzzTestBuilder + .jestTestFile("integration.fuzz.js") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(fuzzingExitCode); + }); + + it("print corpus directories", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute sync test") + .runs(1) + .verbose() + .build(); + try { + fuzzTest.execute(); + } catch (ignored) { + // ignored + } + expect(fuzzTest.stderr).toContain("INFO: using inputs from:"); + }); }); - it("execute async test returning a promise", () => { - const fuzzTest = fuzzTestBuilder - .jestTestName("execute async test returning a promise") - .build(); - expect(() => { - fuzzTest.execute(); - }).toThrow(fuzzingExitCode); + describe("timeout", () => { + it("execute async timeout test", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute async timeout test") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(TimeoutExitCode); + assertTimeoutMessageLogged(fuzzTest, 5); + }); + + it("execute async timeout test with method timeout", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute async timeout test with method timeout") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(TimeoutExitCode); + assertTimeoutMessageLogged(fuzzTest, 1); + }); + + it("execute async timeout test using a callback", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute async timeout test using a callback") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(TimeoutExitCode); + assertTimeoutMessageLogged(fuzzTest, 1); + }); }); - it("execute async test using a callback", () => { - const fuzzTest = fuzzTestBuilder - .jestTestName("execute async test using a callback") - .build(); - expect(() => { - fuzzTest.execute(); - }).toThrow(fuzzingExitCode); + describe("mix features", () => { + it("honor test name pattern", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("honor test name pattern$") + .runs(1) + .build() + .execute(); + expect(fuzzTest.stderr).not.toContain( + "This test should not be executed!", + ); + expect(fuzzTest.stderr).toContain("1 passed"); + }); + + it("execute a mocked test", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("mock test function") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(fuzzingExitCode); + expect(fuzzTest.stderr).toContain("the function was mocked"); + }); + + it("load by mapped module name", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("load by mapped module name") + .verbose() + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(fuzzingExitCode); + }); }); }); @@ -91,32 +188,130 @@ describe("Jest integration", () => { .dir(projectDir) .jestTestFile(jestTestFile + ".js"); - it("executes sync test", () => { - const fuzzTest = regressionTestBuilder - .jestTestName("execute sync test") - .build() - .execute(); + describe("execute", () => { + it("execute sync test", () => { + regressionTestBuilder + .jestTestName("execute sync test") + .build() + .execute(); + }); + + it("execute async test", () => { + regressionTestBuilder + .jestTestName("execute async test") + .build() + .execute(); + }); + + it("execute async test returning a promise", () => { + regressionTestBuilder + .jestTestName("execute async test returning a promise") + .build() + .execute(); + }); + + it("execute async test using a callback", () => { + regressionTestBuilder + .jestTestName("execute async test using a callback") + .build() + .execute(); + }); }); - it("execute async test", () => { - const fuzzTest = regressionTestBuilder - .jestTestName("execute async test") - .build() - .execute(); + describe("timeout", () => { + it("execute async timeout test", () => { + const fuzzTest = regressionTestBuilder + .jestTestName("execute async timeout test") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + expect(fuzzTest.stderr).toContain("Exceeded timeout"); + }); + + it("execute async timeout test with method timeout", () => { + const fuzzTest = regressionTestBuilder + .jestTestName("execute async timeout test with method timeout") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + expect(fuzzTest.stderr).toContain("Exceeded timeout"); + }); + + it("execute async timeout test using a callback", () => { + const fuzzTest = regressionTestBuilder + .jestTestName("execute async timeout test using a callback") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + expect(fuzzTest.stderr).toContain("Exceeded timeout"); + }); }); - it("execute async test returning a promise", () => { - const fuzzTest = regressionTestBuilder - .jestTestName("execute async test returning a promise") - .build() - .execute(); + describe("mix features", () => { + it("honor test name pattern", () => { + // Using a "$" suffix, like some IDEs, should also work in regression + // mode and only execute the specific test. + const fuzzTest = regressionTestBuilder + .jestTestName("honor test name pattern$") + .build() + .execute(); + expect(fuzzTest.stderr).not.toContain( + "This test should not be executed!", + ); + expect(fuzzTest.stderr).toContain("1 passed"); + }); + + it("execute a mocked test", () => { + const fuzzTest = regressionTestBuilder + .jestTestName("mock test function") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + expect(fuzzTest.stderr).toContain("the function was mocked"); + }); + + it("load by mapped module name", () => { + regressionTestBuilder + .jestTestName("load by mapped module name") + .build() + .execute(); + }); }); - it("execute async test using a callback", () => { - const fuzzTest = regressionTestBuilder - .jestTestName("execute async test using a callback") - .build() - .execute(); + describe("Run modes", () => { + it.concurrent("only", () => { + const fuzzTest = new FuzzTestBuilder() + .dir(projectDir) + .jestTestName("Run mode only and standard") + .jestTestFile("run-mode-only.fuzz.js") + .build() + .execute(); + expect(fuzzTest.stdout).toContain("only test called"); + }); + + it.concurrent("skipped", () => { + const fuzzTest = new FuzzTestBuilder() + .dir(projectDir) + .jestTestFile(jestTestFile + ".js") + .jestTestName("Run mode skip and standard") + .build() + .execute(); + expect(fuzzTest.stdout).toContain("standard test called"); + }); }); }); }); + +// Deflake the "timeout after N seconds" test to be more tolerant to small variations of N (+-1). +function assertTimeoutMessageLogged(fuzzTest, expectedTimeout) { + const timeoutValue = parseInt( + fuzzTest.stderr.match(/timeout after (\d+) seconds/)[1], + ); + // expect the actual timeout to be in the range [expectedTimeout - 1, expectedTimeout + 1] + expect(timeoutValue).toBeGreaterThanOrEqual(expectedTimeout - 1); + expect(timeoutValue).toBeLessThanOrEqual(expectedTimeout + 1); +} diff --git a/tests/jest_integration/jest_project/integration.fuzz.js b/tests/jest_integration/jest_project/integration.fuzz.js index 0cd64c5f..8f339d71 100644 --- a/tests/jest_integration/jest_project/integration.fuzz.js +++ b/tests/jest_integration/jest_project/integration.fuzz.js @@ -15,6 +15,14 @@ */ const target = require("./target.js"); +const mappedTarget = require("mappedModuleName"); + +jest.mock("./target.js", () => ({ + ...jest.requireActual("./target.js"), + originalFunction: () => { + throw "the function was mocked"; + }, +})); describe("Jest Integration", () => { it.fuzz("execute sync test", (data) => { @@ -32,4 +40,54 @@ describe("Jest Integration", () => { it.fuzz("execute async test using a callback", (data, done) => { target.callbackFuzzMe(data, done); }); + + it.fuzz("execute async timeout test", async (data) => { + await target.asyncTimeout(data); + }); + + it.fuzz( + "execute async timeout test with method timeout", + async (data) => { + await target.asyncTimeout(data); + }, + 10, + ); + + it.fuzz( + "execute async timeout test using a callback", + (data, done) => { + target.callbackTimeout(data, done); + }, + 10, + ); + + // noinspection JSUnusedLocalSymbols + it.fuzz("honor test name pattern", (data) => { + // Do nothing, as this test is only used to check thi test name pattern. + }); + + // noinspection JSUnusedLocalSymbols + it.fuzz("honor test name pattern as well", (data) => { + throw new Error("This test should not be executed!"); + }); + + it.fuzz("mock test function", (data) => { + target.originalFunction(data); + }); + + it.fuzz("load by mapped module name", (data) => { + mappedTarget.fuzzMe(data); + }); +}); + +describe("Run mode", () => { + describe("skip and standard", () => { + it.fuzz("standard test", (data) => { + console.log("standard test called"); + }); + + it.skip.fuzz("skipped test", (data) => { + throw new Error("Skipped test not skipped!"); + }); + }); }); diff --git a/tests/jest_integration/jest_project/package.json b/tests/jest_integration/jest_project/package.json index 685d1699..ad3372e1 100644 --- a/tests/jest_integration/jest_project/package.json +++ b/tests/jest_integration/jest_project/package.json @@ -15,14 +15,17 @@ "displayName": "test" }, { - "runner": "@jazzer.js/jest-runner", + "testRunner": "@jazzer.js/jest-runner", "displayName": { "name": "Jazzer.js", "color": "cyan" }, "testMatch": [ "/**/*.fuzz.js" - ] + ], + "moduleNameMapper": { + "mappedModuleName": "/target.js" + } } ] } diff --git a/tests/jest_integration/jest_project/run-mode-only.fuzz.js b/tests/jest_integration/jest_project/run-mode-only.fuzz.js new file mode 100644 index 00000000..af7ab7d2 --- /dev/null +++ b/tests/jest_integration/jest_project/run-mode-only.fuzz.js @@ -0,0 +1,25 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +describe("Run mode only and standard", () => { + it.fuzz("standard test", (data) => { + throw new Error("Standard test should not be called when only is used!"); + }); + + it.only.fuzz("only test", (data) => { + console.log("only test called"); + }); +}); diff --git a/tests/jest_integration/jest_project/target.js b/tests/jest_integration/jest_project/target.js index b9a92bc2..f131be09 100644 --- a/tests/jest_integration/jest_project/target.js +++ b/tests/jest_integration/jest_project/target.js @@ -14,14 +14,15 @@ * limitations under the License. */ -const fuzzMe = function (data) { +const fuzzMe = (data) => { if (data.toString() === "Awesome") { throw Error("Welcome to Awesome Fuzzing!"); } }; +module.exports.fuzzMe = fuzzMe; -const asyncFuzzMe = function (data) { - return new Promise((resolve, reject) => { +module.exports.asyncFuzzMe = (data) => + new Promise((resolve, reject) => { try { fuzzMe(data); resolve(); @@ -29,9 +30,8 @@ const asyncFuzzMe = function (data) { reject(e); } }); -}; -const callbackFuzzMe = function (data, done) { +module.exports.callbackFuzzMe = (data, done) => { setImmediate(() => { try { fuzzMe(data); @@ -42,6 +42,17 @@ const callbackFuzzMe = function (data, done) { }); }; -module.exports.fuzzMe = fuzzMe; -module.exports.asyncFuzzMe = asyncFuzzMe; -module.exports.callbackFuzzMe = callbackFuzzMe; +module.exports.originalFunction = () => { + throw Error("Original function invoked!"); +}; + +// noinspection JSUnusedLocalSymbols +module.exports.asyncTimeout = (data) => + new Promise(() => { + // Never resolve this promise to provoke a timeout. + }); + +// noinspection JSUnusedLocalSymbols +module.exports.callbackTimeout = (data, done) => { + // Never call done to provoke a timeout. +}; diff --git a/tests/jest_integration/jest_project_with_single_test/.gitignore b/tests/jest_integration/jest_project_with_single_test/.gitignore new file mode 100644 index 00000000..0e38fc1e --- /dev/null +++ b/tests/jest_integration/jest_project_with_single_test/.gitignore @@ -0,0 +1,3 @@ +.jazzerjsrc.json +.cifuzz-corpus +integration.fuzz diff --git a/tests/jest_integration/jest_project_with_single_test/integration.fuzz.js b/tests/jest_integration/jest_project_with_single_test/integration.fuzz.js new file mode 100644 index 00000000..946711a1 --- /dev/null +++ b/tests/jest_integration/jest_project_with_single_test/integration.fuzz.js @@ -0,0 +1,23 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +describe("Jest Integration", () => { + test.fuzz("one test only", (data) => { + if (data.toString() === "Welcome") { + throw Error("Welcome to Awesome Fuzzing!"); + } + }); +}); diff --git a/tests/jest_integration/jest_project_with_single_test/package.json b/tests/jest_integration/jest_project_with_single_test/package.json new file mode 100644 index 00000000..ddb80201 --- /dev/null +++ b/tests/jest_integration/jest_project_with_single_test/package.json @@ -0,0 +1,29 @@ +{ + "name": "jazzerjs-jest-integration-tests-project-single-test", + "version": "1.0.0", + "scripts": { + "test": "jest", + "fuzz": "JAZZER_FUZZ=1 jest " + }, + "devDependencies": { + "@jazzer.js/jest-runner": "file:../../../packages/jest-runner", + "jest": "^29.3.1" + }, + "jest": { + "projects": [ + { + "displayName": "test" + }, + { + "testRunner": "@jazzer.js/jest-runner", + "displayName": { + "name": "Jazzer.js", + "color": "cyan" + }, + "testMatch": [ + "/**/*.fuzz.js" + ] + } + ] + } +} diff --git a/tests/signal_handlers/SIGINT/fuzz.js b/tests/signal_handlers/SIGINT/fuzz.js index 1bf6333e..d91ab76b 100644 --- a/tests/signal_handlers/SIGINT/fuzz.js +++ b/tests/signal_handlers/SIGINT/fuzz.js @@ -18,7 +18,7 @@ let i = 0; module.exports.SIGINT_SYNC = (data) => { if (i === 1000) { - console.error("kill with signal"); + console.log("kill with signal"); process.kill(process.pid, "SIGINT"); } if (i > 1000) { @@ -31,7 +31,7 @@ module.exports.SIGINT_ASYNC = (data) => { // Raising SIGINT in async mode does not stop the fuzzer directly, // as the event is handled asynchronously in the event loop. if (i === 1000) { - console.error("kill with signal"); + console.log("kill with signal"); process.kill(process.pid, "SIGINT"); } i++; diff --git a/tests/signal_handlers/SIGINT/package.json b/tests/signal_handlers/SIGINT/package.json index 5c5cc066..5418555a 100644 --- a/tests/signal_handlers/SIGINT/package.json +++ b/tests/signal_handlers/SIGINT/package.json @@ -12,7 +12,7 @@ "jest": { "projects": [ { - "runner": "@jazzer.js/jest-runner", + "testRunner": "@jazzer.js/jest-runner", "displayName": { "name": "Jazzer.js", "color": "cyan" diff --git a/tests/signal_handlers/SIGSEGV/fuzz.js b/tests/signal_handlers/SIGSEGV/fuzz.js index f2afeec8..b6ac855a 100644 --- a/tests/signal_handlers/SIGSEGV/fuzz.js +++ b/tests/signal_handlers/SIGSEGV/fuzz.js @@ -18,11 +18,11 @@ let i = 0; module.exports.SIGSEGV_SYNC = (data) => { if (i === 1000) { - console.error("kill with signal"); + console.log("kill with signal"); process.kill(process.pid, "SIGSEGV"); } if (i > 1000) { - console.error("Signal has not stopped the fuzzing process"); + console.log("Signal has not stopped the fuzzing process"); } i++; }; @@ -31,7 +31,7 @@ module.exports.SIGSEGV_ASYNC = (data) => { // Raising SIGSEGV in async mode does not stop the fuzzer directly, // as the event is handled asynchronously in the event loop. if (i === 1000) { - console.error("kill with signal"); + console.log("kill with signal"); process.kill(process.pid, "SIGSEGV"); } i++; diff --git a/tests/signal_handlers/SIGSEGV/package.json b/tests/signal_handlers/SIGSEGV/package.json index 07baf8b7..0c3522d2 100644 --- a/tests/signal_handlers/SIGSEGV/package.json +++ b/tests/signal_handlers/SIGSEGV/package.json @@ -12,7 +12,7 @@ "jest": { "projects": [ { - "runner": "@jazzer.js/jest-runner", + "testRunner": "@jazzer.js/jest-runner", "displayName": { "name": "Jazzer.js", "color": "cyan" diff --git a/tests/signal_handlers/signal_handlers.test.js b/tests/signal_handlers/signal_handlers.test.js index b56eef56..7e98d01b 100644 --- a/tests/signal_handlers/signal_handlers.test.js +++ b/tests/signal_handlers/signal_handlers.test.js @@ -131,7 +131,7 @@ describe("SIGSEGV handlers", () => { }); function assertSignalMessagesLogged(fuzzTest) { - expect(fuzzTest.stderr).toContain("kill with signal"); + expect(fuzzTest.stdout).toContain("kill with signal"); // We asked for a coverage report. Here we only look for the universal part of its header. // Jest prints to stdout. @@ -140,7 +140,7 @@ function assertSignalMessagesLogged(fuzzTest) { ); // Count how many times "Signal has not stopped the fuzzing process" has been printed. - const matches = fuzzTest.stderr.match( + const matches = fuzzTest.stdout.match( /Signal has not stopped the fuzzing process/g, ); const signalNotStoppedMessageCount = matches ? matches.length : 0; From 56273506388c397716a704cb93641fafd63a52da Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Mon, 11 Sep 2023 12:14:39 +0200 Subject: [PATCH 10/18] jest: Use generated inputs in coverage runs --- packages/jest-runner/corpus.test.ts | 20 +++++++++++++ packages/jest-runner/corpus.ts | 46 ++++++++++++++++++----------- packages/jest-runner/fuzz.ts | 2 +- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/packages/jest-runner/corpus.test.ts b/packages/jest-runner/corpus.test.ts index 8866c92d..c427cadf 100644 --- a/packages/jest-runner/corpus.test.ts +++ b/packages/jest-runner/corpus.test.ts @@ -70,6 +70,14 @@ describe("Corpus", () => { expect(corpus.inputsPaths()).toHaveLength(0); }); + + it("include generated corpus files in fuzzing mode", () => { + const fuzzTest = mockFuzzTest({ seedFiles: 2, generatedInputFiles: 3 }); + + const corpus = new Corpus(fuzzTest, [], true); + + expect(corpus.inputsPaths()).toHaveLength(5); + }); }); describe("corpusDirectory", () => { @@ -94,6 +102,7 @@ function mockFuzzTest({ seedFiles = 0, subDirs = 0, generatePackageJson = true, + generatedInputFiles = 0, } = {}) { const tmpDir = tmp.dirSync({ unsafeCleanup: true }).name; const fuzzTestName = "fuzztest"; @@ -114,5 +123,16 @@ function mockFuzzTest({ for (let i = 0; i < subDirs; i++) { fs.mkdirSync(path.join(tmpDir, fuzzTestName, i.toString())); } + if (generatedInputFiles > 0) { + fs.mkdirSync(path.join(tmpDir, ".cifuzz-corpus", fuzzTestName), { + recursive: true, + }); + for (let i = 0; i < generatedInputFiles; i++) { + fs.writeFileSync( + path.join(tmpDir, ".cifuzz-corpus", fuzzTestName, i.toString()), + i.toString(), + ); + } + } return fuzzTestFile; } diff --git a/packages/jest-runner/corpus.ts b/packages/jest-runner/corpus.ts index 430d85d5..2c9ee693 100644 --- a/packages/jest-runner/corpus.ts +++ b/packages/jest-runner/corpus.ts @@ -25,8 +25,14 @@ export class Corpus { private readonly _seedInputsDirectory: string; // Directory containing runtime generated fuzzer inputs. private readonly _generatedInputsDirectory: string; + // Indicate if coverage is enabled. + private readonly _coverage: boolean; - constructor(testSourceFilePath: string, testJestPathElements: string[]) { + constructor( + testSourceFilePath: string, + testJestPathElements: string[], + coverage: boolean = false, + ) { this._seedInputsDirectory = directoryPathForTest( testSourceFilePath, testJestPathElements, @@ -36,7 +42,11 @@ export class Corpus { testJestPathElements, Corpus.defaultCorpusDirectory, ); - this.createMissingDirectories(); + this._coverage = coverage; + createMissingDirectories( + this._seedInputsDirectory, + this._generatedInputsDirectory, + ); } get seedInputsDirectory(): string { @@ -48,23 +58,26 @@ export class Corpus { } inputsPaths(): [string, string][] { + const seedInputs = this.inputFiles(this._seedInputsDirectory); + if (this._coverage) { + return seedInputs.concat(this.inputFiles(this._generatedInputsDirectory)); + } + return seedInputs; + } + + private inputFiles(directory: string): [string, string][] { return fs - .readdirSync(this._seedInputsDirectory) + .readdirSync(directory) .filter( - (entry) => - !fs - .lstatSync(path.join(this.seedInputsDirectory, entry)) - .isDirectory(), + (entry) => !fs.lstatSync(path.join(directory, entry)).isDirectory(), ) - .map((file) => [file, path.join(this._seedInputsDirectory, file)]); - } - - private createMissingDirectories() { - fs.mkdirSync(this._seedInputsDirectory, { recursive: true }); - fs.mkdirSync(this._generatedInputsDirectory, { recursive: true }); + .map((file) => [file, path.join(directory, file)]); } } +const createMissingDirectories = (...dirs: string[]) => + dirs.forEach((dir) => fs.mkdirSync(dir, { recursive: true })); + const directoryPathForTest = ( testSourceFilePath: string, testJestPathElements: string[], @@ -86,18 +99,16 @@ const buildRootDirectory = ( ): string => { const inputsRoot = path.parse(testSourceFilePath); const testName = inputsRoot.name; - let mainDir = inputsRoot.dir; if (projectCorpusRoot !== "") { // looking for the root directory of the project - mainDir = path.join( + return path.join( findDirectoryWithPackageJson(inputsRoot).dir, projectCorpusRoot, testName, ); } else { - mainDir = path.join(inputsRoot.dir, testName); + return path.join(inputsRoot.dir, testName); } - return mainDir; }; const findDirectoryWithPackageJson = ( @@ -109,7 +120,6 @@ const findDirectoryWithPackageJson = ( throw new Error("Could not find package.json in any parent directory"); } } - return directory; }; diff --git a/packages/jest-runner/fuzz.ts b/packages/jest-runner/fuzz.ts index 5710cace..498899d7 100644 --- a/packages/jest-runner/fuzz.ts +++ b/packages/jest-runner/fuzz.ts @@ -83,7 +83,7 @@ export function fuzz( return; } - const corpus = new Corpus(testFile, testStatePath); + const corpus = new Corpus(testFile, testStatePath, localConfig.coverage); // Timeout priority is: // 1. Use timeout directly defined in test function From 756b771ed3d1c7cefd98dbb5afa9984f314af6cc Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Tue, 12 Sep 2023 09:59:55 +0200 Subject: [PATCH 11/18] core: Update finding stack cleanup --- packages/core/core.ts | 34 ++++++------- packages/core/finding.test.ts | 96 +++++++++++++++++++++++++++++++++++ packages/core/finding.ts | 92 +++++++++++++++++---------------- 3 files changed, 160 insertions(+), 62 deletions(-) create mode 100644 packages/core/finding.test.ts diff --git a/packages/core/core.ts b/packages/core/core.ts index 8e291cf0..b1829b71 100644 --- a/packages/core/core.ts +++ b/packages/core/core.ts @@ -28,6 +28,7 @@ import { getFirstFinding, printFinding, Finding, + cleanErrorStack, } from "./finding"; import { FileSyncIdStrategy, @@ -322,15 +323,11 @@ export function wrapFuzzFunctionForBugDetection( originalFuzzFn: fuzzer.FuzzTarget, ): fuzzer.FuzzTarget { function throwIfError(fuzzTargetError?: unknown): undefined | never { - const error = getFirstFinding(); - if (error !== undefined) { - // The `firstFinding` is a global variable: we need to clear it after each fuzzing iteration. - clearFirstFinding(); + const error = clearFirstFinding() ?? fuzzTargetError; + if (error) { + cleanErrorStack(error); throw error; - } else if (fuzzTargetError) { - throw fuzzTargetError; } - return undefined; } if (originalFuzzFn.length === 1) { @@ -350,17 +347,16 @@ export function wrapFuzzFunctionForBugDetection( return throwIfError() ?? result; }, (reason) => { + callbacks.runAfterEachCallbacks(); return throwIfError(reason); }, ); + } else { + callbacks.runAfterEachCallbacks(); } } catch (e) { fuzzTargetError = e; } - // Promises are handled above, so we only need to handle sync results here. - if (!(result instanceof Promise)) { - callbacks.runAfterEachCallbacks(); - } return throwIfError(fuzzTargetError) ?? result; }; } else { @@ -368,23 +364,23 @@ export function wrapFuzzFunctionForBugDetection( data: Buffer, done: (err?: Error) => void, ): unknown | Promise => { - let result: unknown | Promise = undefined; try { callbacks.runBeforeEachCallbacks(); // Return result of fuzz target to enable sanity checks in C++ part. - result = originalFuzzFn(data, (err?: Error) => { - const finding = getFirstFinding(); - if (finding !== undefined) { - clearFirstFinding(); - } + const result = originalFuzzFn(data, (err?) => { + const error = clearFirstFinding() ?? err; + cleanErrorStack(error); callbacks.runAfterEachCallbacks(); - done(finding ?? err); + done(error); }); + // Check if any finding was reported by the invocation before the + // callback was executed. As the callback in used for control flow, + // don't run afterEach here. + return throwIfError() ?? result; } catch (e) { callbacks.runAfterEachCallbacks(); throwIfError(e); } - return result; }; } } diff --git a/packages/core/finding.test.ts b/packages/core/finding.test.ts new file mode 100644 index 00000000..a9615cff --- /dev/null +++ b/packages/core/finding.test.ts @@ -0,0 +1,96 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Finding, printFinding } from "./finding"; + +import { sep } from "path"; + +describe("Finding", () => { + it("print a cleaned up finding", () => { + const printer = mockPrinter(); + const error = new Finding("Welcome to Awesome Fuzzing!"); + error.stack = withSystemSeparator(`Error: Welcome to Awesome Fuzzing! + at Object.Error [as fuzzMe] (/home/Code-Intelligence/jazzer.js/tests/jest_integration/jest_project/target.js:19:9) + at fuzzMe (/home/Code-Intelligence/jazzer.js/tests/jest_integration/jest_project/integration.fuzz.js:30:11) + at /home/Code-Intelligence/jazzer.js/packages/core/core.ts:341:5 + at /home/Code-Intelligence/jazzer.js/packages/jest-runner/fuzz.ts:152:6`); + + printFinding(error, printer); + + const lines = printer.printed().split("\n"); + expect(lines).toHaveLength(4); + expect(lines[0]).toMatch(/==\d*== Welcome to Awesome Fuzzing!/); + expect(lines[1]).toContain( + withSystemSeparator( + ` at Object.Error [as fuzzMe] (/home/Code-Intelligence/jazzer.js/tests/jest_integration/jest_project/target.js:19:9)`, + ), + ); + expect(lines[2]).toContain( + withSystemSeparator( + ` at fuzzMe (/home/Code-Intelligence/jazzer.js/tests/jest_integration/jest_project/integration.fuzz.js:30:11)`, + ), + ); + expect(lines[3]).toEqual(""); + }); + + it("print a cleaned up bug detector finding", () => { + const printer = mockPrinter(); + const error = new Finding( + "Command Injection in execSync(): called with 'jaz_zer'", + ); + error.stack = + withSystemSeparator(`Error: Command Injection in execSync(): called with 'jaz_zer' + at reportFinding (/home/Code-Intelligence/jazzer.js/packages/core/finding.ts:54:1) + at Hook.beforeHook [as hookFunction] (/home/Code-Intelligence/jazzer.js/packages/bug-detectors/internal/command-injection.ts:52:17) + at Object.execSync (/home/Code-Intelligence/jazzer.js/packages/hooking/manager.ts:260:3) + at test (/home/Code-Intelligence/jazzer.js/tests/bug-detectors/general/tests.fuzz.js:68:17) + at /home/Code-Intelligence/jazzer.js/tests/bug-detectors/general/tests.fuzz.js:26:3 + at /home/Code-Intelligence/jazzer.js/packages/core/core.ts:341:5 + at /home/Code-Intelligence/jazzer.js/packages/jest-runner/fuzz.ts:152:6`); + + printFinding(error, printer); + + const lines = printer.printed().split("\n"); + expect(lines).toHaveLength(4); + expect(lines[0]).toMatch( + /==\d*== Command Injection in execSync\(\): called with 'jaz_zer'/, + ); + expect(lines[1]).toContain( + withSystemSeparator( + ` at test (/home/Code-Intelligence/jazzer.js/tests/bug-detectors/general/tests.fuzz.js:68:17)`, + ), + ); + expect(lines[2]).toContain( + withSystemSeparator( + ` at /home/Code-Intelligence/jazzer.js/tests/bug-detectors/general/tests.fuzz.js:26:3`, + ), + ); + expect(lines[3]).toEqual(""); + }); +}); + +function mockPrinter() { + const _messages: string[] = []; + const printer = (msg: string) => { + _messages.push(msg); + }; + printer.printed = () => _messages.join(""); + return printer; +} + +function withSystemSeparator(text: string): string { + return text.replaceAll(/\//g, sep); +} diff --git a/packages/core/finding.ts b/packages/core/finding.ts index 2a7b62f3..9cb31cd5 100644 --- a/packages/core/finding.ts +++ b/packages/core/finding.ts @@ -15,6 +15,8 @@ */ import process from "process"; +import { EOL } from "os"; +import { sep } from "path"; export class Finding extends Error {} @@ -29,8 +31,10 @@ export function getFirstFinding(): Finding | undefined { return firstFinding; } -export function clearFirstFinding(): void { +export function clearFirstFinding(): Finding | undefined { + const lastFinding = firstFinding; firstFinding = undefined; + return lastFinding; } /** @@ -49,61 +53,63 @@ export function reportFinding(findingMessage: string): void | never { } /** - * Prints a finding, or more generally some kind of error, to stdout. + * Prints a finding, or more generally some kind of error, to stderr. */ -export function printFinding(error: unknown) { - let errorMessage = `==${process.pid}== `; +export function printFinding( + error: unknown, + print: (msg: string) => void = process.stderr.write.bind(process.stderr), +): void { + print(`==${process.pid}== `); if (!(error instanceof Finding)) { - errorMessage += "Uncaught Exception: Jazzer.js: "; + print("Uncaught Exception: Jazzer.js: "); } - if (error instanceof Error) { - errorMessage += error.message; - console.error(errorMessage); if (error.stack) { - console.error(cleanErrorStack(error)); + cleanErrorStack(error); + print(error.stack); + } else { + print(error.message); } } else if (typeof error === "string" || error instanceof String) { - errorMessage += error; - console.error(errorMessage); + print(error.toString()); } else { - errorMessage += "unknown"; - console.error(errorMessage); + print("unknown"); } + print(EOL); } -function cleanErrorStack(error: Error): string { - if (error.stack === undefined) return ""; +interface WithStack { + stack?: string; +} - // This cleans up the stack of a finding. The changes are independent of each other, since a finding can be - // thrown from the hooking library, by the custom hooks, or by the fuzz target. - if (error instanceof Finding) { - // Remove the message from the stack trace. Also remove the subsequent line of the remaining stack trace that - // always contains `reportFinding()`, which is not relevant for the user. - error.stack = error.stack - ?.replace(`Error: ${error.message}\n`, "") - .replace(/.*\n/, ""); +function hasStack(arg: unknown): arg is WithStack { + return ( + arg !== undefined && arg !== null && (arg as WithStack).stack !== undefined + ); +} - // Remove all lines up to and including the line that mentions the hooking library from the stack trace of a - // finding. - const stack = error.stack.split("\n"); - const index = stack.findIndex((line) => - line.includes("jazzer.js/packages/hooking/manager"), +export function cleanErrorStack(error: unknown): void { + if (!hasStack(error) || !error.stack) return; + if (error instanceof Finding) { + // Remove the "Error :" prefix of the finding message from the stack trace. + error.stack = error.stack.replace( + `Error: ${error.message}\n`, + `${error.message}\n`, ); - if (index !== undefined && index >= 0) { - error.stack = stack.slice(index + 1).join("\n"); - } - - // also delete all lines that mention "jazzer.js/packages/" - error.stack = error.stack.replace(/.*jazzer.js\/packages\/.*\n/g, ""); - } - - const result: string[] = []; - for (const line of error.stack.split("\n")) { - if (line.includes("jazzer.js/packages/core/core.ts")) { - break; - } - result.push(line); } - return result.join("\n"); + // Ignore all lines related to Jazzer.js internals. This includes stack frames on top, + // like bug detector and reporting ones, and stack frames on the bottom, like the function + // wrapper. + const filterCriteria = [ + `@jazzer.js${sep}`, // cli usage + `jazzer.js${sep}packages${sep}`, // jest usage + `jazzer.js${sep}core${sep}`, // jest usage + `..${sep}..${sep}packages${sep}core${sep}`, // local/filesystem dependencies + ]; + error.stack = error.stack + .split("\n") + .filter( + (line) => !filterCriteria.some((criterion) => line.includes(criterion)), + ) + .join("\n"); } From 929d39cf46b63c5590683b7fccc493c36dd19bda Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Tue, 12 Sep 2023 10:28:17 +0200 Subject: [PATCH 12/18] jest: Update error stack cleanup --- packages/jest-runner/errorUtils.test.ts | 100 +-- packages/jest-runner/errorUtils.ts | 36 +- packages/jest-runner/fuzz.ts | 2 +- packages/jest-runner/index.ts | 22 +- tests/bug-detectors/command-injection.test.js | 4 +- tests/bug-detectors/general.test.js | 617 +++++++++--------- tests/bug-detectors/package.json | 3 + tests/bug-detectors/path-traversal.test.js | 7 +- tests/helpers.js | 54 +- tests/jest_integration/integration.test.js | 53 +- .../jest_project/integration.fuzz.js | 2 +- 11 files changed, 490 insertions(+), 410 deletions(-) diff --git a/packages/jest-runner/errorUtils.test.ts b/packages/jest-runner/errorUtils.test.ts index e8db960f..144e6e27 100644 --- a/packages/jest-runner/errorUtils.test.ts +++ b/packages/jest-runner/errorUtils.test.ts @@ -16,24 +16,29 @@ import { cleanupJestError, - cleanupJestRunnerStack, - removeBottomFrames, - removeBottomFramesFromError, removeTopFrames, removeTopFramesFromError, } from "./errorUtils"; describe("ErrorUtils", () => { const error = new Error(); - const stack = `Error: - at /jest_integration/integration.fuzz.js:27:3 - at doneCallbackPromise (jazzer.js/packages/jest-runner/dist/fuzz.js:213:20) - at Promise.then._a (jazzer.js/packages/jest-runner/dist/fuzz.js:169:20) - at new Promise () - at jazzer.js/packages/jest-runner/dist/fuzz.js:162:16 - at Generator.next () - at fulfilled (jazzer.js/packages/jest-runner/dist/fuzz.js:58:24) -`; + const stack = `Error: thrown: "Exceeded timeout of 5000 ms for a test. +Add a timeout value to this test to increase the timeout, if this is a long-running test. See https://jestjs.io/docs/api#testname-fn-timeout." + at /home/Code-Intelligence/jazzer.js/packages/jest-runner/fuzz.ts:163:3 + at _dispatchDescribe (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-circus/build/index.js:91:26) + at describe (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-circus/build/index.js:55:5) + at runInRegressionMode (/home/Code-Intelligence/jazzer.js/packages/jest-runner/fuzz.ts:145:24) + at Function.fuzz (/home/Code-Intelligence/jazzer.js/packages/jest-runner/fuzz.ts:110:20) + at fuzz (/home/Code-Intelligence/jazzer.js/tests/jest_integration/jest_project/integration.fuzz.js:44:5) + at _dispatchDescribe (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-circus/build/index.js:91:26) + at describe (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-circus/build/index.js:55:5) + at Object.describe (/home/Code-Intelligence/jazzer.js/tests/jest_integration/jest_project/integration.fuzz.js:27:1) + at Runtime._execModule (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-runtime/build/index.js:1439:24) + at Runtime._loadModule (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-runtime/build/index.js:1022:12) + at Runtime.requireModule (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-runtime/build/index.js:882:12) + at jestAdapter (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:77:13) + at runTestInternal (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-runner/build/runTest.js:367:16) + at runTest (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-runner/build/runTest.js:444:34)`; beforeEach(() => { error.stack = stack; @@ -41,68 +46,23 @@ describe("ErrorUtils", () => { describe("clean up Jest runner frames", () => { it("in errors", () => { - expect(cleanupJestError(undefined)).toBeUndefined(); const result = cleanupJestError(error); expect(result instanceof Error).toBeTruthy(); if (result instanceof Error) { - expect(result.stack).toMatch(`Error: - at /jest_integration/integration.fuzz.js:27:3 -`); + expect(result.stack) + .toMatch(`Error: thrown: "Exceeded timeout of 5000 ms for a test. +Add a timeout value to this test to increase the timeout, if this is a long-running test. See https://jestjs.io/docs/api#testname-fn-timeout." + at fuzz (/home/Code-Intelligence/jazzer.js/tests/jest_integration/jest_project/integration.fuzz.js:44:5) + at _dispatchDescribe (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-circus/build/index.js:91:26) + at describe (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-circus/build/index.js:55:5) + at Object.describe (/home/Code-Intelligence/jazzer.js/tests/jest_integration/jest_project/integration.fuzz.js:27:1) + at Runtime._execModule (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-runtime/build/index.js:1439:24) + at Runtime._loadModule (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-runtime/build/index.js:1022:12) + at Runtime.requireModule (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-runtime/build/index.js:882:12) + at jestAdapter (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:77:13) + at runTestInternal (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-runner/build/runTest.js:367:16) + at runTest (/home/Code-Intelligence/jazzer.js/tests/jest_integration/node_modules/jest-runner/build/runTest.js:444:34)`); } }); - - it("in stacks", () => { - expect(cleanupJestRunnerStack(undefined)).toBeUndefined(); - expect(cleanupJestRunnerStack(stack)).toMatch(`Error: - at /jest_integration/integration.fuzz.js:27:3 -`); - }); - }); - - describe("remove stack frames", () => { - describe("in errors", () => { - it("on top of remaining", () => { - expect(removeTopFramesFromError(undefined, 1)).toBeUndefined(); - expect(removeTopFramesFromError(error, 3)?.stack).toMatch(`Error: - at new Promise () - at jazzer.js/packages/jest-runner/dist/fuzz.js:162:16 - at Generator.next () - at fulfilled (jazzer.js/packages/jest-runner/dist/fuzz.js:58:24) -`); - }); - - it("on bottom of remaining", () => { - expect(removeBottomFramesFromError(undefined, 1)).toBeUndefined(); - const cleanedUp = removeBottomFramesFromError(error, 3); - expect(cleanedUp?.stack).toMatch(`Error: - at /jest_integration/integration.fuzz.js:27:3 - at doneCallbackPromise (jazzer.js/packages/jest-runner/dist/fuzz.js:213:20) - at Promise.then._a (jazzer.js/packages/jest-runner/dist/fuzz.js:169:20) - at new Promise () -`); - }); - }); - - describe("in stacks", () => { - it("on top of remaining", () => { - expect(removeTopFrames(undefined, 1)).toBeUndefined(); - expect(removeTopFrames(stack, 3)).toMatch(`Error: - at new Promise () - at jazzer.js/packages/jest-runner/dist/fuzz.js:162:16 - at Generator.next () - at fulfilled (jazzer.js/packages/jest-runner/dist/fuzz.js:58:24) -`); - }); - - it("on bottom of remaining", () => { - expect(removeBottomFrames(undefined, 1)).toBeUndefined(); - expect(removeBottomFrames(stack, 3)).toMatch(`Error: - at /jest_integration/integration.fuzz.js:27:3 - at doneCallbackPromise (jazzer.js/packages/jest-runner/dist/fuzz.js:213:20) - at Promise.then._a (jazzer.js/packages/jest-runner/dist/fuzz.js:169:20) - at new Promise () -`); - }); - }); }); }); diff --git a/packages/jest-runner/errorUtils.ts b/packages/jest-runner/errorUtils.ts index 273600dd..3214e668 100644 --- a/packages/jest-runner/errorUtils.ts +++ b/packages/jest-runner/errorUtils.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -export const cleanupJestError = (error: Error | unknown): Error | unknown => { +export const cleanupJestError = (error: unknown): unknown => { if (error == undefined) { return undefined; } @@ -24,20 +24,29 @@ export const cleanupJestError = (error: Error | unknown): Error | unknown => { return error; }; -export const cleanupJestRunnerStack = ( - stack: string | undefined, -): string | undefined => { +export const cleanupJestRunnerStack = (stack?: string): string | undefined => { + function isStackFrame(frame: string) { + return frame.indexOf(" at ") !== -1; + } + function isRunnerFrame(frame: string) { + return ( + frame.indexOf("jest-runner") !== -1 || frame.indexOf("jest-circus") !== -1 + ); + } if (!stack) { return stack; } - let foundFirstJestRunnerFrame = false; + let foundFirstNoneRunnerFrame = false; const newStack = stack .split("\n") .filter((frame) => { - if (frame.indexOf("jest-runner") != -1) { - foundFirstJestRunnerFrame = true; + if (!isStackFrame(frame)) { + return true; } - return !foundFirstJestRunnerFrame; + if (foundFirstNoneRunnerFrame || !isRunnerFrame(frame)) { + foundFirstNoneRunnerFrame = true; + } + return foundFirstNoneRunnerFrame; }) .join("\n"); return stack.endsWith("\n") ? newStack + "\n" : newStack; @@ -66,17 +75,6 @@ export const removeTopFrames = ( return frames.join("\n"); }; -export const removeBottomFramesFromError = ( - error: Error | undefined, - drop: number, -): Error | undefined => { - if (error == undefined) { - return error; - } - error.stack = removeBottomFrames(error.stack, drop); - return error; -}; - export const removeBottomFrames = ( stack: string | undefined, drop: number, diff --git a/packages/jest-runner/fuzz.ts b/packages/jest-runner/fuzz.ts index 498899d7..48971bd1 100644 --- a/packages/jest-runner/fuzz.ts +++ b/packages/jest-runner/fuzz.ts @@ -23,7 +23,7 @@ import { import { TIMEOUT_PLACEHOLDER } from "./config"; import { Corpus } from "./corpus"; import * as fs from "fs"; -import { removeTopFramesFromError } from "./errorUtils"; +import { cleanupJestError, removeTopFramesFromError } from "./errorUtils"; import { defaultOptions, Options, diff --git a/packages/jest-runner/index.ts b/packages/jest-runner/index.ts index 610cd445..a0fbd0a2 100644 --- a/packages/jest-runner/index.ts +++ b/packages/jest-runner/index.ts @@ -23,11 +23,11 @@ import type { JestEnvironment } from "@jest/environment"; import { initFuzzing, setJazzerJsGlobal } from "@jazzer.js/core"; import { loadConfig } from "./config"; -import { cleanupJestError } from "./errorUtils"; import { FuzzTest } from "./fuzz"; import { interceptScriptTransformerCalls } from "./transformerInterceptor"; import { interceptTestState } from "./testStateInterceptor"; import { interceptGlobals } from "./globalsInterceptor"; +import { cleanupJestError, cleanupJestRunnerStack } from "./errorUtils"; export default async function jazzerTestRunner( globalConfig: Config.GlobalConfig, @@ -65,13 +65,25 @@ export default async function jazzerTestRunner( testPath, sendMessageToJest, ).then((result: TestResult) => { - result.testResults.forEach((testResult) => { - testResult.failureDetails?.forEach(cleanupJestError); - }); - return result; + return cleanupTestResultDetails(result); }); } +function cleanupTestResultDetails(result: TestResult) { + // Some errors, like timeouts, are created in Jest's test runner and need to be + // post-processed to remove internal stack frames in this way. + result.testResults.forEach((testResult) => { + testResult.failureDetails?.forEach(cleanupJestError); + testResult.failureMessages = testResult.failureMessages?.map( + (failureMessage) => cleanupJestRunnerStack(failureMessage) ?? "", + ); + }); + if (result.failureMessage) { + result.failureMessage = cleanupJestRunnerStack(result.failureMessage); + } + return result; +} + // Global definition of the Jest fuzz test extension function. // This is required to allow the Typescript compiler to recognize it. declare global { diff --git a/tests/bug-detectors/command-injection.test.js b/tests/bug-detectors/command-injection.test.js index fa937947..1d7f9262 100644 --- a/tests/bug-detectors/command-injection.test.js +++ b/tests/bug-detectors/command-injection.test.js @@ -17,14 +17,16 @@ const { FuzzTestBuilder, FuzzingExitCode } = require("../helpers.js"); const path = require("path"); const fs = require("fs"); +const { cleanCrashFilesIn } = require("../helpers"); describe("Command injection", () => { const bugDetectorDirectory = path.join(__dirname, "command-injection"); const friendlyFilePath = path.join(bugDetectorDirectory, "FRIENDLY"); // Delete files created by the tests. - beforeEach(() => { + beforeEach(async () => { fs.rmSync(friendlyFilePath, { force: true }); + await cleanCrashFilesIn(bugDetectorDirectory); }); it("exec with EVIL command", () => { diff --git a/tests/bug-detectors/general.test.js b/tests/bug-detectors/general.test.js index 04886733..9aa173b6 100644 --- a/tests/bug-detectors/general.test.js +++ b/tests/bug-detectors/general.test.js @@ -18,6 +18,8 @@ const { FuzzTestBuilder, FuzzingExitCode, JestRegressionExitCode, + cleanCrashFilesIn, + fileExists, } = require("../helpers.js"); const path = require("path"); const fs = require("fs"); @@ -36,338 +38,345 @@ describe("General tests", () => { } // Delete files created by the tests. - beforeEach(() => { - fs.rmSync(friendlyFilePath, { force: true }); - fs.rmSync(evilFilePath, { force: true }); - fs.rmSync("../jaz_zer", { force: true, recursive: true }); + beforeEach(async () => { + await fs.promises.rm(friendlyFilePath, { force: true }); + await fs.promises.rm(evilFilePath, { force: true }); + await fs.promises.rm("../jaz_zer", { force: true, recursive: true }); + await cleanCrashFilesIn(bugDetectorDirectory); }); - it("Call with EVIL string; ASYNC", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .fuzzEntryPoint("CallOriginalEvilAsync") - .dir(bugDetectorDirectory) - .build(); - expect(() => { + describe("CLI", () => { + it("Call with EVIL string; ASYNC", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .fuzzEntryPoint("CallOriginalEvilAsync") + .dir(bugDetectorDirectory) + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(FuzzingExitCode); + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + }); + + it("Call with EVIL string; SYNC", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(true) + .fuzzEntryPoint("CallOriginalEvilSync") + .dir(bugDetectorDirectory) + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(FuzzingExitCode); + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + expectErrorToBePrintedOnce(fuzzTest); + }); + + it("Call with FRIENDLY string; ASYNC", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .fuzzEntryPoint("CallOriginalFriendlyAsync") + .dir(bugDetectorDirectory) + .build(); fuzzTest.execute(); - }).toThrow(FuzzingExitCode); - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); - }); + expect(await fileExists(friendlyFilePath)).toBeTruthy(); + }); - it("Call with EVIL string; SYNC", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(true) - .fuzzEntryPoint("CallOriginalEvilSync") - .dir(bugDetectorDirectory) - .build(); - expect(() => { + it("Call with FRIENDLY string; SYNC", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(true) + .fuzzEntryPoint("CallOriginalFriendlySync") + .dir(bugDetectorDirectory) + .build(); fuzzTest.execute(); - }).toThrow(FuzzingExitCode); - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); - expectErrorToBePrintedOnce(fuzzTest); - }); + expect(await fileExists(friendlyFilePath)).toBeTruthy(); + }); - it("Call with FRIENDLY string; ASYNC", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .fuzzEntryPoint("CallOriginalFriendlyAsync") - .dir(bugDetectorDirectory) - .build(); - fuzzTest.execute(); - expect(fs.existsSync(friendlyFilePath)).toBeTruthy(); - }); + it("Call with EVIL string; With done callback", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .fuzzEntryPoint("CallOriginalEvilDoneCallback") + .dir(bugDetectorDirectory) + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(FuzzingExitCode); + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + expectErrorToBePrintedOnce(fuzzTest); + }); - it("Call with FRIENDLY string; SYNC", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(true) - .fuzzEntryPoint("CallOriginalFriendlySync") - .dir(bugDetectorDirectory) - .build(); - fuzzTest.execute(); - expect(fs.existsSync(friendlyFilePath)).toBeTruthy(); - }); + it("Call with EVIL string; With done callback; With try/catch", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .fuzzEntryPoint("CallOriginalEvilDoneCallbackWithTryCatch") + .dir(bugDetectorDirectory) + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(FuzzingExitCode); + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + expectErrorToBePrintedOnce(fuzzTest); + }); - it("Call with EVIL string; With done callback", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .fuzzEntryPoint("CallOriginalEvilDoneCallback") - .dir(bugDetectorDirectory) - .build(); - expect(() => { - fuzzTest.execute(); - }).toThrow(FuzzingExitCode); - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); - expectErrorToBePrintedOnce(fuzzTest); - }); + it("Call with EVIL string; With done callback; With timeout", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .fuzzEntryPoint("CallOriginalEvilDoneCallbackWithTimeout") + .dir(bugDetectorDirectory) + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow( + process.platform === "win32" ? JestRegressionExitCode : FuzzingExitCode, + ); + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + }); - it("Call with EVIL string; With done callback; With try/catch", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .fuzzEntryPoint("CallOriginalEvilDoneCallbackWithTryCatch") - .dir(bugDetectorDirectory) - .build(); - expect(() => { - fuzzTest.execute(); - }).toThrow(FuzzingExitCode); - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); - expectErrorToBePrintedOnce(fuzzTest); - }); + it("Call with EVIL string; With done callback; With timeout; With try/catch", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .fuzzEntryPoint("CallOriginalEvilDoneCallbackWithTimeoutWithTryCatch") + .dir(bugDetectorDirectory) + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(FuzzingExitCode); + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + }); - it("Call with EVIL string; With done callback; With timeout", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .fuzzEntryPoint("CallOriginalEvilDoneCallbackWithTimeout") - .dir(bugDetectorDirectory) - .build(); - expect(() => { + it("Call with FRIENDLY string; With done callback", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .fuzzEntryPoint("CallOriginalFriendlyDoneCallback") + .dir(bugDetectorDirectory) + .build(); fuzzTest.execute(); - }).toThrow( - process.platform === "win32" ? JestRegressionExitCode : FuzzingExitCode, - ); - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); - }); + expect(await fileExists(friendlyFilePath)).toBeTruthy(); + }); - it("Call with EVIL string; With done callback; With timeout; With try/catch", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .fuzzEntryPoint("CallOriginalEvilDoneCallbackWithTimeoutWithTryCatch") - .dir(bugDetectorDirectory) - .build(); - expect(() => { - fuzzTest.execute(); - }).toThrow(FuzzingExitCode); - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); - }); + it("Fork mode: Call with EVIL string; SYNC", async () => { + // TODO: Fork mode does not work in the Windows-Server image used by github actions + if (process.platform === "win32") { + console.error( + "// TODO: Fork mode does not work in the Windows-Server image used by github actions", + ); + return; + } + const fuzzTest = new FuzzTestBuilder() + .sync(false) + .fuzzEntryPoint("ForkModeCallOriginalEvil") + .dir(bugDetectorDirectory) + .runs(200) + .forkMode(3) + .build(); + fuzzTest.execute(); // fork mode doesn't throw errors + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + }); - it("Call with FRIENDLY string; With done callback", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .fuzzEntryPoint("CallOriginalFriendlyDoneCallback") - .dir(bugDetectorDirectory) - .build(); - fuzzTest.execute(); - expect(fs.existsSync(friendlyFilePath)).toBeTruthy(); - }); + it("Fork mode: Call with FRIENDLY string; SYNC", async () => { + // TODO: Fork mode does not work in the Windows-Server image used by github actions + if (process.platform === "win32") { + console.error( + "// TODO: Fork mode does not work in the Windows-Server image used by github actions", + ); + return; + } + const fuzzTest = new FuzzTestBuilder() + .sync(false) + .fuzzEntryPoint("ForkModeCallOriginalFriendly") + .dir(bugDetectorDirectory) + .runs(200) + .forkMode(3) + .build(); + fuzzTest.execute(); + expect(await fileExists(friendlyFilePath)).toBeTruthy(); + }); - it("Fork mode: Call with EVIL string; SYNC", () => { - // TODO: Fork mode does not work in the Windows-Server image used by github actions - if (process.platform === "win32") { - console.error( - "// TODO: Fork mode does not work in the Windows-Server image used by github actions", - ); - return; - } - const fuzzTest = new FuzzTestBuilder() - .sync(false) - .fuzzEntryPoint("ForkModeCallOriginalEvil") - .dir(bugDetectorDirectory) - .runs(200) - .forkMode(3) - .build(); - fuzzTest.execute(); // fork mode doesn't throw errors - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); - }); + it("Fork mode: Call with EVIL string; ASYNC", async () => { + // TODO: Fork mode does not work in the Windows-Server image used by github actions + if (process.platform === "win32") { + console.error( + "// TODO: Fork mode does not work in the Windows-Server image used by github actions", + ); + return; + } + const fuzzTest = new FuzzTestBuilder() + .sync(false) + .fuzzEntryPoint("ForkModeCallOriginalEvilAsync") + .dir(bugDetectorDirectory) + .runs(10) + .forkMode(3) + .build(); + fuzzTest.execute(); + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + }); - it("Fork mode: Call with FRIENDLY string; SYNC", () => { - // TODO: Fork mode does not work in the Windows-Server image used by github actions - if (process.platform === "win32") { - console.error( - "// TODO: Fork mode does not work in the Windows-Server image used by github actions", - ); - return; - } - const fuzzTest = new FuzzTestBuilder() - .sync(false) - .fuzzEntryPoint("ForkModeCallOriginalFriendly") - .dir(bugDetectorDirectory) - .runs(200) - .forkMode(3) - .build(); - fuzzTest.execute(); - expect(fs.existsSync(friendlyFilePath)).toBeTruthy(); - }); + it("Fork mode: Call with FRIENDLY string; ASYNC", async () => { + // TODO: Fork mode does not work in the Windows-Server image used by github actions + if (process.platform === "win32") { + console.error( + "// TODO: Fork mode does not work in the Windows-Server image used by github actions", + ); + return; + } + const fuzzTest = new FuzzTestBuilder() + .sync(false) + .fuzzEntryPoint("ForkModeCallOriginalFriendlyAsync") + .dir(bugDetectorDirectory) + .runs(200) + .forkMode(3) + .build(); + fuzzTest.execute(); // fork mode doesn't throw errors + expect(await fileExists(friendlyFilePath)).toBeTruthy(); + }); - it("Fork mode: Call with EVIL string; ASYNC", () => { - // TODO: Fork mode does not work in the Windows-Server image used by github actions - if (process.platform === "win32") { - console.error( - "// TODO: Fork mode does not work in the Windows-Server image used by github actions", - ); - return; - } - const fuzzTest = new FuzzTestBuilder() - .sync(false) - .fuzzEntryPoint("ForkModeCallOriginalEvilAsync") - .dir(bugDetectorDirectory) - .runs(10) - .forkMode(3) - .build(); - fuzzTest.execute(); - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); + it("Disable all bug detectors; Call with evil", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .fuzzEntryPoint("DisableAllBugDetectors") + .dir(bugDetectorDirectory) + .disableBugDetectors([".*"]) + .build(); + fuzzTest.execute(); + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + expect(await fileExists(evilFilePath)).toBeTruthy(); + expect(await fileExists("../jaz_zer")).toBeTruthy(); + }); }); - it("Fork mode: Call with FRIENDLY string; ASYNC", () => { - // TODO: Fork mode does not work in the Windows-Server image used by github actions - if (process.platform === "win32") { - console.error( - "// TODO: Fork mode does not work in the Windows-Server image used by github actions", - ); - return; - } - const fuzzTest = new FuzzTestBuilder() - .sync(false) - .fuzzEntryPoint("ForkModeCallOriginalFriendlyAsync") - .dir(bugDetectorDirectory) - .runs(200) - .forkMode(3) - .build(); - fuzzTest.execute(); // fork mode doesn't throw errors - expect(fs.existsSync(friendlyFilePath)).toBeTruthy(); - }); + describe("Jest", () => { + it("Test with EVIL command; SYNC", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .dir(bugDetectorDirectory) + .jestTestFile("tests.fuzz.js") + .jestTestName("^Command Injection Jest tests Call with EVIL command$") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + expectErrorToBePrintedOnce(fuzzTest); + }); - it("Disable all bug detectors; Call with evil", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .fuzzEntryPoint("DisableAllBugDetectors") - .dir(bugDetectorDirectory) - .disableBugDetectors([".*"]) - .build(); - fuzzTest.execute(); - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); - expect(fs.existsSync(evilFilePath)).toBeTruthy(); - expect(fs.existsSync("../jaz_zer")).toBeTruthy(); - }); + it("Test with EVIL command; ASYNC", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .dir(bugDetectorDirectory) + .jestTestFile("tests.fuzz.js") + .jestTestName( + "^Command Injection Jest tests Call with EVIL command ASYNC$", + ) + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + expectErrorToBePrintedOnce(fuzzTest); + }); - it("Jest: Test with EVIL command; SYNC", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .dir(bugDetectorDirectory) - .jestTestFile("tests.fuzz.js") - .jestTestName("^Command Injection Jest tests Call with EVIL command$") - .build(); - expect(() => { + it("Test with FRIENDLY command", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .dir(bugDetectorDirectory) + .jestTestFile("tests.fuzz.js") + .jestTestName( + "^Command Injection Jest tests Call with FRIENDLY command$", + ) + .build(); fuzzTest.execute(); - }).toThrow(JestRegressionExitCode); - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); - expectErrorToBePrintedOnce(fuzzTest); - }); + expect(await fileExists(friendlyFilePath)).toBeTruthy(); + }); - it("Jest: Test with EVIL command; ASYNC", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .dir(bugDetectorDirectory) - .jestTestFile("tests.fuzz.js") - .jestTestName( - "^Command Injection Jest tests Call with EVIL command ASYNC$", - ) - .build(); - expect(() => { + it("Test with FRIENDLY command; ASYNC", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .dir(bugDetectorDirectory) + .jestTestFile("tests.fuzz.js") + .jestTestName( + "^Command Injection Jest tests Call with FRIENDLY command ASYNC$", + ) + .build(); fuzzTest.execute(); - }).toThrow(JestRegressionExitCode); - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); - expectErrorToBePrintedOnce(fuzzTest); - }); - - it("Jest: Test with FRIENDLY command", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .dir(bugDetectorDirectory) - .jestTestFile("tests.fuzz.js") - .jestTestName("^Command Injection Jest tests Call with FRIENDLY command$") - .build(); - fuzzTest.execute(); - expect(fs.existsSync(friendlyFilePath)).toBeTruthy(); - }); + expect(await fileExists(friendlyFilePath)).toBeTruthy(); + }); - it("Jest: Test with FRIENDLY command; ASYNC", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .dir(bugDetectorDirectory) - .jestTestFile("tests.fuzz.js") - .jestTestName( - "^Command Injection Jest tests Call with FRIENDLY command ASYNC$", - ) - .build(); - fuzzTest.execute(); - expect(fs.existsSync(friendlyFilePath)).toBeTruthy(); - }); + it("Fuzzing mode; Test with EVIL command", async () => { + const fuzzTest = new FuzzTestBuilder() + .sync(false) + .dir(bugDetectorDirectory) + .jestTestFile("tests.fuzz.js") + .jestTestName( + "^Command Injection Jest tests Fuzzing mode with EVIL command$", + ) + .jestRunInFuzzingMode(true) + .runs(200) + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow( + process.platform === "win32" ? JestRegressionExitCode : FuzzingExitCode, + ); + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + expectErrorToBePrintedOnce(fuzzTest); + }); - it("Jest: Fuzzing mode; Test with EVIL command", () => { - const fuzzTest = new FuzzTestBuilder() - .sync(false) - .dir(bugDetectorDirectory) - .jestTestFile("tests.fuzz.js") - .jestTestName( - "^Command Injection Jest tests Fuzzing mode with EVIL command$", - ) - .jestRunInFuzzingMode(true) - .runs(200) - .build(); - expect(() => { + it("Fuzzing mode; Test with FRIENDLY command", async () => { + const fuzzTest = new FuzzTestBuilder() + .sync(false) + .dir(bugDetectorDirectory) + .jestTestFile("tests.fuzz.js") + .jestTestName( + "^Command Injection Jest tests Fuzzing mode with FRIENDLY command$", + ) + .jestRunInFuzzingMode(true) + .runs(200) + .build(); fuzzTest.execute(); - }).toThrow( - process.platform === "win32" ? JestRegressionExitCode : FuzzingExitCode, - ); - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); - expectErrorToBePrintedOnce(fuzzTest); - }); + expect(await fileExists(friendlyFilePath)).toBeTruthy(); + }); - it("Jest: Fuzzing mode; Test with FRIENDLY command", () => { - const fuzzTest = new FuzzTestBuilder() - .sync(false) - .dir(bugDetectorDirectory) - .jestTestFile("tests.fuzz.js") - .jestTestName( - "^Command Injection Jest tests Fuzzing mode with FRIENDLY command$", - ) - .jestRunInFuzzingMode(true) - .runs(200) - .build(); - fuzzTest.execute(); - expect(fs.existsSync(friendlyFilePath)).toBeTruthy(); - }); + it("Test with EVIL command; Done callback", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .dir(bugDetectorDirectory) + .jestTestFile("tests.fuzz.js") + .jestTestName( + "^Command Injection Jest tests Call with EVIL command and done callback$", + ) + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + expect(await fileExists(friendlyFilePath)).toBeFalsy(); + expectErrorToBePrintedOnce(fuzzTest); + }); - it("Jest: Test with EVIL command; Done callback", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .dir(bugDetectorDirectory) - .jestTestFile("tests.fuzz.js") - .jestTestName( - "^Command Injection Jest tests Call with EVIL command and done callback$", - ) - .build(); - expect(() => { + it("Test with FRIENDLY command; Done callback", async () => { + const fuzzTest = new FuzzTestBuilder() + .runs(0) + .sync(false) + .dir(bugDetectorDirectory) + .jestTestFile("tests.fuzz.js") + .jestTestName( + "^Command Injection Jest tests Call with FRIENDLY command and done callback$", + ) + .build(); fuzzTest.execute(); - }).toThrow(JestRegressionExitCode); - expect(fs.existsSync(friendlyFilePath)).toBeFalsy(); - expectErrorToBePrintedOnce(fuzzTest); - }); - - it("Jest: Test with FRIENDLY command; Done callback", () => { - const fuzzTest = new FuzzTestBuilder() - .runs(0) - .sync(false) - .dir(bugDetectorDirectory) - .jestTestFile("tests.fuzz.js") - .jestTestName( - "^Command Injection Jest tests Call with FRIENDLY command and done callback$", - ) - .build(); - fuzzTest.execute(); - expect(fs.existsSync(friendlyFilePath)).toBeTruthy(); + expect(await fileExists(friendlyFilePath)).toBeTruthy(); + }); }); }); diff --git a/tests/bug-detectors/package.json b/tests/bug-detectors/package.json index 1caf5061..81e01d57 100644 --- a/tests/bug-detectors/package.json +++ b/tests/bug-detectors/package.json @@ -8,5 +8,8 @@ }, "devDependencies": { "@jazzer.js/core": "file:../../packages/core" + }, + "jest": { + "testTimeout": 100000 } } diff --git a/tests/bug-detectors/path-traversal.test.js b/tests/bug-detectors/path-traversal.test.js index b4337025..513e0ae9 100644 --- a/tests/bug-detectors/path-traversal.test.js +++ b/tests/bug-detectors/path-traversal.test.js @@ -17,17 +17,18 @@ const { FuzzTestBuilder, FuzzingExitCode } = require("../helpers.js"); const path = require("path"); const fs = require("fs"); +const { cleanCrashFilesIn } = require("../helpers"); describe("Path Traversal", () => { const SAFE = "../safe_path/"; const EVIL = "../evil_path/"; + const bugDetectorDirectory = path.join(__dirname, "path-traversal"); - beforeEach(() => { + beforeEach(async () => { fs.rmSync(SAFE, { recursive: true, force: true }); + await cleanCrashFilesIn(bugDetectorDirectory); }); - const bugDetectorDirectory = path.join(__dirname, "path-traversal"); - it("openSync with EVIL path", () => { const fuzzTest = new FuzzTestBuilder() .runs(0) diff --git a/tests/helpers.js b/tests/helpers.js index 3667d0e3..7d61ffbe 100644 --- a/tests/helpers.js +++ b/tests/helpers.js @@ -48,6 +48,8 @@ class FuzzTest { verbose, coverage, expectedErrors, + asJson, + timeout, ) { this.includes = includes; this.excludes = excludes; @@ -68,6 +70,8 @@ class FuzzTest { this.verbose = verbose; this.coverage = coverage; this.expectedErrors = expectedErrors; + this.asJson = asJson; + this.timeout = timeout; } execute() { @@ -85,6 +89,7 @@ class FuzzTest { if (this.sync) options.push("--sync"); if (this.coverage) options.push("--coverage"); if (this.dryRun !== undefined) options.push("--dry_run=" + this.dryRun); + if (this.timeout !== undefined) options.push("--timeout=" + this.timeout); for (const include of this.includes) { options.push("-i=" + include); } @@ -148,6 +153,9 @@ class FuzzTest { if (this.jestRunInFuzzingMode !== undefined) { config.mode = this.jestRunInFuzzingMode ? "fuzzing" : "regression"; } + if (this.timeout !== undefined) { + config.timeout = this.timeout; + } // Write jest config file even if it exists fs.writeFileSync( @@ -157,9 +165,11 @@ class FuzzTest { const cmd = "npx"; const options = [ "jest", - this.coverage ? "--coverage" : "", this.jestTestFile, '--testNamePattern="' + this.jestTestNamePattern + '"', + "--no-colors", + this.asJson ? "--json" : "", + this.coverage ? "--coverage" : "", ]; this.runTest(cmd, options, { ...process.env }); } @@ -209,6 +219,8 @@ class FuzzTestBuilder { _dictionaries = []; _coverage = false; _expectedErrors = []; + _asJson = false; + _timeout = undefined; /** * @param {boolean} sync - whether to run the fuzz test in synchronous mode. @@ -375,6 +387,16 @@ class FuzzTestBuilder { return this; } + asJson(asJson) { + this._asJson = asJson === undefined ? true : asJson; + return this; + } + + timeout(timeout) { + this._timeout = timeout; + return this; + } + build() { if (this._jestTestFile === "" && this._fuzzEntryPoint === "") { throw new Error("fuzzEntryPoint or jestTestFile are not set."); @@ -404,6 +426,8 @@ class FuzzTestBuilder { this._verbose, this._coverage, this._expectedErrors, + this._asJson, + this._timeout, ); } } @@ -457,6 +481,31 @@ function describeSkipOnPlatform(platform) { return process.platform === platform ? global.describe.skip : global.describe; } +async function getFiles(dir) { + const result = []; + const files = await fs.promises.readdir(dir); + for (const file of files) { + const filepath = path.join(dir, file); + result.push(filepath); + if ((await fs.promises.stat(filepath)).isDirectory()) { + result.push(...(await getFiles(filepath))); + } + } + return result; +} + +async function fileExists(path) { + return !!(await fs.promises.stat(path).catch((e) => false)); +} + +async function cleanCrashFilesIn(path) { + for (const file in await getFiles(path)) { + if (file.match(/crash-[0-9a-f]{40}/)) { + await fs.promises.rm(file, { force: true }); + } + } +} + module.exports = { FuzzTestBuilder, FuzzingExitCode, @@ -466,4 +515,7 @@ module.exports = { makeFnCalledOnce, callWithTimeout, describeSkipOnPlatform, + getFiles, + fileExists, + cleanCrashFilesIn, }; diff --git a/tests/jest_integration/integration.test.js b/tests/jest_integration/integration.test.js index cdcc54a9..380d163a 100644 --- a/tests/jest_integration/integration.test.js +++ b/tests/jest_integration/integration.test.js @@ -105,7 +105,6 @@ describe("Jest integration", () => { const fuzzTest = fuzzTestBuilder .jestTestName("execute sync test") .runs(1) - .verbose() .build(); try { fuzzTest.execute(); @@ -174,12 +173,33 @@ describe("Jest integration", () => { it("load by mapped module name", () => { const fuzzTest = fuzzTestBuilder .jestTestName("load by mapped module name") - .verbose() .build(); expect(() => { fuzzTest.execute(); }).toThrow(fuzzingExitCode); }); + + it("print proper stacktrace", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute sync test") + .asJson() + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(fuzzingExitCode); + const result = JSON.parse(fuzzTest.stdout); + expect(result.numFailedTests).toBe(1); + + const lines = firstFailureMessage(result).split("\n"); + expect(lines).toHaveLength(3); + expect(lines[0]).toEqual("Error: Welcome to Awesome Fuzzing!"); + expect(lines[1]).toMatch( + /at Object\.Error \[as fuzzMe] \(.*target\.js:\d+:\d+\)/, + ); + expect(lines[2]).toMatch( + /at fuzzMe \(.*integration\.fuzz\.js:\d+:\d+\)/, + ); + }); }); }); @@ -196,9 +216,9 @@ describe("Jest integration", () => { .execute(); }); - it("execute async test", () => { + it("execute async test plain", () => { regressionTestBuilder - .jestTestName("execute async test") + .jestTestName("execute async test plain") .build() .execute(); }); @@ -221,7 +241,8 @@ describe("Jest integration", () => { describe("timeout", () => { it("execute async timeout test", () => { const fuzzTest = regressionTestBuilder - .jestTestName("execute async timeout test") + .jestTestName("execute async timeout test plain") + .asJson() .build(); expect(() => { fuzzTest.execute(); @@ -280,6 +301,22 @@ describe("Jest integration", () => { .build() .execute(); }); + + it("print proper stacktrace", () => { + const fuzzTest = regressionTestBuilder + .jestTestName("execute async timeout test plain") + .asJson() + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + + const result = JSON.parse(fuzzTest.stdout); + const stackFrames = firstFailureMessage(result) + .split("\n") + .filter((line) => line.startsWith(" at")); + expect(stackFrames).toHaveLength(10); + }); }); describe("Run modes", () => { @@ -315,3 +352,9 @@ function assertTimeoutMessageLogged(fuzzTest, expectedTimeout) { expect(timeoutValue).toBeGreaterThanOrEqual(expectedTimeout - 1); expect(timeoutValue).toBeLessThanOrEqual(expectedTimeout + 1); } + +function firstFailureMessage(result) { + return result.testResults[0].assertionResults.filter( + (result) => result.status === "failed", + )[0].failureMessages[0]; +} diff --git a/tests/jest_integration/jest_project/integration.fuzz.js b/tests/jest_integration/jest_project/integration.fuzz.js index 8f339d71..65d0380d 100644 --- a/tests/jest_integration/jest_project/integration.fuzz.js +++ b/tests/jest_integration/jest_project/integration.fuzz.js @@ -41,7 +41,7 @@ describe("Jest Integration", () => { target.callbackFuzzMe(data, done); }); - it.fuzz("execute async timeout test", async (data) => { + it.fuzz("execute async timeout test plain", async (data) => { await target.asyncTimeout(data); }); From 779d47b4d7fd544755afd152560dc5a83e2047f6 Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Thu, 14 Sep 2023 12:49:21 +0200 Subject: [PATCH 13/18] jest: Add Jest ESM example --- docs/fuzz-targets.md | 20 +++++---- examples/protobufjs/fuzz.js | 52 ------------------------ examples/protobufjs/package.json | 24 +++++++++-- examples/protobufjs/protobufjs.fuzz.js | 56 ++++++++++++++++++++++++++ examples/protobufjs/readme.md | 5 +++ 5 files changed, 94 insertions(+), 63 deletions(-) delete mode 100644 examples/protobufjs/fuzz.js create mode 100644 examples/protobufjs/protobufjs.fuzz.js create mode 100644 examples/protobufjs/readme.md diff --git a/docs/fuzz-targets.md b/docs/fuzz-targets.md index 817d1329..50836c62 100644 --- a/docs/fuzz-targets.md +++ b/docs/fuzz-targets.md @@ -115,20 +115,24 @@ loaded properly. If your project internally still relies on calls to `require()`, all of these dependencies will be hooked. However, _pure_ ECMAScript projects will currently not be instrumented! +The Jest integration can improve on this and use Jest's ESM features to properly +transform external code and dependencies. However, +[ESM support](https://jestjs.io/docs/ecmascript-modules) in Jest is also only +experimental. + One such example that Jazzer.js can handle just fine can be found at -[examples/protobufjs/fuzz.js](../examples/protobufjs/fuzz.js): +[examples/protobufjs/fuzz.js](../examples/protobufjs/protobufjs.fuzz.js): ```js import proto from "protobufjs"; import { temporaryWriteSync } from "tempy"; -export function fuzz(data: Buffer) { - try { - // Fuzz logic - } catch (e) { - // Handle expected error logic here - } -} +describe("protobufjs", () => { + test.fuzz("loadSync", (data) => { + const file = temporaryWriteSync(data); + proto.loadSync(file); + }); +}); ``` You also have to adapt your `package.json` accordingly, by adding: diff --git a/examples/protobufjs/fuzz.js b/examples/protobufjs/fuzz.js deleted file mode 100644 index 889276d1..00000000 --- a/examples/protobufjs/fuzz.js +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2023 Code Intelligence GmbH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import proto from "protobufjs"; -import { temporaryWriteSync } from "tempy"; - -/** - * @param { Buffer } data - */ -export function fuzz(data) { - try { - const file = temporaryWriteSync(data); - const root = proto.loadSync(file); - if (root.toString().length >= 30) { - console.error("== Input: " + data.toString() + "\n== " + root.toString()); - } - } catch (e) { - if ( - e.name !== "SyntaxError" && - e.message && - !e.message.includes("illegal token") && - !e.message.includes("illegal string") && - !e.message.includes("illegal path") && - !e.message.includes("illegal comment") && - !e.message.includes("illegal reference") && - !e.message.includes("illegal name") && - !e.message.includes("illegal type") && - !e.message.includes("illegal value") && - !e.message.includes("illegal service") && - !e.message.includes("name must be a string") && - !e.message.includes("path must be relative") && - !e.message.includes("duplicate name") && - !e.message.includes("Unexpected token") && - !e.message.includes("Unexpected end") - ) { - throw e; - } - } -} diff --git a/examples/protobufjs/package.json b/examples/protobufjs/package.json index f610c61d..8198e45e 100644 --- a/examples/protobufjs/package.json +++ b/examples/protobufjs/package.json @@ -3,14 +3,32 @@ "version": "0.0.0", "type": "module", "scripts": { - "fuzz": "npx jazzer fuzz --sync -i protobuf", - "dryRun": "npx jazzer fuzz -m regression --sync -i protobuf -- -runs=100 -seed=123456789" + "fuzz_esm": "JAZZER_FUZZ=1 NODE_OPTIONS=--experimental-vm-modules npx jest", + "dryRun": "echo \"Skipped\"", + "dryRun_esm": "NODE_OPTIONS=--experimental-vm-modules npx jest" }, "dependencies": { "protobufjs": "^7.0.0", "tempy": "^3.0.0" }, "devDependencies": { - "@jazzer.js/cli": "file:../../packages/core" + "@jazzer.js/jest-runner": "file:../../packages/jest-runner" + }, + "engines": { + "node": ">= 18.8.0" + }, + "jest": { + "projects": [ + { + "testRunner": "@jazzer.js/jest-runner", + "displayName": { + "name": "Jazzer.js", + "color": "cyan" + }, + "testMatch": [ + "/**/*.fuzz.js" + ] + } + ] } } diff --git a/examples/protobufjs/protobufjs.fuzz.js b/examples/protobufjs/protobufjs.fuzz.js new file mode 100644 index 00000000..b104ddaf --- /dev/null +++ b/examples/protobufjs/protobufjs.fuzz.js @@ -0,0 +1,56 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import proto from "protobufjs"; +import { temporaryWriteSync } from "tempy"; +import fs from "fs"; + +describe("protobufjs", () => { + test.fuzz("loadSync", (data) => { + const file = temporaryWriteSync(data); + try { + const root = proto.loadSync(file); + if (root.toString().length >= 30) { + console.error( + "== Input: " + data.toString() + "\n== " + root.toString(), + ); + } + } catch (e) { + if ( + e.name !== "SyntaxError" && + e.message && + !e.message.includes("illegal token") && + !e.message.includes("illegal string") && + !e.message.includes("illegal path") && + !e.message.includes("illegal comment") && + !e.message.includes("illegal reference") && + !e.message.includes("illegal name") && + !e.message.includes("illegal type") && + !e.message.includes("illegal value") && + !e.message.includes("illegal service") && + !e.message.includes("name must be a string") && + !e.message.includes("path must be relative") && + !e.message.includes("duplicate name") && + !e.message.includes("Unexpected token") && + !e.message.includes("Unexpected end") + ) { + throw e; + } + } finally { + fs.rmSync(file); + } + }); +}); diff --git a/examples/protobufjs/readme.md b/examples/protobufjs/readme.md new file mode 100644 index 00000000..4024e834 --- /dev/null +++ b/examples/protobufjs/readme.md @@ -0,0 +1,5 @@ +# protobufjs fuzz test + +This fuzz test uses +[ECMAScript Modules](https://jestjs.io/docs/ecmascript-modules) and requires +Node.js 18.8.0+. From 2eb737af65680b2c01807b00e4caae57be5f47ef Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Wed, 6 Sep 2023 14:59:59 +0200 Subject: [PATCH 14/18] docs: Add migration guide --- docs/README.md | 1 + docs/jest-integration.md | 26 +++++--------------------- docs/migration-guide.md | 30 ++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 21 deletions(-) create mode 100644 docs/migration-guide.md diff --git a/docs/README.md b/docs/README.md index 4ec14730..ec377b73 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,6 +5,7 @@ - [Using fuzz targets](fuzz-targets.md) - [Using Jest integration](jest-integration.md) - [Advanced Fuzzing Settings](fuzz-settings.md) +- [Migration Guide](migration-guide.md) ## Internal Documentation diff --git a/docs/jest-integration.md b/docs/jest-integration.md index 0e362e34..c480f31e 100644 --- a/docs/jest-integration.md +++ b/docs/jest-integration.md @@ -34,6 +34,8 @@ npm install --save-dev @jazzer.js/jest-runner This will install the custom Jest runner along with all other required Jazzer.js dependencies. +**Note**: The Jazzer.js Jest runner requires Jest version 29 or higher. + For Jest to pick up the custom fuzz test runner, it has to be added to the Jest configuration in the `package.json` or `jest.config.js`, as described on the [Configuring Jest](https://jestjs.io/docs/configuration) documentation page. The @@ -60,12 +62,12 @@ runner. "displayName": "test" }, { - "runner": "@jazzer.js/jest-runner", "displayName": { "name": "Jazzer.js", "color": "cyan" }, - "testMatch": ["/**/*.fuzz.js"] + "testMatch": ["/**/*.fuzz.js"], + "testRunner": "@jazzer.js/jest-runner" } ] } @@ -117,9 +119,9 @@ actually include test files with the `.fuzz.ts` extension. color: "cyan", }, preset: "ts-jest", - runner: "@jazzer.js/jest-runner", testEnvironment: "node", testMatch: ["/*.fuzz.[jt]s"], + testRunner: "@jazzer.js/jest-runner", }, ``` @@ -235,14 +237,6 @@ describe("Target", () => { }); ``` -### Setup and teardown - -The Jazzer.js fuzz test runner supports Jest's setup and teardown functions, as -described on the [Setup and Teardown](https://jestjs.io/docs/setup-teardown) -documentation page. - -This includes `beforeAll`, `afterAll`, `beforeEach` and `afterEach`. - ## Executing Jest fuzz tests As mentioned above, the Jazzer.js fuzz test runner provides two modes of @@ -402,13 +396,3 @@ enter the debugger for that particular input. ### IntelliJ Jest support ![IntelliJ Jest integration](pictures/jest-integration-intellij.png) - -## Unsupported Jest features - -The Jazzer.js fuzz test runner strives to provide neat Jest integration. That -being said, some Jest features are currently not supported, as Jest does not -offer good extension points and common test framework features have to be -reimplemented. - -- Mock functions -- Isolated workers diff --git a/docs/migration-guide.md b/docs/migration-guide.md new file mode 100644 index 00000000..e4b12e94 --- /dev/null +++ b/docs/migration-guide.md @@ -0,0 +1,30 @@ +# Migration Guide + +This document describes breaking changes in major versions and steps to migrate +from one to the next. + +## 2.0.0 + +This version fundamentally changed the Jest integration in module +`@jazzer.js/jest-runner`. The new approach provides a tighter integration with +Jest and allows fuzz tests to use all available Jest features. Most notably this +includes the widely missed mocking functionality. + +### Migration steps + +- In the Jest configuration, move `@jazzer.js/jest-runner` from `runner` to + `testRunner`. A valid configuration looks like this: + +```diff +{ + displayName: { + name: "Jazzer.js", + color: "cyan", + }, + preset: "ts-jest", +- runner: "@jazzer.js/jest-runner", ++ testRunner: "@jazzer.js/jest-runner", + testEnvironment: "node", + testMatch: ["/*.fuzz.[jt]s"], +} +``` From 39ede0156ad05ee96125b52bfc98d54d9b04291e Mon Sep 17 00:00:00 2001 From: Peter Samarin Date: Fri, 15 Sep 2023 12:05:10 +0200 Subject: [PATCH 15/18] jest: reportFinding can be called from Jest tests --- .../command-injection/custom-hooks.js | 7 ++- .../internal/command-injection.ts | 7 ++- .../bug-detectors/internal/path-traversal.ts | 7 ++- .../internal/prototype-pollution.ts | 5 +- packages/core/api.ts | 2 +- packages/core/core.ts | 19 +++---- packages/core/finding.ts | 49 ++++++++++++++----- packages/core/globals.ts | 11 +---- packages/jest-runner/index.ts | 12 +++-- .../prototype-pollution/tests.fuzz.js | 1 + tests/jest_integration/integration.test.js | 13 +++++ .../jest_project/integration.fuzz.js | 11 +++++ 12 files changed, 102 insertions(+), 42 deletions(-) diff --git a/examples/bug-detectors/command-injection/custom-hooks.js b/examples/bug-detectors/command-injection/custom-hooks.js index a777589c..ca616255 100644 --- a/examples/bug-detectors/command-injection/custom-hooks.js +++ b/examples/bug-detectors/command-injection/custom-hooks.js @@ -15,7 +15,10 @@ */ const { registerReplaceHook } = require("@jazzer.js/hooking"); -const { guideTowardsEquality, reportFinding } = require("@jazzer.js/core"); +const { + guideTowardsEquality, + reportAndThrowFinding, +} = require("@jazzer.js/core"); /** * Custom bug detector for command injection. This hook does not call the original function (execSync) for two reasons: @@ -32,7 +35,7 @@ registerReplaceHook( } const command = params[0]; if (command.includes("jaz_zer")) { - reportFinding( + reportAndThrowFinding( `Command Injection in spawnSync(): called with '${command}'`, ); } diff --git a/packages/bug-detectors/internal/command-injection.ts b/packages/bug-detectors/internal/command-injection.ts index 41a18f56..be2f1bb4 100644 --- a/packages/bug-detectors/internal/command-injection.ts +++ b/packages/bug-detectors/internal/command-injection.ts @@ -14,7 +14,10 @@ * limitations under the License. */ -import { guideTowardsContainment, reportFinding } from "@jazzer.js/core"; +import { + guideTowardsContainment, + reportAndThrowFinding, +} from "@jazzer.js/core"; import { registerBeforeHook } from "@jazzer.js/hooking"; /** @@ -49,7 +52,7 @@ for (const functionName of functionNames) { return; } if (firstArgument.includes(goal)) { - reportFinding( + reportAndThrowFinding( `Command Injection in ${functionName}(): called with '${firstArgument}'`, ); } diff --git a/packages/bug-detectors/internal/path-traversal.ts b/packages/bug-detectors/internal/path-traversal.ts index 81938ea6..fdcaf459 100644 --- a/packages/bug-detectors/internal/path-traversal.ts +++ b/packages/bug-detectors/internal/path-traversal.ts @@ -14,7 +14,10 @@ * limitations under the License. */ -import { reportFinding, guideTowardsContainment } from "@jazzer.js/core"; +import { + reportAndThrowFinding, + guideTowardsContainment, +} from "@jazzer.js/core"; import { callSiteId, registerBeforeHook } from "@jazzer.js/hooking"; /** @@ -202,7 +205,7 @@ function detectFindingAndGuideFuzzing( ) { const argument = input.toString(); if (argument.includes(goal)) { - reportFinding( + reportAndThrowFinding( `Path Traversal in ${functionName}(): called with '${argument}'`, ); } diff --git a/packages/bug-detectors/internal/prototype-pollution.ts b/packages/bug-detectors/internal/prototype-pollution.ts index 187ce961..e8b029c2 100644 --- a/packages/bug-detectors/internal/prototype-pollution.ts +++ b/packages/bug-detectors/internal/prototype-pollution.ts @@ -17,6 +17,7 @@ import { AssignmentExpression, Identifier, Node } from "@babel/types"; import { NodePath, PluginTarget, types } from "@babel/core"; import { + reportAndThrowFinding, reportFinding, registerAfterEachCallback, addDictionary, @@ -334,6 +335,7 @@ function detectPrototypePollutionOfBasicObjects( if (!currentProtoSnapshots[i]) { reportFinding( `Prototype Pollution: Prototype of ${BASIC_OBJECT_NAMES[i]} changed.`, + false, ); return; } @@ -344,6 +346,7 @@ function detectPrototypePollutionOfBasicObjects( if (equalityResult) { reportFinding( `Prototype Pollution: Prototype of ${BASIC_OBJECT_NAMES[i]} changed. ${equalityResult}`, + false, ); return; } @@ -401,7 +404,7 @@ function detectPrototypePollution( message = `Prototype Pollution: __proto__ value is ${protoValue}`; } if (report) { - reportFinding(message); + reportAndThrowFinding(message); } // If prototype pollution is detected, always stop analyzing the prototype chain. return; diff --git a/packages/core/api.ts b/packages/core/api.ts index 3b28fde6..fa0e1b3d 100644 --- a/packages/core/api.ts +++ b/packages/core/api.ts @@ -28,7 +28,7 @@ export { registerBeforeEachCallback, } from "./callback"; export { addDictionary } from "./dictionary"; -export { reportFinding } from "./finding"; +export { reportAndThrowFinding, reportFinding } from "./finding"; export { getJazzerJsGlobal, setJazzerJsGlobal } from "./globals"; export const guideTowardsEquality = fuzzer.tracer.guideTowardsEquality; diff --git a/packages/core/core.ts b/packages/core/core.ts index b1829b71..24d7c728 100644 --- a/packages/core/core.ts +++ b/packages/core/core.ts @@ -25,7 +25,6 @@ import * as fuzzer from "@jazzer.js/fuzzer"; import * as hooking from "@jazzer.js/hooking"; import { clearFirstFinding, - getFirstFinding, printFinding, Finding, cleanErrorStack, @@ -39,6 +38,7 @@ import { import { callbacks } from "./callback"; import { ensureFilepath, importModule } from "./utils"; import { buildFuzzerOption, Options } from "./options"; +import { jazzerJs } from "./globals"; // Remove temporary files on exit tmp.setGracefulCleanup(); @@ -58,12 +58,7 @@ declare global { var options: Options; } -export async function initFuzzing( - options: Options, - globals?: unknown[], -): Promise { - registerGlobals(options, globals); - +export async function initFuzzing(options: Options): Promise { const instrumentor = new Instrumentor( options.includes, options.excludes, @@ -103,12 +98,16 @@ export async function initFuzzing( return instrumentor; } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function registerGlobals(options: Options, globals: any[] = [globalThis]) { +export function registerGlobals( + options: Options, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + globals: any[] = [globalThis], +) { globals.forEach((global) => { global.Fuzzer = fuzzer.fuzzer; global.HookManager = hooking.hookManager; global.options = options; + global.JazzerJS = jazzerJs; }); } @@ -154,6 +153,7 @@ function getFilteredBugDetectorPaths( } export async function startFuzzing(options: Options) { + registerGlobals(options); await initFuzzing(options); const fuzzFn = await loadFuzzFunction(options); @@ -355,6 +355,7 @@ export function wrapFuzzFunctionForBugDetection( callbacks.runAfterEachCallbacks(); } } catch (e) { + callbacks.runAfterEachCallbacks(); fuzzTargetError = e; } return throwIfError(fuzzTargetError) ?? result; diff --git a/packages/core/finding.ts b/packages/core/finding.ts index 9cb31cd5..96d1d7ee 100644 --- a/packages/core/finding.ts +++ b/packages/core/finding.ts @@ -17,39 +17,62 @@ import process from "process"; import { EOL } from "os"; import { sep } from "path"; +import { getJazzerJsGlobal, setJazzerJsGlobal } from "./api"; + +const firstFinding = "firstFinding"; export class Finding extends Error {} -// The first finding reported by any bug detector will be saved here. +// The first finding reported by any bug detector will be saved in the global jazzerJs object. // This variable has to be cleared every time when the fuzzer is finished // processing an input (only relevant for modes where the fuzzing continues // after finding an error, e.g. fork mode, Jest regression mode, fuzzing that // ignores errors mode, etc.). -let firstFinding: Finding | undefined; -export function getFirstFinding(): Finding | undefined { - return firstFinding; +function getFirstFinding(): Finding | undefined { + return getJazzerJsGlobal(firstFinding); } export function clearFirstFinding(): Finding | undefined { - const lastFinding = firstFinding; - firstFinding = undefined; + const lastFinding = getFirstFinding(); + setJazzerJsGlobal(firstFinding, undefined); return lastFinding; } /** - * Save the first finding reported by any bug detector and throw it to - * potentially abort the current execution. + * Save the first finding reported by any bug detector. * - * @param findingMessage - The finding to be saved and thrown. + * @param findingMessage - The finding to be saved. + * @param containStack - Whether the finding should contain a stack trace or not. */ -export function reportFinding(findingMessage: string): void | never { +export function reportFinding( + findingMessage: string, + containStack = true, +): Finding | undefined { // After saving the first finding, ignore all subsequent errors. - if (firstFinding) { + if (getFirstFinding()) { return; } - firstFinding = new Finding(findingMessage); - throw firstFinding; + const reportedFinding = new Finding(findingMessage); + if (!containStack) { + reportedFinding.stack = findingMessage; + } + setJazzerJsGlobal(firstFinding, reportedFinding); + return reportedFinding; +} + +/** + * Save the first finding reported by any bug detector and throw it to + * potentially abort the current execution. + * + * @param findingMessage - The finding to be saved and thrown. + * @param containStack - Whether the finding should contain a stack trace or not. + */ +export function reportAndThrowFinding( + findingMessage: string, + containStack = true, +): void | never { + throw reportFinding(findingMessage, containStack); } /** diff --git a/packages/core/globals.ts b/packages/core/globals.ts index 97f4db1f..f21bce3b 100644 --- a/packages/core/globals.ts +++ b/packages/core/globals.ts @@ -14,16 +14,9 @@ * limitations under the License. */ +export const jazzerJs = new Map(); + export function setJazzerJsGlobal(name: string, value: unknown) { - // @ts-ignore - if (globalThis.JazzerJS === undefined) { - Object.defineProperty(globalThis, "JazzerJS", { - value: new Map(), - enumerable: true, - configurable: false, - writable: false, - }); - } // @ts-ignore globalThis.JazzerJS.set(name, value); } diff --git a/packages/jest-runner/index.ts b/packages/jest-runner/index.ts index a0fbd0a2..b4ba5b1c 100644 --- a/packages/jest-runner/index.ts +++ b/packages/jest-runner/index.ts @@ -20,7 +20,11 @@ import { TestResult } from "@jest/test-result"; import { Config } from "@jest/types"; import type { JestEnvironment } from "@jest/environment"; -import { initFuzzing, setJazzerJsGlobal } from "@jazzer.js/core"; +import { + initFuzzing, + registerGlobals, + setJazzerJsGlobal, +} from "@jazzer.js/core"; import { loadConfig } from "./config"; import { FuzzTest } from "./fuzz"; @@ -39,14 +43,16 @@ export default async function jazzerTestRunner( ): Promise { const vmContext = environment.getVmContext(); if (vmContext === null) throw new Error("vmContext is undefined"); - setJazzerJsGlobal("vmContext", vmContext); const jazzerConfig = loadConfig({ coverage: globalConfig.collectCoverage, coverageReporters: globalConfig.coverageReporters as reports.ReportType[], }); const globalEnvironments = [environment.getVmContext(), globalThis]; - const instrumentor = await initFuzzing(jazzerConfig, globalEnvironments); + registerGlobals(jazzerConfig, globalEnvironments); + setJazzerJsGlobal("vmContext", vmContext); + const instrumentor = await initFuzzing(jazzerConfig); + interceptScriptTransformerCalls(runtime, instrumentor); const testState = interceptTestState(environment, jazzerConfig); diff --git a/tests/bug-detectors/prototype-pollution/tests.fuzz.js b/tests/bug-detectors/prototype-pollution/tests.fuzz.js index 545392f8..679154c0 100644 --- a/tests/bug-detectors/prototype-pollution/tests.fuzz.js +++ b/tests/bug-detectors/prototype-pollution/tests.fuzz.js @@ -18,6 +18,7 @@ describe("Prototype Pollution Jest tests", () => { it.fuzz("Pollution of Object", (data) => { const a = {}; a.__proto__.a = 10; + throw new Error("err"); }); it.fuzz("Assignments", (data) => { diff --git a/tests/jest_integration/integration.test.js b/tests/jest_integration/integration.test.js index 380d163a..29a5f62c 100644 --- a/tests/jest_integration/integration.test.js +++ b/tests/jest_integration/integration.test.js @@ -317,6 +317,19 @@ describe("Jest integration", () => { .filter((line) => line.startsWith(" at")); expect(stackFrames).toHaveLength(10); }); + + it("prioritize finding over error", () => { + const fuzzTest = regressionTestBuilder + .jestTestName("prioritize finding over error") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + expect(fuzzTest.stderr).toContain("Finding reported!"); + expect(fuzzTest.stderr).not.toContain( + "This error should not be reported!", + ); + }); }); describe("Run modes", () => { diff --git a/tests/jest_integration/jest_project/integration.fuzz.js b/tests/jest_integration/jest_project/integration.fuzz.js index 65d0380d..0739751b 100644 --- a/tests/jest_integration/jest_project/integration.fuzz.js +++ b/tests/jest_integration/jest_project/integration.fuzz.js @@ -16,6 +16,7 @@ const target = require("./target.js"); const mappedTarget = require("mappedModuleName"); +const { reportFinding } = require("@jazzer.js/core"); jest.mock("./target.js", () => ({ ...jest.requireActual("./target.js"), @@ -24,6 +25,11 @@ jest.mock("./target.js", () => ({ }, })); +// These should not be seen in the stack trace, because the test +// explicitly looks for these strings. +const findingMessage = "Finding reported!"; +const errorMessage = "This error should not be reported!"; + describe("Jest Integration", () => { it.fuzz("execute sync test", (data) => { target.fuzzMe(data); @@ -78,6 +84,11 @@ describe("Jest Integration", () => { it.fuzz("load by mapped module name", (data) => { mappedTarget.fuzzMe(data); }); + + it.fuzz("prioritize finding over error", (data) => { + reportFinding(findingMessage); + throw new Error(errorMessage); + }); }); describe("Run mode", () => { From 6130db1ea5200c1cb030a9263af546c8420482e7 Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Mon, 18 Sep 2023 09:28:55 +0200 Subject: [PATCH 16/18] core: Extend global Jazzer.js object API --- packages/core/api.ts | 6 ++++- packages/core/globals.test.ts | 43 +++++++++++++++++++++++++++++++++++ packages/core/globals.ts | 27 ++++++++++++++++++---- 3 files changed, 70 insertions(+), 6 deletions(-) create mode 100644 packages/core/globals.test.ts diff --git a/packages/core/api.ts b/packages/core/api.ts index fa0e1b3d..aa1fb96e 100644 --- a/packages/core/api.ts +++ b/packages/core/api.ts @@ -29,7 +29,11 @@ export { } from "./callback"; export { addDictionary } from "./dictionary"; export { reportAndThrowFinding, reportFinding } from "./finding"; -export { getJazzerJsGlobal, setJazzerJsGlobal } from "./globals"; +export { + getJazzerJsGlobal, + setJazzerJsGlobal, + getOrSetJazzerJsGlobal, +} from "./globals"; export const guideTowardsEquality = fuzzer.tracer.guideTowardsEquality; export const guideTowardsContainment = fuzzer.tracer.guideTowardsContainment; diff --git a/packages/core/globals.test.ts b/packages/core/globals.test.ts new file mode 100644 index 00000000..7d53d923 --- /dev/null +++ b/packages/core/globals.test.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + getJazzerJsGlobal, + getOrSetJazzerJsGlobal, + setJazzerJsGlobal, +} from "./globals"; + +describe("globals", () => { + beforeEach(() => { + globalThis.JazzerJS = new Map(); + }); + + it("should set and get a global", () => { + setJazzerJsGlobal("test", 1); + expect(getJazzerJsGlobal("test")).toBe(1); + }); + + it("should throw if not initialized", () => { + globalThis.JazzerJS = undefined; + expect(() => setJazzerJsGlobal("test", "foo")).toThrow(); + }); + + it("should set default value if not already defined", () => { + expect(getJazzerJsGlobal("test")).toBeUndefined(); + expect(getOrSetJazzerJsGlobal("test", "foo")).toBe("foo"); + expect(getOrSetJazzerJsGlobal("test", "baz")).toBe("foo"); + }); +}); diff --git a/packages/core/globals.ts b/packages/core/globals.ts index f21bce3b..edd2ee2a 100644 --- a/packages/core/globals.ts +++ b/packages/core/globals.ts @@ -14,14 +14,31 @@ * limitations under the License. */ +declare global { + // eslint-disable-next-line no-var + var JazzerJS: Map | undefined; +} + +// Require the external initialization to set this map in the globalThis object +// before it is used here. export const jazzerJs = new Map(); -export function setJazzerJsGlobal(name: string, value: unknown) { - // @ts-ignore +export function setJazzerJsGlobal(name: string, value: T): void { + if (!globalThis.JazzerJS) { + throw new Error("JazzerJS global not initialized"); + } globalThis.JazzerJS.set(name, value); } -export function getJazzerJsGlobal(name: string): unknown { - // @ts-ignore - return globalThis.JazzerJS?.get(name); +export function getJazzerJsGlobal(name: string): T | undefined { + return globalThis.JazzerJS?.get(name) as T; +} + +export function getOrSetJazzerJsGlobal(name: string, defaultValue: T): T { + const value = getJazzerJsGlobal(name); + if (value === undefined) { + setJazzerJsGlobal(name, defaultValue); + return defaultValue; + } + return value; } From 3b54af34b6750473e60f721efc4c283dc5eb2315 Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Mon, 18 Sep 2023 09:43:08 +0200 Subject: [PATCH 17/18] core: Use global Jazzer.js object for dictionary --- packages/core/dictionary.test.ts | 7 +++++-- packages/core/dictionary.ts | 21 ++++++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/core/dictionary.test.ts b/packages/core/dictionary.test.ts index 509e8c42..3f6644b7 100644 --- a/packages/core/dictionary.test.ts +++ b/packages/core/dictionary.test.ts @@ -14,14 +14,17 @@ * limitations under the License. */ import fs from "fs"; +import tmp from "tmp"; import { addDictionary, useDictionaryByParams } from "./dictionary"; -const tmp = require("tmp"); - // Cleanup created files on exit tmp.setGracefulCleanup(); describe("Dictionary", () => { + beforeEach(() => { + globalThis.JazzerJS = new Map(); + }); + it("use explicit dictionary", () => { const content = ` # comment diff --git a/packages/core/dictionary.ts b/packages/core/dictionary.ts index 0388e703..094fcb26 100644 --- a/packages/core/dictionary.ts +++ b/packages/core/dictionary.ts @@ -16,32 +16,35 @@ import fs from "fs"; import tmp from "tmp"; +import { getOrSetJazzerJsGlobal } from "./api"; /** * Dictionaries can be used to provide additional mutation suggestions to the * fuzzer. */ -export class Dictionaries { - private _dictionary: string[] = []; +export class Dictionary { + private _entries: string[] = []; - get dictionary() { - return this._dictionary; + get entries() { + return [...this._entries]; } - addDictionary(dictionary: string[]) { - this._dictionary.push(...dictionary); + addEntries(dictionary: string[]) { + this._entries.push(...dictionary); } } -const dictionaries = new Dictionaries(); +function getDictionary(): Dictionary { + return getOrSetJazzerJsGlobal("dictionary", new Dictionary()); +} export function addDictionary(...dictionary: string[]) { - dictionaries.addDictionary(dictionary); + getDictionary().addEntries(dictionary); } export function useDictionaryByParams(options: string[]): string[] { const opts = [...options]; - const dictionary = Array.from(dictionaries.dictionary); + const dictionary = getDictionary().entries; // This diverges from the libFuzzer behavior, which allows only one dictionary (the last one). // We merge all dictionaries into one and pass that to libfuzzer. From 87483a274aed077553266f9987e535bc6a6f96e6 Mon Sep 17 00:00:00 2001 From: Norbert Schneider Date: Mon, 18 Sep 2023 10:50:12 +0200 Subject: [PATCH 18/18] core: Use global Jazzer.js object for callbacks --- packages/core/callback.ts | 19 ++++++------- packages/core/callbacks.test.ts | 47 +++++++++++++++++++++++++++++++++ packages/core/core.ts | 4 ++- 3 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 packages/core/callbacks.test.ts diff --git a/packages/core/callback.ts b/packages/core/callback.ts index 0179e865..e8763118 100644 --- a/packages/core/callback.ts +++ b/packages/core/callback.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { getOrSetJazzerJsGlobal } from "./api"; + export type Thunk = () => void; /** @@ -33,24 +35,23 @@ export class Callbacks { } runAfterEachCallbacks() { - for (const c of this._afterEachCallbacks) { - c(); - } + this._afterEachCallbacks.forEach((c) => c()); } runBeforeEachCallbacks() { - for (const c of this._beforeEachCallbacks) { - c(); - } + this._beforeEachCallbacks.forEach((c) => c()); } } -export const callbacks = new Callbacks(); +const defaultCallbacks = new Callbacks(); +export function getCallbacks(): Callbacks { + return getOrSetJazzerJsGlobal("callbacks", defaultCallbacks); +} export function registerAfterEachCallback(callback: Thunk) { - callbacks.registerAfterEachCallback(callback); + getCallbacks().registerAfterEachCallback(callback); } export function registerBeforeEachCallback(callback: Thunk) { - callbacks.registerBeforeEachCallback(callback); + getCallbacks().registerBeforeEachCallback(callback); } diff --git a/packages/core/callbacks.test.ts b/packages/core/callbacks.test.ts new file mode 100644 index 00000000..b2ccb057 --- /dev/null +++ b/packages/core/callbacks.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + getCallbacks, + registerAfterEachCallback, + registerBeforeEachCallback, +} from "./callback"; + +describe("callbacks", () => { + beforeEach(() => { + globalThis.JazzerJS = new Map(); + }); + + it("executes registered beforeEach callbacks", () => { + const callback = jest.fn(); + registerBeforeEachCallback(callback); + registerBeforeEachCallback(callback); + registerBeforeEachCallback(callback); + const callbacks = getCallbacks(); + callbacks.runBeforeEachCallbacks(); + expect(callback).toBeCalledTimes(3); + }); + + it("executes registered afterEach callbacks", () => { + const callback = jest.fn(); + registerAfterEachCallback(callback); + registerAfterEachCallback(callback); + registerAfterEachCallback(callback); + const callbacks = getCallbacks(); + callbacks.runAfterEachCallbacks(); + expect(callback).toBeCalledTimes(3); + }); +}); diff --git a/packages/core/core.ts b/packages/core/core.ts index 24d7c728..6cf28f3f 100644 --- a/packages/core/core.ts +++ b/packages/core/core.ts @@ -35,7 +35,7 @@ import { MemorySyncIdStrategy, registerInstrumentor, } from "@jazzer.js/instrumentor"; -import { callbacks } from "./callback"; +import { getCallbacks } from "./callback"; import { ensureFilepath, importModule } from "./utils"; import { buildFuzzerOption, Options } from "./options"; import { jazzerJs } from "./globals"; @@ -334,6 +334,7 @@ export function wrapFuzzFunctionForBugDetection( return (data: Buffer): unknown | Promise => { let fuzzTargetError: unknown; let result: unknown | Promise = undefined; + const callbacks = getCallbacks(); try { callbacks.runBeforeEachCallbacks(); result = (originalFuzzFn as fuzzer.FuzzTargetAsyncOrValue)(data); @@ -365,6 +366,7 @@ export function wrapFuzzFunctionForBugDetection( data: Buffer, done: (err?: Error) => void, ): unknown | Promise => { + const callbacks = getCallbacks(); try { callbacks.runBeforeEachCallbacks(); // Return result of fuzz target to enable sanity checks in C++ part.