diff --git a/packages/jest-runner/config.ts b/packages/jest-runner/config.ts index 35009c91..2bbf6a41 100644 --- a/packages/jest-runner/config.ts +++ b/packages/jest-runner/config.ts @@ -61,6 +61,10 @@ export function loadConfig(optionsKey = "jazzerjs"): Options { config.mode = "fuzzing"; } + if (config.mode === "fuzzing") { + config.dryRun = false; + } + if (config.verbose) { process.env.JAZZER_DEBUG = "1"; } diff --git a/tests/bug-detectors/command-injection.test.js b/tests/bug-detectors/command-injection.test.js index 433538df..e3a7d07c 100644 --- a/tests/bug-detectors/command-injection.test.js +++ b/tests/bug-detectors/command-injection.test.js @@ -29,6 +29,7 @@ describe("Command injection", () => { it("exec with EVIL command", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("execEVIL") .dir(bugDetectorDirectory) @@ -41,6 +42,7 @@ describe("Command injection", () => { it("exec with FRIENDLY command", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("execFRIENDLY") .dir(bugDetectorDirectory) @@ -51,6 +53,7 @@ describe("Command injection", () => { it("execFile with EVIL file", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("execFileEVIL") .dir(bugDetectorDirectory) @@ -63,6 +66,7 @@ describe("Command injection", () => { it("execFile with FRIENDLY file", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("execFileFRIENDLY") .dir(bugDetectorDirectory) @@ -73,6 +77,7 @@ describe("Command injection", () => { it("execFileSync with EVIL file", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("execFileSyncEVIL") .dir(bugDetectorDirectory) @@ -85,6 +90,7 @@ describe("Command injection", () => { it("execFileSync with FRIENDLY file", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("execFileSyncFRIENDLY") .dir(bugDetectorDirectory) @@ -95,6 +101,7 @@ describe("Command injection", () => { it("spawn with EVIL command", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("spawnEVIL") .dir(bugDetectorDirectory) @@ -107,6 +114,7 @@ describe("Command injection", () => { it("spawn with FRIENDLY command", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("spawnFRIENDLY") .dir(bugDetectorDirectory) @@ -117,6 +125,7 @@ describe("Command injection", () => { it("spawnSync with EVIL command", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("spawnSyncEVIL") .dir(bugDetectorDirectory) @@ -129,6 +138,7 @@ describe("Command injection", () => { it("spawnSync with FRIENDLY command", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("spawnSyncFRIENDLY") .dir(bugDetectorDirectory) @@ -139,6 +149,7 @@ describe("Command injection", () => { it("fork with EVIL command", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("forkEVIL") .dir(bugDetectorDirectory) @@ -151,6 +162,7 @@ describe("Command injection", () => { it("fork with FRIENDLY command", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("forkFRIENDLY") .dir(bugDetectorDirectory) diff --git a/tests/bug-detectors/general.test.js b/tests/bug-detectors/general.test.js index 14f3a293..6bab2fe0 100644 --- a/tests/bug-detectors/general.test.js +++ b/tests/bug-detectors/general.test.js @@ -56,6 +56,7 @@ describe("General tests", () => { it("Call with EVIL string; SYNC", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(true) .fuzzEntryPoint("CallOriginalEvilSync") .dir(bugDetectorDirectory) @@ -69,6 +70,7 @@ describe("General tests", () => { it("Call with FRIENDLY string; ASYNC", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("CallOriginalFriendlyAsync") .dir(bugDetectorDirectory) @@ -79,6 +81,7 @@ describe("General tests", () => { it("Call with FRIENDLY string; SYNC", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(true) .fuzzEntryPoint("CallOriginalFriendlySync") .dir(bugDetectorDirectory) @@ -89,6 +92,7 @@ describe("General tests", () => { it("Call with EVIL string; With done callback", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("CallOriginalEvilDoneCallback") .dir(bugDetectorDirectory) @@ -102,6 +106,7 @@ describe("General tests", () => { it("Call with EVIL string; With done callback; With try/catch", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("CallOriginalEvilDoneCallbackWithTryCatch") .dir(bugDetectorDirectory) @@ -115,6 +120,7 @@ describe("General tests", () => { it("Call with EVIL string; With done callback; With timeout", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("CallOriginalEvilDoneCallbackWithTimeout") .dir(bugDetectorDirectory) @@ -129,6 +135,7 @@ describe("General tests", () => { it("Call with EVIL string; With done callback; With timeout; With try/catch", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("CallOriginalEvilDoneCallbackWithTimeoutWithTryCatch") .dir(bugDetectorDirectory) @@ -141,6 +148,7 @@ describe("General tests", () => { it("Call with FRIENDLY string; With done callback", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("CallOriginalFriendlyDoneCallback") .dir(bugDetectorDirectory) @@ -227,6 +235,7 @@ describe("General tests", () => { it("Disable all bug detectors; Call with evil", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("DisableAllBugDetectors") .dir(bugDetectorDirectory) @@ -240,6 +249,7 @@ describe("General tests", () => { it("Jest: Test with EVIL command; SYNC", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .dir(bugDetectorDirectory) .jestTestFile("tests.fuzz.js") @@ -254,6 +264,7 @@ describe("General tests", () => { it("Jest: Test with EVIL command; ASYNC", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .dir(bugDetectorDirectory) .jestTestFile("tests.fuzz.js") @@ -270,6 +281,7 @@ describe("General tests", () => { it("Jest: Test with FRIENDLY command", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .dir(bugDetectorDirectory) .jestTestFile("tests.fuzz.js") @@ -281,6 +293,7 @@ describe("General tests", () => { it("Jest: Test with FRIENDLY command; ASYNC", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .dir(bugDetectorDirectory) .jestTestFile("tests.fuzz.js") @@ -329,6 +342,7 @@ describe("General tests", () => { it("Jest: Test with EVIL command; Done callback", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .dir(bugDetectorDirectory) .jestTestFile("tests.fuzz.js") @@ -345,6 +359,7 @@ describe("General tests", () => { it("Jest: Test with FRIENDLY command; Done callback", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .dir(bugDetectorDirectory) .jestTestFile("tests.fuzz.js") diff --git a/tests/bug-detectors/path-traversal.test.js b/tests/bug-detectors/path-traversal.test.js index 66998231..fa7251a3 100644 --- a/tests/bug-detectors/path-traversal.test.js +++ b/tests/bug-detectors/path-traversal.test.js @@ -30,6 +30,7 @@ describe("Path Traversal", () => { it("openSync with EVIL path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(true) .fuzzEntryPoint("PathTraversalFsOpenEvilSync") .dir(bugDetectorDirectory) @@ -41,6 +42,7 @@ describe("Path Traversal", () => { it("open with EVIL path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("PathTraversalFsOpenEvilAsync") .dir(bugDetectorDirectory) @@ -52,6 +54,7 @@ describe("Path Traversal", () => { it("mkdirSync with EVIL path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(true) .fuzzEntryPoint("PathTraversalFsMkdirEvilSync") .dir(bugDetectorDirectory) @@ -65,6 +68,7 @@ describe("Path Traversal", () => { it("mkdirSync with SAFE path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(true) .fuzzEntryPoint("PathTraversalFsMkdirSafeSync") .dir(bugDetectorDirectory) @@ -76,6 +80,7 @@ describe("Path Traversal", () => { it("mkdirAsync with EVIL path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("PathTraversalFsMkdirEvilAsync") .dir(bugDetectorDirectory) @@ -89,6 +94,7 @@ describe("Path Traversal", () => { it("mkdirAsync with SAFE path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("PathTraversalFsMkdirSafeAsync") .dir(bugDetectorDirectory) @@ -100,6 +106,7 @@ describe("Path Traversal", () => { it("mkdir PROMISES with SAFE path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("PathTraversalFspMkdirSafeAsync") .dir(bugDetectorDirectory) @@ -111,6 +118,7 @@ describe("Path Traversal", () => { it("mkdir PROMISES with EVIL path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("PathTraversalFspMkdirEvilAsync") .dir(bugDetectorDirectory) @@ -124,6 +132,7 @@ describe("Path Traversal", () => { it("open PROMISES with EVIL path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("PathTraversalFspOpenEvilAsync") .dir(bugDetectorDirectory) @@ -135,6 +144,7 @@ describe("Path Traversal", () => { it("joinSync with EVIL path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(true) .fuzzEntryPoint("PathTraversalJoinEvilSync") .dir(bugDetectorDirectory) @@ -146,6 +156,7 @@ describe("Path Traversal", () => { it("joinSync with SAFE path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(true) .fuzzEntryPoint("PathTraversalJoinSafeSync") .dir(bugDetectorDirectory) @@ -155,6 +166,7 @@ describe("Path Traversal", () => { it("join with EVIL path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("PathTraversalJoinEvilAsync") .dir(bugDetectorDirectory) @@ -166,6 +178,7 @@ describe("Path Traversal", () => { it("join with SAFE path", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .sync(false) .fuzzEntryPoint("PathTraversalJoinSafeAsync") .dir(bugDetectorDirectory) diff --git a/tests/bug-detectors/prototype-pollution.test.js b/tests/bug-detectors/prototype-pollution.test.js index b69ac1a9..8ec303df 100644 --- a/tests/bug-detectors/prototype-pollution.test.js +++ b/tests/bug-detectors/prototype-pollution.test.js @@ -26,6 +26,7 @@ describe("Prototype Pollution", () => { it("{} Pollution", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .dir(bugDetectorDirectory) .fuzzEntryPoint("BaseObjectPollution") .sync(true) @@ -38,6 +39,7 @@ describe("Prototype Pollution", () => { it("{} Pollution using square braces", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .dir(bugDetectorDirectory) .fuzzEntryPoint("BaseObjectPollutionWithSquareBraces") .sync(true) @@ -50,6 +52,7 @@ describe("Prototype Pollution", () => { it("[] Pollution", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .dir(bugDetectorDirectory) .fuzzEntryPoint("ArrayObjectPollution") .sync(true) @@ -62,10 +65,10 @@ describe("Prototype Pollution", () => { it("Function Pollution", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .dir(bugDetectorDirectory) .fuzzEntryPoint("FunctionObjectPollution") .sync(true) - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -77,6 +80,7 @@ describe("Prototype Pollution", () => { it('"" Pollution', () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .dir(bugDetectorDirectory) .fuzzEntryPoint("StringObjectPollution") .sync(true) @@ -91,6 +95,7 @@ describe("Prototype Pollution", () => { it("0 Pollution", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .dir(bugDetectorDirectory) .fuzzEntryPoint("NumberObjectPollution") .sync(true) @@ -105,6 +110,7 @@ describe("Prototype Pollution", () => { it("Boolean Pollution", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .dir(bugDetectorDirectory) .fuzzEntryPoint("BooleanObjectPollution") .sync(true) @@ -119,6 +125,7 @@ describe("Prototype Pollution", () => { it("Pollute using constructor.prototype", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) @@ -136,6 +143,7 @@ describe("Prototype Pollution", () => { it("Test instrumentation and local pollution with single assignment", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) @@ -151,23 +159,23 @@ describe("Prototype Pollution", () => { it("Test no instrumentation and polluting __proto__ of a class", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .dir(bugDetectorDirectory) .fuzzEntryPoint("PollutingAClass") .sync(true) - .verbose(true) .build(); fuzzTest.execute(); }); it("Instrumentation on and polluting __proto__ of a class", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) .dir(bugDetectorDirectory) .fuzzEntryPoint("PollutingAClass") .sync(true) - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -177,23 +185,23 @@ describe("Prototype Pollution", () => { it("Instrumentation on with excluded exact match", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all-exclude-one.config.js"), ]) .dir(bugDetectorDirectory) .fuzzEntryPoint("PollutingAClass") .sync(true) - .verbose(true) .build(); fuzzTest.execute(); }); it("Detect changed toString() of {}", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .dir(bugDetectorDirectory) .fuzzEntryPoint("ChangedToString") .sync(true) - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -203,10 +211,10 @@ describe("Prototype Pollution", () => { it("Detect deleted toString() of {}", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .dir(bugDetectorDirectory) .fuzzEntryPoint("DeletedToString") .sync(true) - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -216,13 +224,13 @@ describe("Prototype Pollution", () => { it("Two-stage prototype pollution with object creation", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) .dir(bugDetectorDirectory) .fuzzEntryPoint("TwoStagePollutionWithObjectCreation") .sync(true) - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -232,12 +240,12 @@ describe("Prototype Pollution", () => { it("Async assignment instrumentation", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) .dir(bugDetectorDirectory) .fuzzEntryPoint("AsyncAssignment") - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -247,12 +255,12 @@ describe("Prototype Pollution", () => { it("Async variable declaration instrumentation", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) .dir(bugDetectorDirectory) .fuzzEntryPoint("AsyncVariableDeclaration") - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -262,13 +270,13 @@ describe("Prototype Pollution", () => { it("Equal assignments should be instrumented", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) .dir(bugDetectorDirectory) .fuzzEntryPoint("EqualExpressionInstrumentation") .sync(true) - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -278,13 +286,13 @@ describe("Prototype Pollution", () => { it("Equal variable declarations should be instrumented", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) .dir(bugDetectorDirectory) .fuzzEntryPoint("EqualVariableDeclarationsInstrumentation") .sync(true) - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -315,11 +323,11 @@ describe("Prototype Pollution Jest tests", () => { it("PP pollution of Object", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .dir(bugDetectorDirectory) .dryRun(true) .jestTestFile("tests.fuzz.js") .jestTestName("Pollution of Object") - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -331,6 +339,7 @@ describe("Prototype Pollution Jest tests", () => { it("Instrumentation of assignment expressions", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) @@ -338,7 +347,6 @@ describe("Prototype Pollution Jest tests", () => { .dryRun(false) .jestTestFile("tests.fuzz.js") .jestTestName("Assignments") - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -350,6 +358,7 @@ describe("Prototype Pollution Jest tests", () => { it("Instrumentation of variable declarations", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) @@ -357,7 +366,6 @@ describe("Prototype Pollution Jest tests", () => { .dryRun(false) .jestTestFile("tests.fuzz.js") .jestTestName("Variable declarations") - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -369,12 +377,12 @@ describe("Prototype Pollution Jest tests", () => { it("Fuzzing mode pollution of Object", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .dir(bugDetectorDirectory) .dryRun(true) .jestRunInFuzzingMode(true) .jestTestFile("tests.fuzz.js") .jestTestName("Fuzzing mode pollution of Object") - .verbose(true) .build(); expect(() => { fuzzTest.execute(); @@ -385,24 +393,6 @@ describe("Prototype Pollution Jest tests", () => { "Prototype Pollution: Prototype of Object changed", ); }); - - it("Fuzzing mode instrumentation off - variable declaration", () => { - const fuzzTest = new FuzzTestBuilder() - .customHooks([ - path.join(bugDetectorDirectory, "instrument-all.config.js"), - ]) - .dir(bugDetectorDirectory) - .dryRun(true) - .jestRunInFuzzingMode(true) - .jestTestFile("tests.fuzz.js") - .jestTestName("Variable declarations") - .verbose(true) - .build(); - expect(() => { - fuzzTest.execute(); - }).toThrow(); - expect(fuzzTest.stderr).toContain("[Prototype Pollution Configuration]"); - }); }); describe("Prototype Pollution instrumentation correctness tests", () => { @@ -411,45 +401,46 @@ describe("Prototype Pollution instrumentation correctness tests", () => { it("Basic assignment", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) .dir(bugDetectorDirectory) .fuzzEntryPoint("OnePlusOne") .fuzzFile(fuzzFile) - .verbose(true) .build(); fuzzTest.execute(); }); it("Assign to called lambda", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) .dir(bugDetectorDirectory) .fuzzEntryPoint("LambdaAssignmentAndExecution") .fuzzFile(fuzzFile) - .verbose(true) .build(); fuzzTest.execute(); }); it("Assign to lambda and then execute", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) .dir(bugDetectorDirectory) .fuzzEntryPoint("LambdaAssignmentAndExecutionLater") .fuzzFile(fuzzFile) - .verbose(true) .build(); fuzzTest.execute(); }); it("Lambda variable declaration", () => { const fuzzTest = new FuzzTestBuilder() + .runs(0) .customHooks([ path.join(bugDetectorDirectory, "instrument-all.config.js"), ]) @@ -457,7 +448,6 @@ describe("Prototype Pollution instrumentation correctness tests", () => { .dryRun(false) .fuzzEntryPoint("LambdaVariableDeclaration") .fuzzFile(fuzzFile) - .verbose(true) .build(); fuzzTest.execute(); }); diff --git a/tests/helpers.js b/tests/helpers.js index 4959e17b..b20f6907 100644 --- a/tests/helpers.js +++ b/tests/helpers.js @@ -28,6 +28,8 @@ const WindowsExitCode = "1"; class FuzzTest { constructor( + includes, + excludes, customHooks, dictionaries, dir, @@ -45,6 +47,8 @@ class FuzzTest { verbose, coverage, ) { + this.includes = includes; + this.excludes = excludes; this.customHooks = customHooks; this.dictionaries = dictionaries; this.dir = dir; @@ -64,26 +68,35 @@ class FuzzTest { } execute() { - if (this.jestTestFile !== "") { + if (this.jestTestFile) { this.executeWithJest(); - return; + } else { + this.executeWithCli(); } + } + + executeWithCli() { const options = ["jazzer", this.fuzzFile]; options.push("-f " + this.fuzzEntryPoint); if (this.sync) options.push("--sync"); + if (this.coverage) options.push("--coverage"); + if (this.dryRun !== undefined) options.push("--dryRun=" + this.dryRun); + for (const include of this.includes) { + options.push("-i=" + include); + } + for (const exclude of this.excludes) { + options.push("-e=" + exclude); + } for (const bugDetector of this.disableBugDetectors) { options.push("--disable_bug_detectors=" + bugDetector); } - - if (this.customHooks) { - options.push("--custom_hooks=" + this.customHooks); + for (const customHook of this.customHooks) { + options.push("--custom_hooks=" + customHook); } - options.push("--dryRun=" + this.dryRun); - if (this.coverage) options.push("--coverage"); options.push("--"); - options.push("-runs=" + this.runs); + if (this.runs !== undefined) options.push("-runs=" + this.runs); if (this.forkMode) options.push("-fork=" + this.forkMode); - options.push("-seed=" + this.seed); + if (this.seed) options.push("-seed=" + this.seed); for (const dictionary of this.dictionaries) { options.push("-dict=" + dictionary); } @@ -91,23 +104,43 @@ class FuzzTest { } executeWithJest() { - // Put together the libfuzzer options. - const fuzzerOptions = ["-runs=" + this.runs, "-seed=" + this.seed]; + const fuzzerOptions = []; + if (this.runs) { + fuzzerOptions.push("-runs=" + this.runs); + } + if (this.seed) { + fuzzerOptions.push("-seed=" + this.seed); + } const dictionaries = this.dictionaries.map( (dictionary) => "-dict=" + dictionary, ); fuzzerOptions.push(...dictionaries); - // Put together the jest config. - const config = { - sync: this.sync, - include: [this.jestTestFile], - disableBugDetectors: this.disableBugDetectors, - fuzzerOptions: fuzzerOptions, - customHooks: this.customHooks, - dryRun: this.dryRun, - mode: this.jestRunInFuzzingMode ? "fuzzing" : "regression", - }; + const config = {}; + if (this.sync !== undefined) { + config.sync = this.sync; + } + if (this.dryRun !== undefined) { + config.dryRun = this.dryRun; + } + if (this.includes.length > 0) { + config.includes = this.includes; + } + if (this.excludes.length > 0) { + config.excludes = this.excludes; + } + if (this.disableBugDetectors.length > 0) { + config.disableBugDetectors = this.disableBugDetectors; + } + if (fuzzerOptions.length > 0) { + config.fuzzerOptions = fuzzerOptions; + } + if (this.customHooks.length > 0) { + config.customHooks = this.customHooks; + } + if (this.jestRunInFuzzingMode !== undefined) { + config.mode = this.jestRunInFuzzingMode ? "fuzzing" : "regression"; + } // Write jest config file even if it exists fs.writeFileSync( @@ -125,6 +158,9 @@ class FuzzTest { } runTest(cmd, options, env) { + if (this.verbose) { + console.log("COMMAND: " + cmd + " " + options.join(" ")); + } const proc = spawnSync(cmd, options, { stdio: "pipe", cwd: this.dir, @@ -147,20 +183,22 @@ class FuzzTest { } class FuzzTestBuilder { - _sync = false; - _runs = 0; + _sync = undefined; + _dryRun = undefined; + _runs = undefined; _verbose = false; - _dryRun = false; _fuzzEntryPoint = ""; _dir = ""; _fuzzFile = "fuzz"; + _includes = []; + _excludes = []; _disableBugDetectors = []; - _customHooks = undefined; + _customHooks = []; _forkMode = 0; _seed = 100; _jestTestFile = ""; _jestTestName = ""; - _jestRunInFuzzingMode = false; + _jestRunInFuzzingMode = undefined; _dictionaries = []; _coverage = false; @@ -220,6 +258,28 @@ class FuzzTestBuilder { return this; } + /** + * @param {string[]} includes - instrumentation include pattern + */ + includes(includes) { + if (!Array.isArray(includes)) { + includes = [includes]; + } + this._includes = includes; + return this; + } + + /** + * @param {string[]} excludes - instrumentation excludes pattern + */ + excludes(excludes) { + if (!Array.isArray(excludes)) { + excludes = [excludes]; + } + this._excludes = excludes; + return this; + } + /** * @param {string[]} bugDetectors - bug detectors to disable. This will set Jazzer.js's command line flag * --disableBugDetectors=bugDetector1 --disableBugDetectors=bugDetector2 ... @@ -312,6 +372,8 @@ class FuzzTestBuilder { ); } return new FuzzTest( + this._includes, + this._excludes, this._customHooks, this._dictionaries, this._dir, diff --git a/tests/jest_integration/integration.test.js b/tests/jest_integration/integration.test.js new file mode 100644 index 00000000..b00d61f8 --- /dev/null +++ b/tests/jest_integration/integration.test.js @@ -0,0 +1,122 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { + FuzzTestBuilder, + FuzzingExitCode, + WindowsExitCode, +} = require("../helpers.js"); +const path = require("path"); +const fs = require("fs"); + +describe("Jest integration", () => { + const projectDir = path.join(__dirname, "jest_project"); + const jestTestFile = "integration.fuzz"; + + beforeEach(() => { + fs.rmSync(path.join(projectDir, ".jazzerjsrc.json"), { + force: true, + }); + fs.rmSync(path.join(projectDir, ".cifuzz-corpus"), { + force: true, + recursive: true, + }); + fs.rmSync(path.join(projectDir, jestTestFile), { + force: true, + recursive: true, + }); + }); + + describe("Fuzzing mode", () => { + const fuzzingExitCode = + process.platform === "win32" ? WindowsExitCode : FuzzingExitCode; + const fuzzTestBuilder = new FuzzTestBuilder() + .dir(projectDir) + .runs(1_000_000) + .jestRunInFuzzingMode(true) + .jestTestFile(jestTestFile + ".js"); + + it("executes sync test", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute sync test") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(fuzzingExitCode); + }); + + it("execute async test", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute async test") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(fuzzingExitCode); + }); + + it("execute async test returning a promise", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute async test returning a promise") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(fuzzingExitCode); + }); + + it("execute async test using a callback", () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute async test using a callback") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(fuzzingExitCode); + }); + }); + + describe("Regression mode", () => { + const regressionTestBuilder = new FuzzTestBuilder() + .dir(projectDir) + .jestTestFile(jestTestFile + ".js"); + + it("executes sync test", () => { + const fuzzTest = regressionTestBuilder + .jestTestName("execute sync test") + .build() + .execute(); + }); + + it("execute async test", () => { + const fuzzTest = regressionTestBuilder + .jestTestName("execute async test") + .build() + .execute(); + }); + + it("execute async test returning a promise", () => { + const fuzzTest = regressionTestBuilder + .jestTestName("execute async test returning a promise") + .build() + .execute(); + }); + + it("execute async test using a callback", () => { + const fuzzTest = regressionTestBuilder + .jestTestName("execute async test using a callback") + .build() + .execute(); + }); + }); +}); diff --git a/tests/jest_integration/jest_project/.gitignore b/tests/jest_integration/jest_project/.gitignore new file mode 100644 index 00000000..0e38fc1e --- /dev/null +++ b/tests/jest_integration/jest_project/.gitignore @@ -0,0 +1,3 @@ +.jazzerjsrc.json +.cifuzz-corpus +integration.fuzz diff --git a/tests/jest_integration/jest_project/integration.fuzz.js b/tests/jest_integration/jest_project/integration.fuzz.js new file mode 100644 index 00000000..0cd64c5f --- /dev/null +++ b/tests/jest_integration/jest_project/integration.fuzz.js @@ -0,0 +1,35 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const target = require("./target.js"); + +describe("Jest Integration", () => { + it.fuzz("execute sync test", (data) => { + target.fuzzMe(data); + }); + + it.fuzz("execute async test", async (data) => { + await target.asyncFuzzMe(data); + }); + + it.fuzz("execute async test returning a promise", (data) => { + return target.asyncFuzzMe(data); + }); + + it.fuzz("execute async test using a callback", (data, done) => { + target.callbackFuzzMe(data, done); + }); +}); diff --git a/tests/jest_integration/jest_project/package.json b/tests/jest_integration/jest_project/package.json new file mode 100644 index 00000000..685d1699 --- /dev/null +++ b/tests/jest_integration/jest_project/package.json @@ -0,0 +1,29 @@ +{ + "name": "jazzerjs-jest-integration-tests-project", + "version": "1.0.0", + "scripts": { + "test": "jest", + "fuzz": "JAZZER_FUZZ=1 jest " + }, + "devDependencies": { + "@jazzer.js/jest-runner": "file:../../../packages/jest-runner", + "jest": "^29.3.1" + }, + "jest": { + "projects": [ + { + "displayName": "test" + }, + { + "runner": "@jazzer.js/jest-runner", + "displayName": { + "name": "Jazzer.js", + "color": "cyan" + }, + "testMatch": [ + "/**/*.fuzz.js" + ] + } + ] + } +} diff --git a/tests/jest_integration/jest_project/target.js b/tests/jest_integration/jest_project/target.js new file mode 100644 index 00000000..b9a92bc2 --- /dev/null +++ b/tests/jest_integration/jest_project/target.js @@ -0,0 +1,47 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const fuzzMe = function (data) { + if (data.toString() === "Awesome") { + throw Error("Welcome to Awesome Fuzzing!"); + } +}; + +const asyncFuzzMe = function (data) { + return new Promise((resolve, reject) => { + try { + fuzzMe(data); + resolve(); + } catch (e) { + reject(e); + } + }); +}; + +const callbackFuzzMe = function (data, done) { + setImmediate(() => { + try { + fuzzMe(data); + done(); + } catch (e) { + done(e); + } + }); +}; + +module.exports.fuzzMe = fuzzMe; +module.exports.asyncFuzzMe = asyncFuzzMe; +module.exports.callbackFuzzMe = callbackFuzzMe; diff --git a/tests/jest_integration/package.json b/tests/jest_integration/package.json new file mode 100644 index 00000000..986c3cce --- /dev/null +++ b/tests/jest_integration/package.json @@ -0,0 +1,12 @@ +{ + "name": "jazzerjs-jest-integration-tests", + "version": "1.0.0", + "description": "Jest integration tests.", + "scripts": { + "fuzz": "jest" + }, + "devDependencies": { + "@types/jest": "^29.5.3", + "jest": "^29.6.2" + } +}