Deeper Jest integration

.npmignore

@@ -18,7 +18,7 @@ docs
 example
 
 # Exclude all tests, test helpers and such
-*test*
+*.test.*
 
 # Exclude all TypeScript source files
 *.ts

docs/README.md

@@ -5,6 +5,7 @@
 - [Using fuzz targets](fuzz-targets.md)
 - [Using Jest integration](jest-integration.md)
 - [Advanced Fuzzing Settings](fuzz-settings.md)
+- [Migration Guide](migration-guide.md)
 
 ## Internal Documentation
 

docs/fuzz-targets.md

@@ -115,20 +115,24 @@ loaded properly. If your project internally still relies on calls to
 `require()`, all of these dependencies will be hooked. However, _pure_
 ECMAScript projects will currently not be instrumented!
 
+The Jest integration can improve on this and use Jest's ESM features to properly
+transform external code and dependencies. However,
+[ESM support](https://jestjs.io/docs/ecmascript-modules) in Jest is also only
+experimental.
+
 One such example that Jazzer.js can handle just fine can be found at
-[examples/protobufjs/fuzz.js](../examples/protobufjs/fuzz.js):
+[examples/protobufjs/fuzz.js](../examples/protobufjs/protobufjs.fuzz.js):
 
 ```js
 import proto from "protobufjs";
 import { temporaryWriteSync } from "tempy";
 
-export function fuzz(data: Buffer) {
-	try {
-		// Fuzz logic
-	} catch (e) {
-		// Handle expected error logic here
-	}
-}
+describe("protobufjs", () => {
+	test.fuzz("loadSync", (data) => {
+		const file = temporaryWriteSync(data);
+		proto.loadSync(file);
+	});
+});
 ```
 
 You also have to adapt your `package.json` accordingly, by adding:

docs/jest-integration.md

@@ -34,6 +34,8 @@ npm install --save-dev @jazzer.js/jest-runner
 This will install the custom Jest runner along with all other required Jazzer.js
 dependencies.
 
+**Note**: The Jazzer.js Jest runner requires Jest version 29 or higher.
+
 For Jest to pick up the custom fuzz test runner, it has to be added to the Jest
 configuration in the `package.json` or `jest.config.js`, as described on the
 [Configuring Jest](https://jestjs.io/docs/configuration) documentation page. The
@@ -60,12 +62,12 @@ runner.
 				"displayName": "test"
 			},
 			{
-				"runner": "@jazzer.js/jest-runner",
 				"displayName": {
 					"name": "Jazzer.js",
 					"color": "cyan"
 				},
-				"testMatch": ["<rootDir>/**/*.fuzz.js"]
+				"testMatch": ["<rootDir>/**/*.fuzz.js"],
+				"testRunner": "@jazzer.js/jest-runner"
 			}
 		]
 	}
@@ -117,9 +119,9 @@ actually include test files with the `.fuzz.ts` extension.
     color: "cyan",
   },
   preset: "ts-jest",
-  runner: "@jazzer.js/jest-runner",
   testEnvironment: "node",
   testMatch: ["<rootDir>/*.fuzz.[jt]s"],
+  testRunner: "@jazzer.js/jest-runner",
 },
 ```
 
@@ -235,14 +237,6 @@ describe("Target", () => {
 });
 ```
 
-### Setup and teardown
-
-The Jazzer.js fuzz test runner supports Jest's setup and teardown functions, as
-described on the [Setup and Teardown](https://jestjs.io/docs/setup-teardown)
-documentation page.
-
-This includes `beforeAll`, `afterAll`, `beforeEach` and `afterEach`.
-
 ## Executing Jest fuzz tests
 
 As mentioned above, the Jazzer.js fuzz test runner provides two modes of
@@ -402,13 +396,3 @@ enter the debugger for that particular input.
 ### IntelliJ Jest support
 
 ![IntelliJ Jest integration](pictures/jest-integration-intellij.png)
-
-## Unsupported Jest features
-
-The Jazzer.js fuzz test runner strives to provide neat Jest integration. That
-being said, some Jest features are currently not supported, as Jest does not
-offer good extension points and common test framework features have to be
-reimplemented.
-
-- Mock functions
-- Isolated workers

docs/migration-guide.md

@@ -0,0 +1,30 @@
+# Migration Guide
+
+This document describes breaking changes in major versions and steps to migrate
+from one to the next.
+
+## 2.0.0
+
+This version fundamentally changed the Jest integration in module
+`@jazzer.js/jest-runner`. The new approach provides a tighter integration with
+Jest and allows fuzz tests to use all available Jest features. Most notably this
+includes the widely missed mocking functionality.
+
+### Migration steps
+
+- In the Jest configuration, move `@jazzer.js/jest-runner` from `runner` to
+  `testRunner`. A valid configuration looks like this:
+
+```diff
+{
+  displayName: {
+  	name: "Jazzer.js",
+  	color: "cyan",
+  },
+  preset: "ts-jest",
+-  runner: "@jazzer.js/jest-runner",
++  testRunner: "@jazzer.js/jest-runner",
+  testEnvironment: "node",
+  testMatch: ["<rootDir>/*.fuzz.[jt]s"],
+}
+```

end-to-end/README.md

@@ -12,11 +12,14 @@ jazzer.js than the other examples.
 ## Running Locally
 
 ```bash
+rm -rf node_modules package-lock.json *.tgz
 ./package-jazzer-js.sh
-npm install --save-dev *.tgz
+npm install --prefer-online --save-dev *.tgz
 npx jest
 ```
 
 _Note_: running just `npm install` may result in caching issues where the
 contents of the tarballs in this directory are ignored and older versions from
-somewhere are used instead.
+the cache are used instead. `--prefer-online` forces npm to check for updated
+files, which could cause hash mismatches compared to `package-lock.json`. Hence,
+remove `package-lock.json` and other dependencies before running the tests.

end-to-end/jest.config.ts

@@ -28,7 +28,7 @@ const config: Config = {
 				color: "cyan",
 			},
 			preset: "ts-jest",
-			runner: "@jazzer.js/jest-runner",
+			testRunner: "@jazzer.js/jest-runner",
 			testEnvironment: "node",
 			testMatch: ["<rootDir>/*.fuzz.[jt]s"],
 		},

end-to-end/package.json

@@ -1,7 +1,6 @@
 {
-	"name": "jest_typescript_integration",
+	"name": "end-to-end",
 	"version": "1.0.0",
-	"description": "An example showing how Jazzer.js integrates with Jest and TypeScript",
 	"scripts": {
 		"build": "tsc",
 		"dryRun": "jest",

examples/bug-detectors/command-injection/custom-hooks.js

@@ -15,7 +15,10 @@
  */
 
 const { registerReplaceHook } = require("@jazzer.js/hooking");
-const { guideTowardsEquality, reportFinding } = require("@jazzer.js/core");
+const {
+	guideTowardsEquality,
+	reportAndThrowFinding,
+} = require("@jazzer.js/core");
 
 /**
  * Custom bug detector for command injection. This hook does not call the original function (execSync) for two reasons:
@@ -32,7 +35,7 @@ registerReplaceHook(
 		}
 		const command = params[0];
 		if (command.includes("jaz_zer")) {
-			reportFinding(
+			reportAndThrowFinding(
 				`Command Injection in spawnSync(): called with '${command}'`,
 			);
 		}

examples/jest_integration/.jazzerjsrc.json

@@ -1,5 +1,9 @@
 {
 	"includes": ["target", "integration.fuzz", "worker.fuzz"],
-	"excludes": ["node_modules"],
-	"fuzzerOptions": ["-rss_limit_mb=16000", "-runs=100000"]
+	"excludes": ["@babel"],
+	"fuzzerOptions": [
+		"-rss_limit_mb=16000",
+		"-use_value_profile=1",
+		"-runs=1000000"
+	]
 }

examples/jest_integration/package.json

@@ -20,7 +20,7 @@
 				"displayName": "test"
 			},
 			{
-				"runner": "@jazzer.js/jest-runner",
+				"testRunner": "@jazzer.js/jest-runner",
 				"displayName": {
 					"name": "Jazzer.js",
 					"color": "cyan"

examples/jest_integration/target.js

@@ -19,14 +19,10 @@
  */
 const fuzzMe = function (data) {
 	const s = data.toString();
-	if (s.length !== 16) {
+	if (s.length !== 7) {
 		return;
 	}
-	if (
-		s.slice(0, 8) === "Awesome " &&
-		s.slice(8, 15) === "Fuzzing" &&
-		s[15] === "!"
-	) {
+	if (s.slice(0, 7) === "Awesome") {
 		throw Error("Welcome to Awesome Fuzzing!");
 	}
 };

examples/jest_integration/worker.fuzz.js

@@ -24,7 +24,7 @@ const addCallLog = (uniqueId) => {
 
 beforeAll(() => {
 	return new Promise((resolve) => {
-		setTimeout(() => {
+		setImmediate(() => {
 			addCallLog("Top-level beforeAll");
 			resolve(undefined);
 		}, 100);
@@ -33,8 +33,7 @@ beforeAll(() => {
 
 describe("Hooks", () => {
 	beforeAll((done) => {
-		const callLog = "My describe: beforeAll";
-		addCallLog(callLog);
+		addCallLog("My describe: beforeAll");
 		done();
 	});
 

examples/jest_typescript_integration/jest.config.ts

@@ -29,7 +29,7 @@ const config: Config = {
 				color: "cyan",
 			},
 			preset: "ts-jest",
-			runner: "@jazzer.js/jest-runner",
+			testRunner: "@jazzer.js/jest-runner",
 			testEnvironment: "node",
 			testMatch: ["<rootDir>/*.fuzz.[jt]s"],
 		},

examples/protobufjs/package.json

@@ -3,14 +3,32 @@
 	"version": "0.0.0",
 	"type": "module",
 	"scripts": {
-		"fuzz": "npx jazzer fuzz --sync -i protobuf",
-		"dryRun": "npx jazzer fuzz -m regression --sync -i protobuf -- -runs=100 -seed=123456789"
+		"fuzz_esm": "JAZZER_FUZZ=1 NODE_OPTIONS=--experimental-vm-modules npx jest",
+		"dryRun": "echo \"Skipped\"",
+		"dryRun_esm": "NODE_OPTIONS=--experimental-vm-modules npx jest"
 	},
 	"dependencies": {
 		"protobufjs": "^7.0.0",
 		"tempy": "^3.0.0"
 	},
 	"devDependencies": {
-		"@jazzer.js/cli": "file:../../packages/core"
+		"@jazzer.js/jest-runner": "file:../../packages/jest-runner"
+	},
+	"engines": {
+		"node": ">= 18.8.0"
+	},
+	"jest": {
+		"projects": [
+			{
+				"testRunner": "@jazzer.js/jest-runner",
+				"displayName": {
+					"name": "Jazzer.js",
+					"color": "cyan"
+				},
+				"testMatch": [
+					"<rootDir>/**/*.fuzz.js"
+				]
+			}
+		]
 	}
 }