Jazzer.js now supports coverage report generation. Simply add "--coverage".

.config/test-jazzerjsrc.json

@@ -1,5 +1,5 @@
 {
 	"includes": ["target"],
 	"excludes": ["nothing"],
-	"fuzzerOptions": ["-runs=-1", "-rss_limit_mb=16000"]
+	"fuzzerOptions": ["-rss_limit_mb=16000"]
 }

docs/fuzz-targets.md

@@ -173,3 +173,39 @@ flag, so that only the most important parameters are discussed here.
 | `-h`, `--custom_hooks`                                                  | Filenames with custom hooks. Several hooks per file are possible. See further details in [docs/fuzz-settings.md](fuzz-settings.md).                                                                                                                                                                      |
 | `--help`                                                                | Detailed help message containing all flags.                                                                                                                                                                                                                                                              |
 | `-- <fuzzingEngineFlags>`                                               | Parameters after `--` are forwarded to the internal fuzzing engine (`libFuzzer`). Available settings can be found in its [options documentation](https://www.llvm.org/docs/LibFuzzer.html#options).                                                                                                      |
+
+## Coverage report generation
+
+To generate a coverage report, add the `--coverage` flag to the Jazzer.js CLI.
+In this example, the `--coverage` flag is combined with the dry run flag `-d`
+that disables internal instrumentation used by the fuzzer.
+
+```shell
+npx jazzer -d <fuzzer parameters> --corpus <corpus directories> --coverage -- <libFuzzer parameters>
+```
+
+Alternatively, you can add a new script to your package.json:
+
+```json
+"scripts": {
+	"coverage": "jazzer -d -i target -i another_target -e nothing <fuzzer parameters> --corpus <corpus directories> --coverage -- <libFuzzer parameters>"
+}
+```
+
+Files matched by the flags `--include` or `--custom_hooks`, and not matched by
+the flag `--exclude` will be included in the coverage report. It is recommended
+to disable coverage report generation during fuzzing, because of the substantial
+overhead that it adds.
+
+### Coverage report directory
+
+By default, the coverage reports can be found in the `./coverage` directory.
+This default directory can be changed by setting the flag
+`--coverageDirectory=<another coverage directory>`.
+
+### Coverage reporters
+
+The desired report format can be set by the flag `--coverageReports`, which by
+default is set to `--coverageReports clover json lcov text`. See
+[here](https://github.com/istanbuljs/istanbuljs/tree/master/packages/istanbul-reports/lib)
+for a list of supported coverage reporters.

docs/jest-integration.md

@@ -28,7 +28,7 @@ your project. To do so, execute the following command in your project root
 directory.
 
 ```shell
-npm install --save-dev @jazzer.js/jest-integration
+npm install --save-dev @jazzer.js/jest-runner
 ```
 
 This will install the custom Jest runner along with all other required Jazzer.js
@@ -47,7 +47,8 @@ runner.
 	"name": "jest-integration-example",
 	"scripts": {
 		"test": "jest",
-		"fuzz": "JAZZER_FUZZ=1 jest"
+		"fuzz": "JAZZER_FUZZ=1 jest",
+		"coverage": "jest --coverage"
 	},
 	"devDependencies": {
 		"@jazzer.js/jest-runner": "1.1.0",
@@ -283,6 +284,22 @@ Time:        0.335 s, estimated 1 s
 Ran all test suites.
 ```
 
+### Coverage report generation
+
+To generate a coverage report, run jest with the `--coverage` flag:
+
+```shell
+npx jest --coverage
+```
+
+Additional options for coverage report generation are described in the
+[fuzz targets documentation](./fuzz-targets.md#coverage-report-generation).
+
+The desired report format can be set by the flag `--coverageReports`, which by
+default is set to `--coverageReports clover json lcov text`. See
+[here](https://github.com/istanbuljs/istanbuljs/tree/master/packages/istanbul-reports/lib)
+for a list of supported coverage reporters.
+
 ## IDE Integration
 
 As the Jest test framework foundations are used by the Jazzer.js fuzz test
@@ -309,6 +326,5 @@ offer good extension points and common test framework features have to be
 reimplemented.
 
 - Mock functions
-- Coverage generation
 - Isolated workers
 - Test-based timeouts (third parameter to `test` functions)

examples/custom-hooks/package.json

@@ -8,7 +8,8 @@
 	},
 	"scripts": {
 		"fuzz": "jazzer fuzz -i jpeg-js -e nothing -h custom-hooks --sync",
-		"dryRun": "jazzer fuzz -i jpeg-js -e nothing --sync -h custom-hooks -- -runs=100 -seed=123456789"
+		"dryRun": "jazzer fuzz -i jpeg-js -e nothing --sync -h custom-hooks -- -runs=100 -seed=123456789",
+		"coverage": "jazzer fuzz -i jpeg-js -i fuzz.js -i custom-hooks.js -e nothing -h custom-hooks --sync --coverage -- -max_total_time=10"
 	},
 	"devDependencies": {
 		"@jazzer.js/core": "file:../../packages/core"

examples/jest_integration/.jazzerjsrc.json

@@ -1,5 +1,5 @@
 {
-	"includes": ["target", "integration.fuzz"],
+	"includes": ["target", "integration.fuzz", "worker.fuzz"],
 	"excludes": ["node_modules"],
-	"fuzzerOptions": ["-runs=-1", "-rss_limit_mb=16000"]
+	"fuzzerOptions": ["-rss_limit_mb=16000"]
 }

examples/jest_integration/package.json

@@ -5,7 +5,8 @@
 	"scripts": {
 		"test": "jest",
 		"dryRun": "jest",
-		"fuzz": "JAZZER_FUZZ=1 jest --testNamePattern=\"My describe\""
+		"fuzz": "JAZZER_FUZZ=1 jest --coverage --testNamePattern=\"My describe\"",
+		"coverage": "jest --coverage"
 	},
 	"devDependencies": {
 		"@jazzer.js/jest-runner": "file:../../packages/jest-runner",

examples/jpeg/fuzz.js

@@ -29,4 +29,8 @@ const ignored = [
 	"DecoderBuffer",
 	"invalid table spec",
 	"SOI not found",
+	"Could not",
+	"limit exceeded by",
+	"sampling factor",
+	"Cannot read properties of undefined",
 ];

examples/jpeg/package.json

@@ -10,7 +10,8 @@
 	},
 	"scripts": {
 		"fuzz": "jazzer fuzz -i jpeg-js -e nothing --sync",
-		"dryRun": "jazzer fuzz -i jpeg-js -e nothing --sync -- -runs=100 -seed=123456789"
+		"dryRun": "jazzer fuzz -i jpeg-js -e nothing --sync -- -runs=100 -seed=123456789",
+		"coverage": "jazzer fuzz -i jpeg-js/lib -i fuzz.js -e nothing --sync --coverage -- -max_total_time=1 -seed=123456789"
 	},
 	"devDependencies": {
 		"@jazzer.js/core": "file:../../packages/core"

jest.config.js

@@ -18,7 +18,11 @@
 module.exports = {
 	preset: "ts-jest",
 	testEnvironment: "node",
-	modulePathIgnorePatterns: ["dist", "packages/fuzzer/build"],
+	modulePathIgnorePatterns: [
+		"dist",
+		"packages/fuzzer/build",
+		"tests/code_coverage",
+	],
 	collectCoverageFrom: ["packages/**/*.ts"],
 	coveragePathIgnorePatterns: ["/node_modules/", "/dist/"],
 };

package.json

@@ -47,6 +47,7 @@
 		"eslint-plugin-markdownlint": "^0.4.0",
 		"husky": "^8.0.3",
 		"jest": "^29.4.0",
+		"istanbul-lib-coverage": "^3.2.0",
 		"lint-staged": "^13.1.0",
 		"prettier": "2.8.3",
 		"run-script-os": "^1.1.6",

packages/core/cli.ts

@@ -161,6 +161,26 @@ yargs(process.argv.slice(2))
 					alias: "v",
 					group: "Fuzzer:",
 					default: false,
+				})
+				.boolean("coverage")
+				.option("coverage", {
+					describe: "Enable code coverage.",
+					type: "boolean",
+					group: "Fuzzer:",
+					default: false,
+				})
+				.option("coverageDirectory", {
+					describe: "Directory for storing coverage reports.",
+					type: "string",
+					default: "coverage",
+					group: "Fuzzer:",
+				})
+				.array("coverageReporters")
+				.option("coverageReporters", {
+					describe: "A list of reporter names for writing coverage reports.",
+					type: "string",
+					group: "Fuzzer:",
+					default: ["json", "text", "lcov", "clover"],
 				});
 		},
 		// eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -177,9 +197,12 @@ yargs(process.argv.slice(2))
 				dryRun: args.dry_run,
 				sync: args.sync,
 				fuzzerOptions: args.corpus.concat(args._),
-				customHooks: args.custom_hooks.map(ensureFilepath),
+				customHooks: args.custom_hooks,
 				expectedErrors: args.expected_errors,
 				idSyncFile: args.id_sync_file,
+				coverage: args.coverage,
+				coverageDirectory: args.coverageDirectory,
+				coverageReporters: args.coverageReporters,
 			});
 		}
 	)

packages/core/core.test.ts

@@ -15,6 +15,7 @@
  */
 
 import { ensureFilepath } from "./core";
+
 import path from "path";
 
 describe("core", () => {