diff --git a/.config/test-jazzerjsrc.json b/.config/test-jazzerjsrc.json index b5a7818e..bd27cebf 100644 --- a/.config/test-jazzerjsrc.json +++ b/.config/test-jazzerjsrc.json @@ -1,5 +1,5 @@ { "includes": ["target"], "excludes": ["nothing"], - "fuzzerOptions": ["-runs=-1", "-rss_limit_mb=16000"] + "fuzzerOptions": ["-rss_limit_mb=16000"] } diff --git a/docs/fuzz-targets.md b/docs/fuzz-targets.md index a2d3315e..fe63fbff 100644 --- a/docs/fuzz-targets.md +++ b/docs/fuzz-targets.md @@ -173,3 +173,39 @@ flag, so that only the most important parameters are discussed here. | `-h`, `--custom_hooks` | Filenames with custom hooks. Several hooks per file are possible. See further details in [docs/fuzz-settings.md](fuzz-settings.md). | | `--help` | Detailed help message containing all flags. | | `-- ` | Parameters after `--` are forwarded to the internal fuzzing engine (`libFuzzer`). Available settings can be found in its [options documentation](https://www.llvm.org/docs/LibFuzzer.html#options). | + +## Coverage report generation + +To generate a coverage report, add the `--coverage` flag to the Jazzer.js CLI. +In this example, the `--coverage` flag is combined with the dry run flag `-d` +that disables internal instrumentation used by the fuzzer. + +```shell +npx jazzer -d --corpus --coverage -- +``` + +Alternatively, you can add a new script to your package.json: + +```json +"scripts": { + "coverage": "jazzer -d -i target -i another_target -e nothing --corpus --coverage -- " +} +``` + +Files matched by the flags `--include` or `--custom_hooks`, and not matched by +the flag `--exclude` will be included in the coverage report. It is recommended +to disable coverage report generation during fuzzing, because of the substantial +overhead that it adds. + +### Coverage report directory + +By default, the coverage reports can be found in the `./coverage` directory. +This default directory can be changed by setting the flag +`--coverageDirectory=`. + +### Coverage reporters + +The desired report format can be set by the flag `--coverageReports`, which by +default is set to `--coverageReports clover json lcov text`. See +[here](https://github.com/istanbuljs/istanbuljs/tree/master/packages/istanbul-reports/lib) +for a list of supported coverage reporters. diff --git a/docs/jest-integration.md b/docs/jest-integration.md index 818c1d15..e7cc6dcd 100644 --- a/docs/jest-integration.md +++ b/docs/jest-integration.md @@ -28,7 +28,7 @@ your project. To do so, execute the following command in your project root directory. ```shell -npm install --save-dev @jazzer.js/jest-integration +npm install --save-dev @jazzer.js/jest-runner ``` This will install the custom Jest runner along with all other required Jazzer.js @@ -47,7 +47,8 @@ runner. "name": "jest-integration-example", "scripts": { "test": "jest", - "fuzz": "JAZZER_FUZZ=1 jest" + "fuzz": "JAZZER_FUZZ=1 jest", + "coverage": "jest --coverage" }, "devDependencies": { "@jazzer.js/jest-runner": "1.1.0", @@ -283,6 +284,22 @@ Time: 0.335 s, estimated 1 s Ran all test suites. ``` +### Coverage report generation + +To generate a coverage report, run jest with the `--coverage` flag: + +```shell +npx jest --coverage +``` + +Additional options for coverage report generation are described in the +[fuzz targets documentation](./fuzz-targets.md#coverage-report-generation). + +The desired report format can be set by the flag `--coverageReports`, which by +default is set to `--coverageReports clover json lcov text`. See +[here](https://github.com/istanbuljs/istanbuljs/tree/master/packages/istanbul-reports/lib) +for a list of supported coverage reporters. + ## IDE Integration As the Jest test framework foundations are used by the Jazzer.js fuzz test @@ -309,6 +326,5 @@ offer good extension points and common test framework features have to be reimplemented. - Mock functions -- Coverage generation - Isolated workers - Test-based timeouts (third parameter to `test` functions) diff --git a/examples/custom-hooks/package.json b/examples/custom-hooks/package.json index 32e51518..ea0af690 100644 --- a/examples/custom-hooks/package.json +++ b/examples/custom-hooks/package.json @@ -8,7 +8,8 @@ }, "scripts": { "fuzz": "jazzer fuzz -i jpeg-js -e nothing -h custom-hooks --sync", - "dryRun": "jazzer fuzz -i jpeg-js -e nothing --sync -h custom-hooks -- -runs=100 -seed=123456789" + "dryRun": "jazzer fuzz -i jpeg-js -e nothing --sync -h custom-hooks -- -runs=100 -seed=123456789", + "coverage": "jazzer fuzz -i jpeg-js -i fuzz.js -i custom-hooks.js -e nothing -h custom-hooks --sync --coverage -- -max_total_time=10" }, "devDependencies": { "@jazzer.js/core": "file:../../packages/core" diff --git a/examples/jest_integration/.jazzerjsrc.json b/examples/jest_integration/.jazzerjsrc.json index fbae6aa3..c1ef48bd 100644 --- a/examples/jest_integration/.jazzerjsrc.json +++ b/examples/jest_integration/.jazzerjsrc.json @@ -1,5 +1,5 @@ { - "includes": ["target", "integration.fuzz"], + "includes": ["target", "integration.fuzz", "worker.fuzz"], "excludes": ["node_modules"], - "fuzzerOptions": ["-runs=-1", "-rss_limit_mb=16000"] + "fuzzerOptions": ["-rss_limit_mb=16000"] } diff --git a/examples/jest_integration/package.json b/examples/jest_integration/package.json index 8d9b1f15..19e13759 100644 --- a/examples/jest_integration/package.json +++ b/examples/jest_integration/package.json @@ -5,7 +5,8 @@ "scripts": { "test": "jest", "dryRun": "jest", - "fuzz": "JAZZER_FUZZ=1 jest --testNamePattern=\"My describe\"" + "fuzz": "JAZZER_FUZZ=1 jest --coverage --testNamePattern=\"My describe\"", + "coverage": "jest --coverage" }, "devDependencies": { "@jazzer.js/jest-runner": "file:../../packages/jest-runner", diff --git a/examples/jpeg/fuzz.js b/examples/jpeg/fuzz.js index c7ecccca..987d1028 100644 --- a/examples/jpeg/fuzz.js +++ b/examples/jpeg/fuzz.js @@ -29,4 +29,8 @@ const ignored = [ "DecoderBuffer", "invalid table spec", "SOI not found", + "Could not", + "limit exceeded by", + "sampling factor", + "Cannot read properties of undefined", ]; diff --git a/examples/jpeg/package.json b/examples/jpeg/package.json index dd1bf484..7dd23fc5 100644 --- a/examples/jpeg/package.json +++ b/examples/jpeg/package.json @@ -10,7 +10,8 @@ }, "scripts": { "fuzz": "jazzer fuzz -i jpeg-js -e nothing --sync", - "dryRun": "jazzer fuzz -i jpeg-js -e nothing --sync -- -runs=100 -seed=123456789" + "dryRun": "jazzer fuzz -i jpeg-js -e nothing --sync -- -runs=100 -seed=123456789", + "coverage": "jazzer fuzz -i jpeg-js/lib -i fuzz.js -e nothing --sync --coverage -- -max_total_time=1 -seed=123456789" }, "devDependencies": { "@jazzer.js/core": "file:../../packages/core" diff --git a/jest.config.js b/jest.config.js index 023f0aec..5d0848f3 100644 --- a/jest.config.js +++ b/jest.config.js @@ -18,7 +18,11 @@ module.exports = { preset: "ts-jest", testEnvironment: "node", - modulePathIgnorePatterns: ["dist", "packages/fuzzer/build"], + modulePathIgnorePatterns: [ + "dist", + "packages/fuzzer/build", + "tests/code_coverage", + ], collectCoverageFrom: ["packages/**/*.ts"], coveragePathIgnorePatterns: ["/node_modules/", "/dist/"], }; diff --git a/package-lock.json b/package-lock.json index 6169b2e7..aefc4d32 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "eslint-plugin-jest": "^27.2.1", "eslint-plugin-markdownlint": "^0.4.0", "husky": "^8.0.3", + "istanbul-lib-coverage": "^3.2.0", "jest": "^29.4.0", "lint-staged": "^13.1.0", "prettier": "2.8.3", @@ -1184,6 +1185,12 @@ "@babel/types": "^7.3.0" } }, + "node_modules/@types/babel-types": { + "version": "7.0.11", + "resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.11.tgz", + "integrity": "sha512-pkPtJUUY+Vwv6B1inAz55rQvivClHJxc9aVEPPmaq2cbyeMLCiDpbKpcKyX4LAwpNGi+SHBv0tHv6+0gXv0P2A==", + "dev": true + }, "node_modules/@types/bindings": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/@types/bindings/-/bindings-1.5.1.tgz", @@ -1212,6 +1219,17 @@ "integrity": "sha512-PVTxogFmhYu3tJRhBA8PCHWuB07lX44ZbYnCUKonclSVfl+7DAV+7brYOIhVYHY5eAljT6KDfVa0855QGJWtKg==", "dev": true }, + "node_modules/@types/istanbul-lib-instrument": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-instrument/-/istanbul-lib-instrument-1.7.4.tgz", + "integrity": "sha512-1i1VVkU2KrpZCmti+t5J/zBb2KLKxHgU1EYL+0QtnDnVyZ59aSKcpnG6J0I6BZGDON566YzPNIlNfk7m+9l1JA==", + "dev": true, + "dependencies": { + "@types/babel-types": "*", + "@types/istanbul-lib-coverage": "*", + "source-map": "^0.6.1" + } + }, "node_modules/@types/istanbul-lib-report": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz", @@ -8431,6 +8449,9 @@ "dependencies": { "@jazzer.js/hooking": "*", "@jazzer.js/instrumentor": "*", + "istanbul-lib-coverage": "^3.2.0", + "istanbul-lib-report": "^3.0.0", + "istanbul-reports": "^3.1.5", "tmp": "^0.2.1", "yargs": "^17.6.2" }, @@ -8492,12 +8513,14 @@ "@jazzer.js/fuzzer": "*", "@jazzer.js/hooking": "*", "istanbul-lib-hook": "^3.0.0", + "istanbul-lib-instrument": "^5.2.1", "proper-lockfile": "^4.1.2", "source-map-support": "^0.5.21" }, "devDependencies": { "@types/babel__core": "^7.20.0", "@types/istanbul-lib-hook": "^2.0.1", + "@types/istanbul-lib-instrument": "^1.7.4", "@types/node": "^18.11.18", "@types/proper-lockfile": "^4.1.2", "@types/source-map-support": "^0.5.6" @@ -8535,9 +8558,11 @@ "dependencies": { "@jazzer.js/core": "*", "cosmiconfig": "^8.0.0", + "istanbul-reports": "^3.1.5", "jest": "^29.4.0" }, "devDependencies": { + "@types/istanbul-reports": "^3.0.1", "@types/tmp": "^0.2.3", "tmp": "^0.2.1" }, @@ -9091,6 +9116,9 @@ "@jazzer.js/hooking": "*", "@jazzer.js/instrumentor": "*", "@types/yargs": "^17.0.20", + "istanbul-lib-coverage": "^3.2.0", + "istanbul-lib-report": "^3.0.0", + "istanbul-reports": "^3.1.5", "tmp": "^0.2.1", "yargs": "^17.6.2" } @@ -9124,10 +9152,12 @@ "@jazzer.js/hooking": "*", "@types/babel__core": "^7.20.0", "@types/istanbul-lib-hook": "^2.0.1", + "@types/istanbul-lib-instrument": "^1.7.4", "@types/node": "^18.11.18", "@types/proper-lockfile": "^4.1.2", "@types/source-map-support": "^0.5.6", "istanbul-lib-hook": "^3.0.0", + "istanbul-lib-instrument": "^5.2.1", "proper-lockfile": "^4.1.2", "source-map-support": "^0.5.21" }, @@ -9147,8 +9177,10 @@ "version": "file:packages/jest-runner", "requires": { "@jazzer.js/core": "*", + "@types/istanbul-reports": "^3.0.1", "@types/tmp": "^0.2.3", "cosmiconfig": "^8.0.0", + "istanbul-reports": "^3.1.5", "jest": "^29.4.0", "tmp": "^0.2.1" } @@ -9483,6 +9515,12 @@ "@babel/types": "^7.3.0" } }, + "@types/babel-types": { + "version": "7.0.11", + "resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.11.tgz", + "integrity": "sha512-pkPtJUUY+Vwv6B1inAz55rQvivClHJxc9aVEPPmaq2cbyeMLCiDpbKpcKyX4LAwpNGi+SHBv0tHv6+0gXv0P2A==", + "dev": true + }, "@types/bindings": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/@types/bindings/-/bindings-1.5.1.tgz", @@ -9511,6 +9549,17 @@ "integrity": "sha512-PVTxogFmhYu3tJRhBA8PCHWuB07lX44ZbYnCUKonclSVfl+7DAV+7brYOIhVYHY5eAljT6KDfVa0855QGJWtKg==", "dev": true }, + "@types/istanbul-lib-instrument": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-instrument/-/istanbul-lib-instrument-1.7.4.tgz", + "integrity": "sha512-1i1VVkU2KrpZCmti+t5J/zBb2KLKxHgU1EYL+0QtnDnVyZ59aSKcpnG6J0I6BZGDON566YzPNIlNfk7m+9l1JA==", + "dev": true, + "requires": { + "@types/babel-types": "*", + "@types/istanbul-lib-coverage": "*", + "source-map": "^0.6.1" + } + }, "@types/istanbul-lib-report": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz", diff --git a/package.json b/package.json index 28e4ca7c..8d861acf 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "eslint-plugin-markdownlint": "^0.4.0", "husky": "^8.0.3", "jest": "^29.4.0", + "istanbul-lib-coverage": "^3.2.0", "lint-staged": "^13.1.0", "prettier": "2.8.3", "run-script-os": "^1.1.6", diff --git a/packages/core/cli.ts b/packages/core/cli.ts index 0dcc80f2..fc4658e4 100644 --- a/packages/core/cli.ts +++ b/packages/core/cli.ts @@ -161,6 +161,26 @@ yargs(process.argv.slice(2)) alias: "v", group: "Fuzzer:", default: false, + }) + .boolean("coverage") + .option("coverage", { + describe: "Enable code coverage.", + type: "boolean", + group: "Fuzzer:", + default: false, + }) + .option("coverageDirectory", { + describe: "Directory for storing coverage reports.", + type: "string", + default: "coverage", + group: "Fuzzer:", + }) + .array("coverageReporters") + .option("coverageReporters", { + describe: "A list of reporter names for writing coverage reports.", + type: "string", + group: "Fuzzer:", + default: ["json", "text", "lcov", "clover"], }); }, // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -177,9 +197,12 @@ yargs(process.argv.slice(2)) dryRun: args.dry_run, sync: args.sync, fuzzerOptions: args.corpus.concat(args._), - customHooks: args.custom_hooks.map(ensureFilepath), + customHooks: args.custom_hooks, expectedErrors: args.expected_errors, idSyncFile: args.id_sync_file, + coverage: args.coverage, + coverageDirectory: args.coverageDirectory, + coverageReporters: args.coverageReporters, }); } ) diff --git a/packages/core/core.test.ts b/packages/core/core.test.ts index 178e6ac6..a493bd4f 100644 --- a/packages/core/core.test.ts +++ b/packages/core/core.test.ts @@ -15,6 +15,7 @@ */ import { ensureFilepath } from "./core"; + import path from "path"; describe("core", () => { diff --git a/packages/core/core.ts b/packages/core/core.ts index d451ee60..699913b9 100644 --- a/packages/core/core.ts +++ b/packages/core/core.ts @@ -13,12 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - import path from "path"; import * as process from "process"; import * as tmp from "tmp"; import * as fs from "fs"; +import * as libCoverage from "istanbul-lib-coverage"; +import * as libReport from "istanbul-lib-report"; +import * as reports from "istanbul-reports"; + import * as fuzzer from "@jazzer.js/fuzzer"; import * as hooking from "@jazzer.js/hooking"; import { trackedHooks } from "@jazzer.js/hooking"; @@ -51,6 +54,9 @@ export interface Options { expectedErrors: string[]; timeout?: number; idSyncFile?: string; + coverage: boolean; // Enables source code coverage report generation. + coverageDirectory: string; + coverageReporters: reports.ReportType[]; } interface FuzzModule { @@ -61,23 +67,24 @@ interface FuzzModule { declare global { var Fuzzer: fuzzer.Fuzzer; var HookManager: hooking.HookManager; + var __coverage__: libCoverage.CoverageMapData; } export async function initFuzzing(options: Options) { registerGlobals(); - await Promise.all(options.customHooks.map(importModule)); - if (!options.dryRun) { - registerInstrumentor( - new Instrumentor( - options.includes, - options.excludes, - - options.idSyncFile !== undefined - ? new FileSyncIdStrategy(options.idSyncFile) - : new MemorySyncIdStrategy() - ) - ); - } + registerInstrumentor( + new Instrumentor( + options.includes, + options.excludes, + options.customHooks, + options.coverage, + options.dryRun, + options.idSyncFile !== undefined + ? new FileSyncIdStrategy(options.idSyncFile) + : new MemorySyncIdStrategy() + ) + ); + await Promise.all(options.customHooks.map(ensureFilepath).map(importModule)); } export function registerGlobals() { @@ -91,10 +98,20 @@ export async function startFuzzing(options: Options) { const fuzzFn = await loadFuzzFunction(options); await startFuzzingNoInit(fuzzFn, options).then( () => { - stopFuzzing(undefined, options.expectedErrors); + stopFuzzing( + undefined, + options.expectedErrors, + options.coverageDirectory, + options.coverageReporters + ); }, (err: unknown) => { - stopFuzzing(err, options.expectedErrors); + stopFuzzing( + err, + options.expectedErrors, + options.coverageDirectory, + options.coverageReporters + ); } ); } @@ -165,10 +182,28 @@ ${jazzerArgs.map((s) => '"' + s + '"').join(" ")} -- ${isWindows ? "%*" : "$@"} return scriptTempFile.name; } -function stopFuzzing(err: unknown, expectedErrors: string[]) { +function stopFuzzing( + err: unknown, + expectedErrors: string[], + coverageDirectory: string, + coverageReporters: reports.ReportType[] +) { if (process.env.JAZZER_DEBUG) { trackedHooks.categorizeUnknown(HookManager.hooks).print(); } + // Generate a coverage report in fuzzing mode (non-jest). The coverage report for our jest-runner is generated + // by jest internally (as long as '--coverage' is set). + if (global.__coverage__) { + const coverageMap = libCoverage.createCoverageMap(global.__coverage__); + const context = libReport.createContext({ + dir: coverageDirectory, + watermarks: {}, + coverageMap: coverageMap, + }); + coverageReporters.forEach((reporter) => + reports.create(reporter).execute(context) + ); + } // No error found, check if one is expected. if (!err) { @@ -249,9 +284,10 @@ function buildFuzzerOptions(options: Options): string[] { if (!options || !options.fuzzerOptions) { return []; } - // Last occurrence of a parameter is used. + let opts = options.fuzzerOptions; if (options.dryRun) { + // the last provided option takes precedence opts = opts.concat("-runs=0"); } if (options.timeout != undefined) { diff --git a/packages/core/package.json b/packages/core/package.json index 4b7877e7..89632e80 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -22,6 +22,9 @@ "@jazzer.js/hooking": "*", "@jazzer.js/instrumentor": "*", "tmp": "^0.2.1", + "istanbul-lib-coverage": "^3.2.0", + "istanbul-lib-report": "^3.0.0", + "istanbul-reports": "^3.1.5", "yargs": "^17.6.2" }, "devDependencies": { diff --git a/packages/instrumentor/instrument.test.ts b/packages/instrumentor/instrument.test.ts index 7c3854a4..66fc131a 100644 --- a/packages/instrumentor/instrument.test.ts +++ b/packages/instrumentor/instrument.test.ts @@ -19,49 +19,54 @@ import { codeCoverage } from "./plugins/codeCoverage"; import { MemorySyncIdStrategy } from "./edgeIdStrategy"; import { Instrumentor } from "./instrument"; +import { sourceCodeCoverage } from "./plugins/sourceCodeCoverage"; describe("shouldInstrument check", () => { it("should consider includes and excludes", () => { const instrumentor = new Instrumentor(["include"], ["exclude"]); - expect(instrumentor.shouldInstrument("include")).toBeTruthy(); - expect(instrumentor.shouldInstrument("exclude")).toBeFalsy(); + expect(instrumentor.shouldInstrumentForFuzzing("include")).toBeTruthy(); + expect(instrumentor.shouldInstrumentForFuzzing("exclude")).toBeFalsy(); expect( - instrumentor.shouldInstrument("/some/package/include/files") + instrumentor.shouldInstrumentForFuzzing("/some/package/include/files") ).toBeTruthy(); expect( - instrumentor.shouldInstrument("/some/package/exclude/files") + instrumentor.shouldInstrumentForFuzzing("/some/package/exclude/files") + ).toBeFalsy(); + expect( + instrumentor.shouldInstrumentForFuzzing("/something/else") ).toBeFalsy(); - expect(instrumentor.shouldInstrument("/something/else")).toBeFalsy(); }); it("should include everything with *", () => { const instrumentor = new Instrumentor(["*"], []); - expect(instrumentor.shouldInstrument("include")).toBeTruthy(); - expect(instrumentor.shouldInstrument("/something/else")).toBeTruthy(); + expect(instrumentor.shouldInstrumentForFuzzing("include")).toBeTruthy(); + expect( + instrumentor.shouldInstrumentForFuzzing("/something/else") + ).toBeTruthy(); }); it("should include nothing with emtpy string", () => { const instrumentorWithEmptyInclude = new Instrumentor(["include", ""], []); expect( - instrumentorWithEmptyInclude.shouldInstrument("include") + instrumentorWithEmptyInclude.shouldInstrumentForFuzzing("include") ).toBeTruthy(); expect( - instrumentorWithEmptyInclude.shouldInstrument("/something/else") + instrumentorWithEmptyInclude.shouldInstrumentForFuzzing("/something/else") ).toBeFalsy(); const instrumentorWithEmptyExclude = new Instrumentor(["include"], [""]); expect( - instrumentorWithEmptyExclude.shouldInstrument("include") + instrumentorWithEmptyExclude.shouldInstrumentForFuzzing("include") ).toBeTruthy(); expect( - instrumentorWithEmptyExclude.shouldInstrument("/something/else") + instrumentorWithEmptyExclude.shouldInstrumentForFuzzing("/something/else") ).toBeFalsy(); }); it("should exclude with precedence", () => { const instrumentor = new Instrumentor(["include"], ["*"]); expect( - instrumentor.shouldInstrument("/some/package/include/files") + instrumentor.shouldInstrumentForFuzzing("/some/package/include/files") ).toBeFalsy(); }); }); @@ -102,6 +107,45 @@ describe("transform", () => { } }); }); + + describe("transform", () => { + it("should use source maps to correct error stack traces, also with enabled coverage", () => { + withSourceMap((instrumentor: Instrumentor) => { + const sourceFileName = "sourcemap-test002.js"; + const errorLocation = sourceFileName + ":5:13"; + const content = ` + module.exports.functionThrowingAnError = function foo () { + // eslint-disable-next-line no-constant-condition + if (1 < 2) { + throw Error("Expected test error"); // error thrown at ${errorLocation} + } + }; + // sourceURL is required for the snippet to reference a filename during + // eval and so be able to lookup the appropriate source map later on. + // This is only necessary for this test and not when using normal + // import/require without eval. + //@ sourceURL=${sourceFileName}`; + try { + // Use the codeCoverage plugin to add additional lines, so that the + // resulting error stack does not match the original code anymore. + const result = instrumentor.transform(sourceFileName, content, [ + sourceCodeCoverage(sourceFileName), + codeCoverage(new MemorySyncIdStrategy()), + ]); + const fn = eval(result?.code || ""); + fn(); + fail("Error expected but not thrown."); + } catch (e: unknown) { + if (!(e instanceof Error && e.stack)) { + throw e; + } + // Verify that the received error was corrected via a source map + // by checking the original error location. + expect(e.stack).toContain(errorLocation); + } + }); + }); + }); }); function withSourceMap(fn: (instrumentor: Instrumentor) => void) { @@ -111,6 +155,7 @@ function withSourceMap(fn: (instrumentor: Instrumentor) => void) { globalThis.Fuzzer = { // @ts-ignore coverageTracker: { + // eslint-disable-next-line @typescript-eslint/no-unused-vars incrementCounter: (edgeId: number) => { // ignore }, diff --git a/packages/instrumentor/instrument.ts b/packages/instrumentor/instrument.ts index cba8fc48..3fbe6c41 100644 --- a/packages/instrumentor/instrument.ts +++ b/packages/instrumentor/instrument.ts @@ -24,6 +24,7 @@ import { } from "@babel/core"; import { hookRequire, TransformerOptions } from "istanbul-lib-hook"; import { codeCoverage } from "./plugins/codeCoverage"; +import { sourceCodeCoverage } from "./plugins/sourceCodeCoverage"; import { compareHooks } from "./plugins/compareHooks"; import { functionHooks } from "./plugins/functionHooks"; import { hookManager } from "@jazzer.js/hooking"; @@ -45,6 +46,9 @@ export class Instrumentor { constructor( private readonly includes: string[] = ["*"], private readonly excludes: string[] = ["node_modules"], + private readonly customHooks: string[] = [], + private readonly shouldCollectSourceCodeCoverage = false, + private readonly isDryRun = false, private readonly idStrategy: EdgeIdStrategy = new MemorySyncIdStrategy() ) {} @@ -58,7 +62,7 @@ export class Instrumentor { instrument(code: string, filename: string): string { const transformations: PluginItem[] = []; - const shouldInstrumentFile = this.shouldInstrument(filename); + const shouldInstrumentFile = this.shouldInstrumentForFuzzing(filename); if (shouldInstrumentFile) { transformations.push(codeCoverage(this.idStrategy), compareHooks); @@ -68,6 +72,10 @@ export class Instrumentor { transformations.push(functionHooks(filename)); } + if (this.shouldCollectCodeCoverage(filename)) { + transformations.push(sourceCodeCoverage(filename)); + } + if (shouldInstrumentFile) { this.idStrategy.startForSourceFile(filename); } @@ -148,13 +156,28 @@ export class Instrumentor { delete require.cache[require.resolve(module)]; }); } - shouldInstrument(filepath: string): boolean { - const cleanup = (settings: string[]) => - settings - .filter((setting) => setting) - .map((setting) => (setting === "*" ? "" : setting)); // empty string matches every file - const cleanedIncludes = cleanup(this.includes); - const cleanedExcludes = cleanup(this.excludes); + shouldInstrumentForFuzzing(filepath: string): boolean { + return ( + !this.isDryRun && + Instrumentor.doesMatchFilters(filepath, this.includes, this.excludes) + ); + } + + private shouldCollectCodeCoverage(filepath: string): boolean { + return ( + this.shouldCollectSourceCodeCoverage && + (Instrumentor.doesMatchFilters(filepath, this.includes, this.excludes) || + Instrumentor.doesMatchFilters(filepath, this.customHooks, ["nothing"])) + ); + } + + private static doesMatchFilters( + filepath: string, + includes: string[], + excludes: string[] + ): boolean { + const cleanedIncludes = Instrumentor.cleanup(includes); + const cleanedExcludes = Instrumentor.cleanup(excludes); const included = cleanedIncludes.find((include) => filepath.includes(include)) !== undefined; @@ -163,6 +186,12 @@ export class Instrumentor { undefined; return included && !excluded; } + + private static cleanup(settings: string[]): string[] { + return settings + .filter((setting) => setting) + .map((setting) => (setting === "*" ? "" : setting)); // empty string matches every file + } } export function registerInstrumentor(instrumentor: Instrumentor) { diff --git a/packages/instrumentor/package.json b/packages/instrumentor/package.json index 1bfd84e9..b248af03 100644 --- a/packages/instrumentor/package.json +++ b/packages/instrumentor/package.json @@ -22,11 +22,13 @@ "@jazzer.js/hooking": "*", "istanbul-lib-hook": "^3.0.0", "proper-lockfile": "^4.1.2", + "istanbul-lib-instrument": "^5.2.1", "source-map-support": "^0.5.21" }, "devDependencies": { "@types/babel__core": "^7.20.0", "@types/istanbul-lib-hook": "^2.0.1", + "@types/istanbul-lib-instrument": "^1.7.4", "@types/node": "^18.11.18", "@types/proper-lockfile": "^4.1.2", "@types/source-map-support": "^0.5.6" diff --git a/packages/instrumentor/plugins/codeCoverage.test.ts b/packages/instrumentor/plugins/codeCoverage.test.ts index 45f5954f..47c0b7bc 100644 --- a/packages/instrumentor/plugins/codeCoverage.test.ts +++ b/packages/instrumentor/plugins/codeCoverage.test.ts @@ -199,6 +199,9 @@ describe("code coverage instrumentation", () => { const instrumentor = new Instrumentor( ["*"], ["do_not_instrument"], + [], + false, + false, new FileSyncIdStrategy(idSyncFile.name) ); diff --git a/packages/instrumentor/plugins/sourceCodeCoverage.test.ts b/packages/instrumentor/plugins/sourceCodeCoverage.test.ts new file mode 100644 index 00000000..1db2510b --- /dev/null +++ b/packages/instrumentor/plugins/sourceCodeCoverage.test.ts @@ -0,0 +1,209 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { codeCoverage } from "./codeCoverage"; +import { ZeroEdgeIdStrategy } from "../edgeIdStrategy"; +import { removeIndentation } from "./testhelpers"; +import { Instrumentor } from "../instrument"; +import * as libCoverage from "istanbul-lib-coverage"; +import { sourceCodeCoverage } from "./sourceCodeCoverage"; + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const fuzzer = require("@jazzer.js/fuzzer").fuzzer; +jest.mock("@jazzer.js/fuzzer"); +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +global.Fuzzer = fuzzer; + +/* eslint no-var: 0 */ +declare global { + var __coverage__: libCoverage.CoverageMapData; +} + +// Each test instruments, evaluates instrumented code, and checks the coverage data accumulated +// in the global variable __coverage__. +describe("Source code coverage instrumentation", () => { + it("No code, expect no coverage", () => { + const code = ``; + instrumentAndEval(code); + expect(getStatementMap()).toEqual({}); + expect(getFunctionMap()).toEqual({}); + expect(getBranchMap()).toEqual({}); + // Check coverage data + expect(getCoveredStatements()).toEqual({}); + expect(getCoveredFunctions()).toEqual({}); + expect(getCoveredBranches()).toEqual({}); + }); + it("Statements should be all covered", () => { + const code = ` + |let x; // this is not a statement + |x = 1; + |x++; + |x++; + `; + instrumentAndEval(code); + const statementMap = getStatementMap(); + expect(statementMap["0"]).toEqual(makeCoverageRange(2, 2, 0, 6)); + expect(statementMap["1"]).toEqual(makeCoverageRange(3, 3, 0, 4)); + expect(statementMap["2"]).toEqual(makeCoverageRange(4, 4, 0, 4)); + expect(Object.keys(statementMap).length).toEqual(3); + expect(getBranchMap()).toEqual({}); + expect(getFunctionMap()).toEqual({}); + // Check coverage data + expect(getCoveredStatements()).toEqual({ "0": 1, "1": 1, "2": 1 }); + expect(getCoveredFunctions()).toEqual({}); + expect(getCoveredBranches()).toEqual({}); + }); + it("Functions: foo covered twice; bar uncovered", () => { + const code = ` + |function foo() { + |return 1; + |} + |function bar() { + |return 1; + |} + |foo(); + |foo(); + `; + instrumentAndEval(code); + // Check positions of functions, statements, and branches + const statementMap = getStatementMap(); + expect(statementMap["0"]).toEqual(makeCoverageRange(2, 2, 0, 9)); + expect(statementMap["1"]).toEqual(makeCoverageRange(5, 5, 0, 9)); + expect(statementMap["2"]).toEqual(makeCoverageRange(7, 7, 0, 6)); + expect(statementMap["3"]).toEqual(makeCoverageRange(8, 8, 0, 6)); + expect(Object.keys(statementMap).length).toEqual(4); + const functionMap = getFunctionMap(); + expect(functionMap["0"]).toEqual( + makeFunctionMapping("foo", 1, 1, 9, 12, 1, 3, 15, 1, 1) + ); + expect(functionMap["1"]).toEqual( + makeFunctionMapping("bar", 4, 4, 9, 12, 4, 6, 15, 1, 4) + ); + expect(Object.keys(functionMap).length).toEqual(2); + expect(getBranchMap()).toEqual({}); // program has no branches + // Check coverage data + expect(getCoveredStatements()).toEqual({ "0": 2, "1": 0, "2": 1, "3": 1 }); + expect(getCoveredFunctions()).toEqual({ "0": 2, "1": 0 }); + expect(getCoveredBranches()).toEqual({}); + }); + it("Branches coverage", () => { + const code = ` + |let x; + |if (true) { + |x++; + |} else { + |x--; + |} + `; + instrumentAndEval(code); + const statementMap = getStatementMap(); + expect(statementMap["0"]).toEqual(makeCoverageRange(2, 6, 0, 1)); + expect(statementMap["1"]).toEqual(makeCoverageRange(3, 3, 0, 4)); + expect(statementMap["2"]).toEqual(makeCoverageRange(5, 5, 0, 4)); + expect(Object.keys(statementMap).length).toEqual(3); + const branchMap = getBranchMap(); + expect(branchMap["0"].loc).toEqual(makeCoverageRange(2, 6, 0, 1)); + expect(branchMap["0"].locations[0]).toEqual(makeCoverageRange(2, 6, 0, 1)); + expect(branchMap["0"].locations[1]).toEqual(makeCoverageRange(4, 6, 7, 1)); // else + expect(Object.keys(branchMap).length).toEqual(1); + expect(getFunctionMap()).toEqual({}); + // Check coverage data + expect(getCoveredStatements()).toEqual({ "0": 1, "1": 1, "2": 0 }); + expect(getCoveredFunctions()).toEqual({}); + expect(getCoveredBranches()).toEqual({ "0": [1, 0] }); + }); +}); + +const mockFilename = "testfile.js"; + +function getStatementMap() { + return global.__coverage__[mockFilename].statementMap; +} + +function getFunctionMap() { + return global.__coverage__[mockFilename].fnMap; +} + +function getBranchMap() { + return global.__coverage__[mockFilename].branchMap; +} + +function getCoveredStatements() { + return global.__coverage__[mockFilename].s; +} + +function getCoveredFunctions() { + return global.__coverage__[mockFilename].f; +} + +function getCoveredBranches() { + return global.__coverage__[mockFilename].b; +} + +function instrumentAndEval(input: string) { + const code = removeIndentation(input); + const instrumentor = new Instrumentor(); + const plugins = [ + sourceCodeCoverage(mockFilename), + codeCoverage(new ZeroEdgeIdStrategy()), + ]; + const instrumented = + instrumentor.transform(mockFilename, code, plugins)?.code || code; + eval(instrumented); +} + +function makeCoverageRange( + startLine: number, + endLine: number, + startColumn: number, + endColumn: number +): libCoverage.Range { + return { + start: { line: startLine, column: startColumn }, + end: { line: endLine, column: endColumn }, + }; +} + +function makeFunctionMapping( + name: string, + declStartLine: number, + declEndLine: number, + declStartColumn: number, + declEndColumn: number, + locStartLine: number, + locEndLine: number, + locStartColumn: number, + locEndColumn: number, + line: number +): libCoverage.FunctionMapping { + return { + name: name, + decl: makeCoverageRange( + declStartLine, + declEndLine, + declStartColumn, + declEndColumn + ), + loc: makeCoverageRange( + locStartLine, + locEndLine, + locStartColumn, + locEndColumn + ), + line: line, + }; +} diff --git a/packages/instrumentor/plugins/sourceCodeCoverage.ts b/packages/instrumentor/plugins/sourceCodeCoverage.ts new file mode 100644 index 00000000..29b87c5a --- /dev/null +++ b/packages/instrumentor/plugins/sourceCodeCoverage.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PluginTarget } from "@babel/core"; +import { programVisitor, VisitorOptions } from "istanbul-lib-instrument"; + +export function sourceCodeCoverage( + filename?: string, + opts: Partial = {} +): PluginTarget { + return ({ types }) => { + const ee = programVisitor(types, filename, opts); + return { + visitor: { + Program: { + enter: ee.enter, + exit(path: string) { + ee.exit(path); + }, + }, + }, + }; + }; +} diff --git a/packages/instrumentor/plugins/testhelpers.ts b/packages/instrumentor/plugins/testhelpers.ts index 63601c84..2228da4c 100644 --- a/packages/instrumentor/plugins/testhelpers.ts +++ b/packages/instrumentor/plugins/testhelpers.ts @@ -40,6 +40,6 @@ function expectInstrumentation( return result; } -function removeIndentation(text?: string | null): string { +export function removeIndentation(text?: string | null): string { return text ? text.replace(/^\s*\|/gm, "").replace(/^\s*[\n\r]+/gm, "") : ""; } diff --git a/packages/jest-runner/config.ts b/packages/jest-runner/config.ts index 56d8be8e..cd685c37 100644 --- a/packages/jest-runner/config.ts +++ b/packages/jest-runner/config.ts @@ -28,6 +28,9 @@ export const defaultOptions: Options = { sync: false, expectedErrors: [], timeout: 5000, // default Jest timeout + coverage: false, + coverageDirectory: "coverage", + coverageReporters: ["json", "text", "lcov", "clover"], // default Jest reporters }; // Looks up Jazzer.js options via the `jazzer-runner` configuration from diff --git a/packages/jest-runner/index.ts b/packages/jest-runner/index.ts index b66a4fb4..3132b130 100644 --- a/packages/jest-runner/index.ts +++ b/packages/jest-runner/index.ts @@ -30,10 +30,15 @@ import { JazzerWorker } from "./worker"; import { registerGlobals, initFuzzing } from "@jazzer.js/core"; import { loadConfig } from "./config"; import { cleanupJestRunnerStack } from "./errorUtils"; +import * as reports from "istanbul-reports"; class FuzzRunner extends CallbackTestRunner { + shouldCollectCoverage: boolean; + coverageReporters: Config.CoverageReporters; constructor(globalConfig: Config.GlobalConfig, context: TestRunnerContext) { super(globalConfig, context); + this.shouldCollectCoverage = globalConfig.collectCoverage; + this.coverageReporters = globalConfig.coverageReporters; registerGlobals(); } @@ -46,7 +51,10 @@ class FuzzRunner extends CallbackTestRunner { options: TestRunnerOptions // eslint-disable-line @typescript-eslint/no-unused-vars ): Promise { const config = loadConfig(); - initFuzzing(config); + config.coverage = this.shouldCollectCoverage; + config.coverageReporters = this.coverageReporters as reports.ReportType[]; + + await initFuzzing(config); return this.#runTestsInBand(tests, watcher, onStart, onResult, onFailure); } diff --git a/packages/jest-runner/package.json b/packages/jest-runner/package.json index f110f7c3..ee61b733 100644 --- a/packages/jest-runner/package.json +++ b/packages/jest-runner/package.json @@ -18,9 +18,11 @@ "dependencies": { "@jazzer.js/core": "*", "cosmiconfig": "^8.0.0", - "jest": "^29.4.0" + "jest": "^29.4.0", + "istanbul-reports": "^3.1.5" }, "devDependencies": { + "@types/istanbul-reports": "^3.0.1", "@types/tmp": "^0.2.3", "tmp": "^0.2.1" }, diff --git a/packages/jest-runner/worker.ts b/packages/jest-runner/worker.ts index 123a4d08..43562556 100644 --- a/packages/jest-runner/worker.ts +++ b/packages/jest-runner/worker.ts @@ -370,7 +370,7 @@ export class JazzerWorker { const runtime = this.#testSummary.end - this.#testSummary.start; return { - // coverage: globalThis.__coverage__, + coverage: globalThis.__coverage__, console: undefined, failureMessage: this.#testResults .filter((t) => t.errors.length > 0) diff --git a/scripts/run_all.bat b/scripts/run_all.bat index f71c95ad..be1ddf1d 100644 --- a/scripts/run_all.bat +++ b/scripts/run_all.bat @@ -10,7 +10,7 @@ FOR /D %%G in ("*") DO ( cd %%G IF EXIST "package.json" ( npm install - npm run "%command%" + npm run "%command%" || cmd /c exit -1073741510 ) cd .. ) diff --git a/tests/FuzzedDataProvider/fuzz.js b/tests/FuzzedDataProvider/fuzz.js index f3e47355..9595da7f 100644 --- a/tests/FuzzedDataProvider/fuzz.js +++ b/tests/FuzzedDataProvider/fuzz.js @@ -22,11 +22,10 @@ const { FuzzedDataProvider } = require("@jazzer.js/core"); */ module.exports.fuzz = function (fuzzerInputData) { const data = new FuzzedDataProvider(fuzzerInputData); - const s1 = data.consumeString(data.consumeIntegralInRange(1, 15), "utf-8"); + const s1 = data.consumeString(data.consumeIntegralInRange(10, 15), "utf-8"); const i1 = data.consumeIntegral(1); const i2 = data.consumeIntegral(2); - let i3 = 0; - if (data.consumeBoolean()) i3 = data.consumeIntegral(4); + let i3 = data.consumeIntegral(4); if (i3 === 1000) { if (s1 === "Hello World!") { diff --git a/tests/FuzzedDataProvider/package.json b/tests/FuzzedDataProvider/package.json index d51bbf1f..df1af150 100644 --- a/tests/FuzzedDataProvider/package.json +++ b/tests/FuzzedDataProvider/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "description": "An example showing how to use FuzzedDataProvider in Jazzer.js", "scripts": { - "fuzz": "jazzer fuzz --sync -x Error -i jazzer.js -- -use_value_profile=1 -print_pcs=1 -print_final_stats=1 -max_len=52 -max_total_time=180", + "fuzz": "jazzer fuzz --sync -x Error -i jazzer.js -- -use_value_profile=1 -print_pcs=1 -print_final_stats=1 -max_len=52 -max_total_time=180 -seed=123", "dryRun": "jazzer fuzz -d --sync -- -runs=100 -seed=123456789" }, "devDependencies": { diff --git a/tests/code_coverage/coverage.test.js b/tests/code_coverage/coverage.test.js new file mode 100644 index 00000000..919eeb00 --- /dev/null +++ b/tests/code_coverage/coverage.test.js @@ -0,0 +1,314 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint no-undef: 0 */ +// eslint-disable-next-line @typescript-eslint/no-var-requires +const fs = require("fs"); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { spawnSync } = require("child_process"); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const path = require("path"); + +// current working directory +const testDirectory = path.join(process.cwd(), "sample_fuzz_test"); +const defaultCoverageDirectory = path.join(testDirectory, "coverage"); +const expectedCoverageDirectory = path.join(testDirectory, "expected_coverage"); +const libFile = path.join(testDirectory, "lib.js"); +const targetFile = path.join(testDirectory, "fuzz.js"); +const jestRunnerFile = path.join(testDirectory, "codeCoverage.fuzz.js"); +const hookFile = path.join(testDirectory, "custom-hooks.js"); + +describe("Source code coverage reports for regular fuzz targets", () => { + it("Expect no coverage reports", () => { + executeFuzzTest(false, false, false, false, false); + expect(fs.existsSync(defaultCoverageDirectory)).toBe(false); + }); + it("Want coverage, but no includes active. Expect no coverage reports", () => { + executeFuzzTest(false, false, false, false, true); + expect(fs.existsSync(defaultCoverageDirectory)).toBe(false); + }); + it("Want coverage in dry run mode, no custom hooks", () => { + executeFuzzTest(true, true, true, false, true); + expect(fs.existsSync(defaultCoverageDirectory)).toBe(true); + const coverageJson = readCoverageJson(defaultCoverageDirectory); + const expectedCoverage = readExpectedCoverage("fuzz+lib.json"); + expect(coverageJson).toBeTruthy(); + // lib.js + expect(coverageJson[libFile]).toBeTruthy(); + expectEqualCoverage(coverageJson[libFile], expectedCoverage["lib.js"]); + // fuzz.js + expect(coverageJson[targetFile]).toBeTruthy(); + expectEqualCoverage(coverageJson[targetFile], expectedCoverage["fuzz.js"]); + // custom-hooks.js + expect(coverageJson[hookFile]).toBeFalsy(); + }); + + it("Want coverage in dry run mode, with custom hooks", () => { + executeFuzzTest(true, true, true, true, true); + expect(fs.existsSync(defaultCoverageDirectory)).toBe(true); + const coverageJson = readCoverageJson(defaultCoverageDirectory); + const expectedCoverage = readExpectedCoverage("fuzz+lib+customHooks.json"); + expect(coverageJson).toBeTruthy(); + // lib.js + expect(coverageJson[libFile]).toBeTruthy(); + expectEqualCoverage(coverageJson[libFile], expectedCoverage["lib.js"]); + // fuzz.js + expect(coverageJson[targetFile]).toBeTruthy(); + expectEqualCoverage(coverageJson[targetFile], expectedCoverage["fuzz.js"]); + // custom-hooks.js + // work in dry run mode + expect(coverageJson[hookFile]).toBeTruthy(); + expectEqualCoverage( + coverageJson[hookFile], + expectedCoverage["custom-hooks.js"] + ); + }); + + it("Want coverage, instrumentation enabled, with custom hooks", () => { + executeFuzzTest(false, true, true, true, true); + expect(fs.existsSync(defaultCoverageDirectory)).toBe(true); + const coverageJson = readCoverageJson(defaultCoverageDirectory); + const expectedCoverage = readExpectedCoverage("fuzz+lib+customHooks.json"); + expect(coverageJson).toBeTruthy(); + // lib.js + expect(coverageJson[libFile]).toBeTruthy(); + expectEqualCoverage(coverageJson[libFile], expectedCoverage["lib.js"]); + // fuzz.js + expect(coverageJson[targetFile]).toBeTruthy(); + expectEqualCoverage(coverageJson[targetFile], expectedCoverage["fuzz.js"]); + // custom-hooks.js + // work in dry run mode + expect(coverageJson[hookFile]).toBeTruthy(); + expectEqualCoverage( + coverageJson[hookFile], + expectedCoverage["custom-hooks.js"] + ); + }); + + it("Want coverage in a non-default directory, instrumentation enabled, with custom hooks", () => { + const coverageDirectory = "coverage002"; + const coverageAbsoluteDirectory = path.join( + testDirectory, + coverageDirectory + ); + executeFuzzTest(false, true, true, true, true, coverageDirectory); + expect(fs.existsSync(coverageAbsoluteDirectory)).toBe(true); + const coverageJson = readCoverageJson(coverageAbsoluteDirectory); + const expectedCoverage = readExpectedCoverage("fuzz+lib+customHooks.json"); + expect(coverageJson).toBeTruthy(); + // lib.js + expect(coverageJson[libFile]).toBeTruthy(); + expectEqualCoverage(coverageJson[libFile], expectedCoverage["lib.js"]); + // fuzz.js + expect(coverageJson[targetFile]).toBeTruthy(); + expectEqualCoverage(coverageJson[targetFile], expectedCoverage["fuzz.js"]); + // custom-hooks.js + // work in dry run mode + expect(coverageJson[hookFile]).toBeTruthy(); + expectEqualCoverage( + coverageJson[hookFile], + expectedCoverage["custom-hooks.js"] + ); + }); +}); + +describe("Source code coverage reports for our custom Jest runner", () => { + it("Jest runner: Expect no coverage reports", () => { + const coverageDirectory = defaultCoverageDirectory; + executeJestRunner(false, false, false, true); + expect(fs.existsSync(coverageDirectory)).toBe(true); + const coverageJson = readCoverageJson(coverageDirectory); + // Jest generates an empty coverage report (unlike our non-jest fuzzer) + expect(coverageJson).toBeTruthy(); + expect(coverageJson).toStrictEqual({}); + expect(coverageJson[targetFile]).toBeFalsy(); + expect(coverageJson[targetFile]).toBeFalsy(); + expect(coverageJson[hookFile]).toBeFalsy(); + }); + + it("Jest runner: want coverage, no custom hooks", () => { + const coverageDirectory = defaultCoverageDirectory; + executeJestRunner(true, true, false, true); + expect(fs.existsSync(coverageDirectory)).toBe(true); + const coverageJson = readCoverageJson(coverageDirectory); + const expectedCoverage = readExpectedCoverage( + "fuzz+lib+codeCoverage-fuzz.json" + ); + expect(coverageJson).toBeTruthy(); + // lib.js + expect(coverageJson[libFile]).toBeTruthy(); + expectEqualCoverage(coverageJson[libFile], expectedCoverage["lib.js"]); + // fuzz.js + expect(coverageJson[targetFile]).toBeTruthy(); + expectEqualCoverage(coverageJson[targetFile], expectedCoverage["fuzz.js"]); + // codeCoverage.fuzz.js (the main fuzz test) + expect(coverageJson[targetFile]).toBeTruthy(); + expectEqualCoverage( + coverageJson[jestRunnerFile], + expectedCoverage["codeCoverage.fuzz.js"] + ); + // custom-hooks.js + expect(coverageJson[hookFile]).toBeFalsy(); + }); + + it("Jest runner: want coverage, with custom hooks", () => { + const coverageDirectory = defaultCoverageDirectory; + executeJestRunner(true, true, false, true); + expect(fs.existsSync(coverageDirectory)).toBe(true); + const coverageJson = readCoverageJson(coverageDirectory); + const expectedCoverage = readExpectedCoverage( + "fuzz+lib+codeCoverage-fuzz.json" + ); + expect(coverageJson).toBeTruthy(); + // lib.js + expect(coverageJson[libFile]).toBeTruthy(); + expectEqualCoverage(coverageJson[libFile], expectedCoverage["lib.js"]); + // fuzz.js + expect(coverageJson[targetFile]).toBeTruthy(); + expectEqualCoverage(coverageJson[targetFile], expectedCoverage["fuzz.js"]); + // codeCoverage.fuzz.js (the main fuzz test) + expect(coverageJson[targetFile]).toBeTruthy(); + expectEqualCoverage( + coverageJson[jestRunnerFile], + expectedCoverage["codeCoverage.fuzz.js"] + ); + // custom-hooks.js + expect(coverageJson[hookFile]).toBeFalsy(); + }); +}); + +function readCoverageJson(coverageDirectory) { + return JSON.parse( + fs.readFileSync(path.join(coverageDirectory, "coverage-final.json")) + ); +} + +function readExpectedCoverage(name) { + return JSON.parse( + fs.readFileSync(path.join(expectedCoverageDirectory, name)) + ); +} + +function expectEqualCoverage(coverage, expectedCoverage) { + expect(coverage.statementMap).toStrictEqual(expectedCoverage.statementMap); + expect(coverage.s).toStrictEqual(expectedCoverage.s); + expect(coverage.fnMap).toStrictEqual(expectedCoverage.fnMap); + expect(coverage.f).toStrictEqual(expectedCoverage.f); + expect(coverage.branchMap).toStrictEqual(expectedCoverage.branchMap); + expect(coverage.b).toStrictEqual(expectedCoverage.b); +} + +function executeJestRunner( + includeLib, + includeTarget, + useCustomHooks, + coverage, + coverageOutputDir = "coverage", + excludePattern = ["nothing"], + verbose = false +) { + try { + // remove the coverage folder if it exists + fs.rmSync(path.join(testDirectory, coverageOutputDir), { + recursive: true, + force: true, + }); + } catch (err) { + // ignore + } + + const includes = []; + if (includeLib) includes.push("lib.js"); + if (includeTarget) includes.push("fuzz.js"); + if (!includeLib && !includeTarget) includes.push("nothing"); + + const config = { + includes: includes, + excludes: excludePattern, + fuzzerOptions: [], + customHooks: useCustomHooks ? ["custom-hooks.js"] : [], + }; + // write the config file, overwriting any existing one + fs.writeFileSync( + path.join(testDirectory, ".jazzerjsrc.json"), + JSON.stringify(config) + ); + + let command = ["jest", "--coverage"]; + const process = spawnSync("npx", command, { + stdio: "pipe", + cwd: testDirectory, + shell: true, + }); + if (verbose) console.log(process.output.toString()); +} + +function executeFuzzTest( + dryRun, + includeLib, + includeTarget, + useCustomHooks, + coverage, + coverageOutputDir = "coverage", + excludePattern = "nothing", + verbose = false +) { + try { + // remove the coverage folder if it exists + fs.rmSync(path.join(testDirectory, coverageOutputDir), { + recursive: true, + force: true, + }); + } catch (err) { + // ignore + } + let options = ["jazzer", "fuzz", "-e", excludePattern, "--corpus", "corpus"]; + // add dry run option + if (dryRun) options.push("-d"); + if (includeLib) { + options.push("-i"); + options.push("lib.js"); + } + if (includeTarget) { + options.push("-i"); + options.push("fuzz.js"); + } + if (!includeLib && !includeTarget) { + options.push("-i"); + options.push("nothing"); + } + + if (useCustomHooks) { + options.push("-h"); + options.push("custom-hooks"); + } + if (coverage) { + options.push("--coverage"); + } + if (coverageOutputDir) { + options.push("--coverageDirectory"); + options.push(coverageOutputDir); + } + options.push("--"); + options.push("-runs=0"); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const process = spawnSync("npx", options, { + stdio: "pipe", + cwd: testDirectory, + shell: true, + }); + if (verbose) console.log(process.output.toString()); +} diff --git a/tests/code_coverage/package.json b/tests/code_coverage/package.json new file mode 100644 index 00000000..5b7613aa --- /dev/null +++ b/tests/code_coverage/package.json @@ -0,0 +1,11 @@ +{ + "name": "jazzerjs-source-code-coverage-tests", + "version": "1.0.0", + "description": "Unit tests for source code coverage.", + "scripts": { + "fuzz": "jest" + }, + "devDependencies": { + "@jazzer.js/core": "file:../../packages/core" + } +} diff --git a/tests/code_coverage/sample_fuzz_test/codeCoverage.fuzz.js b/tests/code_coverage/sample_fuzz_test/codeCoverage.fuzz.js new file mode 100644 index 00000000..27c8af9a --- /dev/null +++ b/tests/code_coverage/sample_fuzz_test/codeCoverage.fuzz.js @@ -0,0 +1,26 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint no-undef: 0, no-constant-condition: 0, @typescript-eslint/no-var-requires:0 */ + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const target = require("./fuzz.js"); + +describe("My describe", () => { + it.fuzz("My fuzz test", (data) => { + target.fuzz(data); + }); +}); diff --git a/tests/code_coverage/sample_fuzz_test/codeCoverage.fuzz/My_describe/My_fuzz_test/a b/tests/code_coverage/sample_fuzz_test/codeCoverage.fuzz/My_describe/My_fuzz_test/a new file mode 100644 index 00000000..2e65efe2 --- /dev/null +++ b/tests/code_coverage/sample_fuzz_test/codeCoverage.fuzz/My_describe/My_fuzz_test/a @@ -0,0 +1 @@ +a \ No newline at end of file diff --git a/tests/code_coverage/sample_fuzz_test/codeCoverage.fuzz/My_describe/My_fuzz_test/b b/tests/code_coverage/sample_fuzz_test/codeCoverage.fuzz/My_describe/My_fuzz_test/b new file mode 100644 index 00000000..bb5d25b4 --- /dev/null +++ b/tests/code_coverage/sample_fuzz_test/codeCoverage.fuzz/My_describe/My_fuzz_test/b @@ -0,0 +1 @@ +bbbbbbbbbbbbbbbbbbbbbbbbbbb \ No newline at end of file diff --git a/tests/code_coverage/sample_fuzz_test/corpus/a b/tests/code_coverage/sample_fuzz_test/corpus/a new file mode 100644 index 00000000..2e65efe2 --- /dev/null +++ b/tests/code_coverage/sample_fuzz_test/corpus/a @@ -0,0 +1 @@ +a \ No newline at end of file diff --git a/tests/code_coverage/sample_fuzz_test/corpus/b b/tests/code_coverage/sample_fuzz_test/corpus/b new file mode 100644 index 00000000..bb5d25b4 --- /dev/null +++ b/tests/code_coverage/sample_fuzz_test/corpus/b @@ -0,0 +1 @@ +bbbbbbbbbbbbbbbbbbbbbbbbbbb \ No newline at end of file diff --git a/tests/code_coverage/sample_fuzz_test/custom-hooks.js b/tests/code_coverage/sample_fuzz_test/custom-hooks.js new file mode 100644 index 00000000..cbe95b97 --- /dev/null +++ b/tests/code_coverage/sample_fuzz_test/custom-hooks.js @@ -0,0 +1,7 @@ +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { registerReplaceHook } = require("@jazzer.js/hooking"); + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +registerReplaceHook("foo", "lib", false, (thisPtr, params, hookId, origFn) => { + console.log("CUSTOM HOOKS CALLED!"); +}); diff --git a/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+codeCoverage-fuzz.json b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+codeCoverage-fuzz.json new file mode 100644 index 00000000..0ff0bf1b --- /dev/null +++ b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+codeCoverage-fuzz.json @@ -0,0 +1,178 @@ +{ + "codeCoverage.fuzz.js": { + "statementMap": { + "0": { + "start": { "line": 20, "column": 15 }, + "end": { "line": 20, "column": 35 } + }, + "1": { + "start": { "line": 22, "column": 0 }, + "end": { "line": 26, "column": 3 } + }, + "2": { + "start": { "line": 23, "column": 1 }, + "end": { "line": 25, "column": 4 } + }, + "3": { + "start": { "line": 24, "column": 2 }, + "end": { "line": 24, "column": 20 } + } + }, + "fnMap": { + "0": { + "name": "(anonymous_0)", + "decl": { + "start": { "line": 22, "column": 24 }, + "end": { "line": 22, "column": 25 } + }, + "loc": { + "start": { "line": 22, "column": 30 }, + "end": { "line": 26, "column": 1 } + }, + "line": 22 + }, + "1": { + "name": "(anonymous_1)", + "decl": { + "start": { "line": 23, "column": 25 }, + "end": { "line": 23, "column": 26 } + }, + "loc": { + "start": { "line": 23, "column": 35 }, + "end": { "line": 25, "column": 2 } + }, + "line": 23 + } + }, + "branchMap": {}, + "s": { "0": 1, "1": 1, "2": 1, "3": 2 }, + "f": { "0": 1, "1": 2 }, + "b": {}, + "_coverageSchema": "1a1c01bbd47fc00a2c39e90264f33305004495a9", + "hash": "25c01ffa3552de53ae353b8e557c73461d766e51" + }, + "fuzz.js": { + "statementMap": { + "0": { + "start": { "line": 18, "column": 12 }, + "end": { "line": 18, "column": 28 } + }, + "1": { + "start": { "line": 23, "column": 0 }, + "end": { "line": 29, "column": 2 } + }, + "2": { + "start": { "line": 24, "column": 1 }, + "end": { "line": 24, "column": 41 } + }, + "3": { + "start": { "line": 25, "column": 1 }, + "end": { "line": 27, "column": 2 } + }, + "4": { + "start": { "line": 26, "column": 2 }, + "end": { "line": 26, "column": 9 } + }, + "5": { + "start": { "line": 28, "column": 1 }, + "end": { "line": 28, "column": 18 } + } + }, + "fnMap": { + "0": { + "name": "(anonymous_0)", + "decl": { + "start": { "line": 23, "column": 22 }, + "end": { "line": 23, "column": 23 } + }, + "loc": { + "start": { "line": 23, "column": 38 }, + "end": { "line": 29, "column": 1 } + }, + "line": 23 + } + }, + "branchMap": { + "0": { + "loc": { + "start": { "line": 25, "column": 1 }, + "end": { "line": 27, "column": 2 } + }, + "type": "if", + "locations": [ + { + "start": { "line": 25, "column": 1 }, + "end": { "line": 27, "column": 2 } + }, + { "start": {}, "end": {} } + ], + "line": 25 + } + }, + "s": { "0": 1, "1": 1, "2": 2, "3": 2, "4": 1, "5": 1 }, + "f": { "0": 2 }, + "b": { "0": [1, 1] }, + "_coverageSchema": "1a1c01bbd47fc00a2c39e90264f33305004495a9", + "hash": "d5b411e8de7efcd2798a7cd78efd7bd8347f647a" + }, + "lib.js": { + "statementMap": { + "0": { + "start": { "line": 2, "column": 1 }, + "end": { "line": 2, "column": 29 } + }, + "1": { + "start": { "line": 3, "column": 1 }, + "end": { "line": 5, "column": 2 } + }, + "2": { + "start": { "line": 4, "column": 2 }, + "end": { "line": 4, "column": 11 } + }, + "3": { + "start": { "line": 6, "column": 1 }, + "end": { "line": 6, "column": 11 } + }, + "4": { + "start": { "line": 9, "column": 0 }, + "end": { "line": 11, "column": 2 } + } + }, + "fnMap": { + "0": { + "name": "foo", + "decl": { + "start": { "line": 1, "column": 9 }, + "end": { "line": 1, "column": 12 } + }, + "loc": { + "start": { "line": 1, "column": 16 }, + "end": { "line": 7, "column": 1 } + }, + "line": 1 + } + }, + "branchMap": { + "0": { + "loc": { + "start": { "line": 3, "column": 1 }, + "end": { "line": 5, "column": 2 } + }, + "type": "if", + "locations": [ + { + "start": { "line": 3, "column": 1 }, + "end": { "line": 5, "column": 2 } + }, + { "start": {}, "end": {} } + ], + "line": 3 + } + }, + "s": { "0": 1, "1": 1, "2": 1, "3": 0, "4": 1 }, + "f": { "0": 1 }, + "b": { "0": [1, 0] }, + "_coverageSchema": "1a1c01bbd47fc00a2c39e90264f33305004495a9", + "hash": "9606a42b6e4a5e9a5c23554dda63403d86a4a9a2" + } +} diff --git a/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+customHooks.json b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+customHooks.json new file mode 100644 index 00000000..6cdf89fd --- /dev/null +++ b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+customHooks.json @@ -0,0 +1,162 @@ +{ + "custom-hooks.js": { + "statementMap": { + "0": { + "start": { "line": 2, "column": 32 }, + "end": { "line": 2, "column": 61 } + }, + "1": { + "start": { "line": 5, "column": 0 }, + "end": { "line": 7, "column": 3 } + }, + "2": { + "start": { "line": 6, "column": 1 }, + "end": { "line": 6, "column": 37 } + } + }, + "fnMap": { + "0": { + "name": "(anonymous_0)", + "decl": { + "start": { "line": 5, "column": 41 }, + "end": { "line": 5, "column": 42 } + }, + "loc": { + "start": { "line": 5, "column": 78 }, + "end": { "line": 7, "column": 1 } + }, + "line": 5 + } + }, + "branchMap": {}, + "s": { "0": 1, "1": 1, "2": 1 }, + "f": { "0": 1 }, + "b": {}, + "_coverageSchema": "1a1c01bbd47fc00a2c39e90264f33305004495a9", + "hash": "7ee4d96bd07f3609d51855142c39faa2adf6ab0a" + }, + "fuzz.js": { + "statementMap": { + "0": { + "start": { "line": 18, "column": 12 }, + "end": { "line": 18, "column": 28 } + }, + "1": { + "start": { "line": 23, "column": 0 }, + "end": { "line": 29, "column": 2 } + }, + "2": { + "start": { "line": 24, "column": 1 }, + "end": { "line": 24, "column": 41 } + }, + "3": { + "start": { "line": 25, "column": 1 }, + "end": { "line": 27, "column": 2 } + }, + "4": { + "start": { "line": 26, "column": 2 }, + "end": { "line": 26, "column": 9 } + }, + "5": { + "start": { "line": 28, "column": 1 }, + "end": { "line": 28, "column": 18 } + } + }, + "fnMap": { + "0": { + "name": "(anonymous_0)", + "decl": { + "start": { "line": 23, "column": 22 }, + "end": { "line": 23, "column": 23 } + }, + "loc": { + "start": { "line": 23, "column": 38 }, + "end": { "line": 29, "column": 1 } + }, + "line": 23 + } + }, + "branchMap": { + "0": { + "loc": { + "start": { "line": 25, "column": 1 }, + "end": { "line": 27, "column": 2 } + }, + "type": "if", + "locations": [ + { + "start": { "line": 25, "column": 1 }, + "end": { "line": 27, "column": 2 } + }, + { "start": {}, "end": {} } + ], + "line": 25 + } + }, + "s": { "0": 1, "1": 1, "2": 3, "3": 3, "4": 2, "5": 1 }, + "f": { "0": 3 }, + "b": { "0": [2, 1] }, + "_coverageSchema": "1a1c01bbd47fc00a2c39e90264f33305004495a9", + "hash": "d5b411e8de7efcd2798a7cd78efd7bd8347f647a" + }, + "lib.js": { + "statementMap": { + "0": { + "start": { "line": 2, "column": 1 }, + "end": { "line": 2, "column": 29 } + }, + "1": { + "start": { "line": 3, "column": 1 }, + "end": { "line": 5, "column": 2 } + }, + "2": { + "start": { "line": 4, "column": 2 }, + "end": { "line": 4, "column": 11 } + }, + "3": { + "start": { "line": 6, "column": 1 }, + "end": { "line": 6, "column": 11 } + }, + "4": { + "start": { "line": 9, "column": 0 }, + "end": { "line": 11, "column": 2 } + } + }, + "fnMap": { + "0": { + "name": "foo", + "decl": { + "start": { "line": 1, "column": 9 }, + "end": { "line": 1, "column": 12 } + }, + "loc": { + "start": { "line": 1, "column": 16 }, + "end": { "line": 7, "column": 1 } + }, + "line": 1 + } + }, + "branchMap": { + "0": { + "loc": { + "start": { "line": 3, "column": 1 }, + "end": { "line": 5, "column": 2 } + }, + "type": "if", + "locations": [ + { + "start": { "line": 3, "column": 1 }, + "end": { "line": 5, "column": 2 } + }, + { "start": {}, "end": {} } + ], + "line": 3 + } + }, + "s": { "0": 0, "1": 0, "2": 0, "3": 0, "4": 1 }, + "f": { "0": 0 }, + "b": { "0": [0, 0] }, + "_coverageSchema": "1a1c01bbd47fc00a2c39e90264f33305004495a9", + "hash": "9606a42b6e4a5e9a5c23554dda63403d86a4a9a2" + } +} diff --git a/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib.json b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib.json new file mode 100644 index 00000000..8647d119 --- /dev/null +++ b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib.json @@ -0,0 +1,126 @@ +{ + "fuzz.js": { + "statementMap": { + "0": { + "start": { "line": 18, "column": 12 }, + "end": { "line": 18, "column": 28 } + }, + "1": { + "start": { "line": 23, "column": 0 }, + "end": { "line": 29, "column": 2 } + }, + "2": { + "start": { "line": 24, "column": 1 }, + "end": { "line": 24, "column": 41 } + }, + "3": { + "start": { "line": 25, "column": 1 }, + "end": { "line": 27, "column": 2 } + }, + "4": { + "start": { "line": 26, "column": 2 }, + "end": { "line": 26, "column": 9 } + }, + "5": { + "start": { "line": 28, "column": 1 }, + "end": { "line": 28, "column": 18 } + } + }, + "fnMap": { + "0": { + "name": "(anonymous_0)", + "decl": { + "start": { "line": 23, "column": 22 }, + "end": { "line": 23, "column": 23 } + }, + "loc": { + "start": { "line": 23, "column": 38 }, + "end": { "line": 29, "column": 1 } + }, + "line": 23 + } + }, + "branchMap": { + "0": { + "loc": { + "start": { "line": 25, "column": 1 }, + "end": { "line": 27, "column": 2 } + }, + "type": "if", + "locations": [ + { + "start": { "line": 25, "column": 1 }, + "end": { "line": 27, "column": 2 } + }, + { "start": {}, "end": {} } + ], + "line": 25 + } + }, + "s": { "0": 1, "1": 1, "2": 3, "3": 3, "4": 2, "5": 1 }, + "f": { "0": 3 }, + "b": { "0": [2, 1] }, + "_coverageSchema": "1a1c01bbd47fc00a2c39e90264f33305004495a9", + "hash": "d5b411e8de7efcd2798a7cd78efd7bd8347f647a" + }, + "lib.js": { + "statementMap": { + "0": { + "start": { "line": 2, "column": 1 }, + "end": { "line": 2, "column": 29 } + }, + "1": { + "start": { "line": 3, "column": 1 }, + "end": { "line": 5, "column": 2 } + }, + "2": { + "start": { "line": 4, "column": 2 }, + "end": { "line": 4, "column": 11 } + }, + "3": { + "start": { "line": 6, "column": 1 }, + "end": { "line": 6, "column": 11 } + }, + "4": { + "start": { "line": 9, "column": 0 }, + "end": { "line": 11, "column": 2 } + } + }, + "fnMap": { + "0": { + "name": "foo", + "decl": { + "start": { "line": 1, "column": 9 }, + "end": { "line": 1, "column": 12 } + }, + "loc": { + "start": { "line": 1, "column": 16 }, + "end": { "line": 7, "column": 1 } + }, + "line": 1 + } + }, + "branchMap": { + "0": { + "loc": { + "start": { "line": 3, "column": 1 }, + "end": { "line": 5, "column": 2 } + }, + "type": "if", + "locations": [ + { + "start": { "line": 3, "column": 1 }, + "end": { "line": 5, "column": 2 } + }, + { "start": {}, "end": {} } + ], + "line": 3 + } + }, + "s": { "0": 1, "1": 1, "2": 1, "3": 0, "4": 1 }, + "f": { "0": 1 }, + "b": { "0": [1, 0] }, + "_coverageSchema": "1a1c01bbd47fc00a2c39e90264f33305004495a9", + "hash": "9606a42b6e4a5e9a5c23554dda63403d86a4a9a2" + } +} diff --git a/tests/code_coverage/sample_fuzz_test/fuzz.js b/tests/code_coverage/sample_fuzz_test/fuzz.js new file mode 100644 index 00000000..8aa449f7 --- /dev/null +++ b/tests/code_coverage/sample_fuzz_test/fuzz.js @@ -0,0 +1,29 @@ +/* + * Copyright 2022 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const lib = require("./lib"); + +/** + * @param { Buffer } data + */ +module.exports.fuzz = function (data) { + console.log("DATA: " + data.toString()); + if (data.length < 3) { + return; + } + lib.foo(data[0]); +}; diff --git a/tests/code_coverage/sample_fuzz_test/lib.js b/tests/code_coverage/sample_fuzz_test/lib.js new file mode 100644 index 00000000..fed72f77 --- /dev/null +++ b/tests/code_coverage/sample_fuzz_test/lib.js @@ -0,0 +1,11 @@ +function foo(a) { + console.log("original foo"); + if (a > 10) { + return 5; + } + return 42; +} + +module.exports = { + foo, +}; diff --git a/tests/code_coverage/sample_fuzz_test/package.json b/tests/code_coverage/sample_fuzz_test/package.json new file mode 100644 index 00000000..5d2ffa3a --- /dev/null +++ b/tests/code_coverage/sample_fuzz_test/package.json @@ -0,0 +1,29 @@ +{ + "name": "Jazzer.js code coverage tests", + "version": "1.0.0", + "description": "", + "scripts": { + "test": "jest" + }, + "devDependencies": { + "@jazzer.js/core": "file:../../../packages/core", + "@jazzer.js/jest-runner": "file:../../../packages/jest-runner" + }, + "jest": { + "projects": [ + { + "displayName": "test" + }, + { + "runner": "@jazzer.js/jest-runner", + "displayName": { + "name": "Jazzer.js", + "color": "cyan" + }, + "testMatch": [ + "/**/*.fuzz.js" + ] + } + ] + } +}