diff --git a/.prettierignore b/.prettierignore index c4582fe5..7dd8e70c 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,4 +3,5 @@ build coverage node_modules .idea +.vscode compile_commands.json diff --git a/examples/jest_typescript_integration/.jazzerjsrc.json b/examples/jest_typescript_integration/.jazzerjsrc.json new file mode 100644 index 00000000..d18e831f --- /dev/null +++ b/examples/jest_typescript_integration/.jazzerjsrc.json @@ -0,0 +1,5 @@ +{ + "includes": ["target", "integration.fuzz"], + "excludes": ["node_modules"], + "fuzzerOptions": ["-rss_limit_mb=16000"] +} diff --git a/examples/jest_typescript_integration/README.md b/examples/jest_typescript_integration/README.md new file mode 100644 index 00000000..c98159f0 --- /dev/null +++ b/examples/jest_typescript_integration/README.md @@ -0,0 +1,68 @@ +# Jest Typscript Integration Example + +Detailed documentation on the Jest integration is available in the main +[Jazzer.js](https://github.com/CodeIntelligenceTesting/jazzer.js/blob/main/docs/jest-integration.md) +documentation. + +## Quickstart + +To use the [Jest](https://jestjs.io/) integration install the +`@jazzer.js/jest-runner` and `ts-jest` packages then configure `jest-runner` as +a dedicated test runner in `package.json`. + +The example below shows how to configure the Jazzer.js Jest integration in +combination with the normal Jest runner. + +```json + "jest": { + "projects": [ + { + "preset": "ts-jest", + "displayName": "tests", + "modulePathIgnorePatterns": ["dist"], + }, + { + "preset": "ts-jest", + "runner": "@jazzer.js/jest-runner", + "testEnvironment": "node", + "modulePathIgnorePatterns": [ + "dist", + "packages/fuzzer/build", + "tests/code_coverage", + ], + "transformIgnorePatterns": ["node_modules"], + "testMatch": ["/*.fuzz.[jt]s"], + "coveragePathIgnorePatterns": ["/node_modules/", "/dist/"], + }, + ], + "collectCoverageFrom": ["**/*.ts"], + } +``` + +Further configuration can be specified in `.jazzerjsrc.json` in the following +format: + +```json +{ + "includes": ["*"], + "excludes": ["node_modules"], + "customHooks": [], + "fuzzerOptions": [], + "sync": false +} +``` + +Write a fuzz test like: + +```typescript +// file: jazzerjs.fuzz.ts +import "@jazzer.js/jest-runner/jest-extension"; +describe("My describe", () => { + it.fuzz("My fuzz test", (data: Buffer) => { + target.fuzzMe(data); + }); +}); +``` + +**Note:** the `import` statement extends `jest`'s `It` interface to include the +`fuzz` property and is necessary for Typescript to compile the test file. diff --git a/examples/jest_typescript_integration/integration.fuzz.ts b/examples/jest_typescript_integration/integration.fuzz.ts new file mode 100644 index 00000000..7b6a9fce --- /dev/null +++ b/examples/jest_typescript_integration/integration.fuzz.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as target from "./target"; +import "@jazzer.js/jest-runner/jest-extension"; + +describe("fuzz testing for target", () => { + it.fuzz("My fuzz test", (data: Buffer) => { + target.fuzzMe(data); + }); + + it.fuzz( + "My fuzz test with an explicit timeout (async)", + async (data: Buffer) => { + target.fuzzMe(data); + }, + 1000 + ); + + it.fuzz( + "My fuzz test with an explicit timeout (sync)", + (data: Buffer) => { + target.fuzzMe(data); + }, + 1000 + ); + + it.fuzz("My callback fuzz test", (data: Buffer, done: () => void) => { + target.callbackFuzzMe(data, done); + }); + + it.fuzz("My async fuzz test", async (data: Buffer) => { + await target.asyncFuzzMe(data); + }); + + // In regression mode sync timeouts can not be detected, as the main event + // loop is blocked and registered timeout handlers can not fire. + // This is not only the case in regression test mode, but also during + // fuzzing runs. As the main event loop is blocked, no errors can be + // propagated to Jest. But the timeout set in libFuzzer will trigger a + // finding and shut down the whole process with exit code 70. + it.skip.fuzz("Sync timeout", () => { + // eslint-disable-next-line no-constant-condition + while (true) { + // Ignore + } + }); + + // Timeouts for async fuzz test functions can be detected in regression and + // fuzzing mode. libFuzzer shuts down the process after Jest received the + // error and displayed its result. + it.skip.fuzz("Async timeout", async () => { + return new Promise(() => { + // don't resolve promise + }); + }); + + // Timeouts for done callback fuzz test functions can be detected in + // regression and fuzzing mode. libFuzzer shuts down the process after Jest + // received the error and displayed its result. + // Two parameters are required to execute the done callback branch. + // eslint-disable-next-line @typescript-eslint/no-unused-vars + it.skip.fuzz("Done callback timeout", (ignore, ignore2) => { + // don't call done + }); +}); diff --git a/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/Async_timeout/seed b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/Async_timeout/seed new file mode 100644 index 00000000..e69de29b diff --git a/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/Done_callback_timeout/seed b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/Done_callback_timeout/seed new file mode 100644 index 00000000..e69de29b diff --git a/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_async_fuzz_test/three b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_async_fuzz_test/three new file mode 100644 index 00000000..2bdf67ab --- /dev/null +++ b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_async_fuzz_test/three @@ -0,0 +1 @@ +three diff --git a/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_callback_fuzz_test/two b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_callback_fuzz_test/two new file mode 100644 index 00000000..f719efd4 --- /dev/null +++ b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_callback_fuzz_test/two @@ -0,0 +1 @@ +two diff --git a/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_fuzz_test/one b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_fuzz_test/one new file mode 100644 index 00000000..5626abf0 --- /dev/null +++ b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_fuzz_test/one @@ -0,0 +1 @@ +one diff --git a/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_fuzz_test_with_an_explicit_timeout_(async)/test b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_fuzz_test_with_an_explicit_timeout_(async)/test new file mode 100644 index 00000000..30d74d25 --- /dev/null +++ b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_fuzz_test_with_an_explicit_timeout_(async)/test @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_fuzz_test_with_an_explicit_timeout_(sync)/test b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_fuzz_test_with_an_explicit_timeout_(sync)/test new file mode 100644 index 00000000..30d74d25 --- /dev/null +++ b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/My_fuzz_test_with_an_explicit_timeout_(sync)/test @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/Sync_timeout/seed b/examples/jest_typescript_integration/integration.fuzz/fuzz_testing_for_target/Sync_timeout/seed new file mode 100644 index 00000000..e69de29b diff --git a/examples/jest_typescript_integration/integration.test.ts b/examples/jest_typescript_integration/integration.test.ts new file mode 100644 index 00000000..3ad4519c --- /dev/null +++ b/examples/jest_typescript_integration/integration.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint no-undef: 0 */ + +import * as target from "./target"; + +describe("My describe", () => { + it("My normal Jest test", () => { + expect(1).toEqual(1); + }); + + it("My done callback Jest test", (done) => { + expect(1).toEqual(1); + done(); + }); + + it("My async Jest test", async () => { + expect(1).toEqual(1); + }); + + it("Test target function", () => { + const data = Buffer.from("a"); + target.fuzzMe(data); + }); +}); diff --git a/examples/jest_typescript_integration/jest.config.js b/examples/jest_typescript_integration/jest.config.js new file mode 100644 index 00000000..6661902f --- /dev/null +++ b/examples/jest_typescript_integration/jest.config.js @@ -0,0 +1,26 @@ +// can be uncommented to force fuzzing on which may be useful in e.g. vscode's jest UI to run fuzzing on a single test +// process.env.JAZZER_FUZZ = 1; +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + projects: [ + { + preset: "ts-jest", + displayName: "tests", + modulePathIgnorePatterns: ["dist"], + }, + { + preset: "ts-jest", + runner: "@jazzer.js/jest-runner", + testEnvironment: "node", + modulePathIgnorePatterns: [ + "dist", + "packages/fuzzer/build", + "tests/code_coverage", + ], + transformIgnorePatterns: ["node_modules"], + testMatch: ["/*.fuzz.[jt]s"], + coveragePathIgnorePatterns: ["/node_modules/", "/dist/"], + }, + ], + collectCoverageFrom: ["**/*.ts"], +}; diff --git a/examples/jest_typescript_integration/package.json b/examples/jest_typescript_integration/package.json new file mode 100644 index 00000000..723b005b --- /dev/null +++ b/examples/jest_typescript_integration/package.json @@ -0,0 +1,19 @@ +{ + "name": "jest_typescript_integration", + "version": "1.0.0", + "description": "An example showing how Jazzer.js integrates with Jest and TypeScript", + "scripts": { + "build": "tsc", + "dryRun": "jest", + "fuzz": "JAZZER_FUZZ=1 jest --coverage", + "coverage": "jest --coverage" + }, + "devDependencies": { + "@jazzer.js/jest-runner": "file:../../packages/jest-runner", + "@types/jest": "^29.4.0", + "jest": "^29.4.1", + "ts-jest": "^29.0.5", + "typescript": "^4.9.5" + }, + "dependencies": {} +} diff --git a/examples/jest_typescript_integration/target.ts b/examples/jest_typescript_integration/target.ts new file mode 100644 index 00000000..df3eede8 --- /dev/null +++ b/examples/jest_typescript_integration/target.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function fuzzMe(data: Buffer) { + const s = data.toString(); + if (s.length !== 16) { + return; + } + if ( + s.slice(0, 8) === "Awesome " && + s.slice(8, 15) === "Fuzzing" && + s[15] === "!" + ) { + throw Error("Welcome to Awesome Fuzzing!"); + } +} + +export function callbackFuzzMe(data: Buffer, done: () => void) { + // Use setImmediate here to unblock the event loop but still have better + // performance compared to setTimeout. + setImmediate(() => { + fuzzMe(data); + done(); + }); +} + +export async function asyncFuzzMe(data: Buffer) { + return callbackFuzzMe(data, () => { + // can't have empty functions + }); +} diff --git a/examples/jest_typescript_integration/tsconfig.json b/examples/jest_typescript_integration/tsconfig.json new file mode 100644 index 00000000..43cb0547 --- /dev/null +++ b/examples/jest_typescript_integration/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "node", + "allowJs": true, + //"checkJs": true, + "rootDir": ".", + "outDir": "./dist", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "declaration": true, + "composite": true, + "sourceMap": true + } +} diff --git a/packages/instrumentor/instrument.ts b/packages/instrumentor/instrument.ts index f1c13092..1007d6c2 100644 --- a/packages/instrumentor/instrument.ts +++ b/packages/instrumentor/instrument.ts @@ -206,6 +206,10 @@ export function registerInstrumentor(instrumentor: Instrumentor) { () => true, (code: string, opts: TransformerOptions): string => { return instrumentor.instrument(code, opts.filename); - } + }, + // required to allow jest to run typescript files + // jest's typescript integration will transform the typescript into javascript before giving it to the + // instrumentor but the filename will still have a .ts extension + { extensions: [".ts", ".js"] } ); } diff --git a/packages/jest-runner/jest-extension.ts b/packages/jest-runner/jest-extension.ts new file mode 100644 index 00000000..27756c9c --- /dev/null +++ b/packages/jest-runner/jest-extension.ts @@ -0,0 +1,10 @@ +import { FuzzTest } from "./fuzz"; + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace jest { + interface It { + fuzz: FuzzTest; + } + } +} diff --git a/packages/jest-runner/package.json b/packages/jest-runner/package.json index 92bf3211..6d0e89f7 100644 --- a/packages/jest-runner/package.json +++ b/packages/jest-runner/package.json @@ -19,7 +19,8 @@ "@jazzer.js/core": "*", "cosmiconfig": "^8.0.0", "jest": "^29.4.2", - "istanbul-reports": "^3.1.5" + "istanbul-reports": "^3.1.5", + "ts-jest": "^29.0.5" }, "devDependencies": { "@types/istanbul-reports": "^3.0.1", diff --git a/packages/jest-runner/worker.ts b/packages/jest-runner/worker.ts index 43562556..f67c340e 100644 --- a/packages/jest-runner/worker.ts +++ b/packages/jest-runner/worker.ts @@ -26,6 +26,8 @@ import { formatResultsErrors } from "jest-message-util"; import { inspect } from "util"; import { fuzz, FuzzerStartError, skip } from "./fuzz"; import { cleanupJestRunnerStack, removeTopFramesFromError } from "./errorUtils"; +import { createScriptTransformer } from "@jest/transform"; +import "./jest-extension"; function isGeneratorFunction(obj?: unknown): boolean { return ( @@ -104,9 +106,7 @@ export class JazzerWorker { globalThis.test.skip.fuzz = skip; // @ts-ignore globalThis.it = circus.it; - // @ts-ignore globalThis.it.fuzz = fuzz; - // @ts-ignore globalThis.it.skip.fuzz = skip; // @ts-ignore globalThis.describe = circus.describe; @@ -146,7 +146,8 @@ export class JazzerWorker { private async loadTests(test: Test): Promise { circus.resetState(); - await this.importFile(test.path); + const transformer = await createScriptTransformer(test.context.config); + await transformer.requireAndTranspileModule(test.path); return circus.getState(); }