From 2825d04a64b1bb760c3d8e555c90e87b8c49e7c6 Mon Sep 17 00:00:00 2001 From: Brian Lewis Date: Thu, 27 Jul 2023 14:35:11 +0200 Subject: [PATCH 1/9] Add release canary test --- .github/workflows/release.yaml | 18 +++++++++++ end-to-end/.gitignore | 2 ++ end-to-end/.jazzerjsrc | 4 +++ end-to-end/README.md | 10 ++++++ end-to-end/integration.fuzz.ts | 35 ++++++++++++++++++++ end-to-end/integration.test.ts | 37 ++++++++++++++++++++++ end-to-end/jest.config.ts | 25 +++++++++++++++ end-to-end/package.json | 19 +++++++++++ end-to-end/target.ts | 58 ++++++++++++++++++++++++++++++++++ end-to-end/tsconfig.json | 17 ++++++++++ 10 files changed, 225 insertions(+) create mode 100644 end-to-end/.gitignore create mode 100644 end-to-end/.jazzerjsrc create mode 100644 end-to-end/README.md create mode 100644 end-to-end/integration.fuzz.ts create mode 100644 end-to-end/integration.test.ts create mode 100644 end-to-end/jest.config.ts create mode 100644 end-to-end/package.json create mode 100644 end-to-end/target.ts create mode 100644 end-to-end/tsconfig.json diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ffeacdfe..c30478f3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -23,3 +23,21 @@ jobs: run: npm publish --workspaces --access public env: NODE_AUTH_TOKEN: ${{secrets.NPM_ACCESS_TOKEN}} + + end-to-end: + name: end-to-end + runs-on: ubuntu-20.04 + needs: ["release"] + steps: + - name: checkout + uses: actions/checkout@v3 + - name: node + uses: actions/setup-node@v3 + with: + node-version: 16 + cache: "npm" + registry-url: https://registry.npmjs.org + - name: build-example + run: cd end-to-end && npm install && npm run build + - name: run tests + run: cd end-to-end && npx jest diff --git a/end-to-end/.gitignore b/end-to-end/.gitignore new file mode 100644 index 00000000..e1d5970f --- /dev/null +++ b/end-to-end/.gitignore @@ -0,0 +1,2 @@ +**/package-lock.json +**/.cifuzz-corpus diff --git a/end-to-end/.jazzerjsrc b/end-to-end/.jazzerjsrc new file mode 100644 index 00000000..b789cbcb --- /dev/null +++ b/end-to-end/.jazzerjsrc @@ -0,0 +1,4 @@ +{ + "includes": ["target"], + "excludes": ["node_modules"] +} diff --git a/end-to-end/README.md b/end-to-end/README.md new file mode 100644 index 00000000..5f9d67e1 --- /dev/null +++ b/end-to-end/README.md @@ -0,0 +1,10 @@ +# Jazzer End to End Canary Test + +This is the code from `examples/jest_typescript_integration` with a single +change to `package.json`: the `@jazzer.js/jest-runner` dependency is now set to +version `*`. This project is meant to be run in our release pipeline after the +release has been created to do a final check to make sure that nothing is broken +in our packaging. + +The Typescript integration example was chosen as that should exercise more of +jazzer.js than the other examples. diff --git a/end-to-end/integration.fuzz.ts b/end-to-end/integration.fuzz.ts new file mode 100644 index 00000000..a85e866a --- /dev/null +++ b/end-to-end/integration.fuzz.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "@jazzer.js/jest-runner"; +import * as target from "./target"; + +describe("Target", () => { + it.fuzz("executes sync methods", (data: Buffer) => { + target.fuzzMe(data); + }); + + it.fuzz("executes async methods", async (data: Buffer) => { + await target.asyncFuzzMe(data); + }); + + it.fuzz( + "executes methods with a done callback", + (data: Buffer, done: (e?: Error) => void) => { + target.callbackFuzzMe(data, done); + }, + ); +}); diff --git a/end-to-end/integration.test.ts b/end-to-end/integration.test.ts new file mode 100644 index 00000000..05172efa --- /dev/null +++ b/end-to-end/integration.test.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as target from "./target"; + +describe("My describe", () => { + it("My normal Jest test", () => { + expect(1).toEqual(1); + }); + + it("My done callback Jest test", (done) => { + expect(1).toEqual(1); + done(); + }); + + it("My async Jest test", async () => { + expect(1).toEqual(1); + }); + + it("Test target function", () => { + const data = Buffer.from("a"); + target.fuzzMe(data); + }); +}); diff --git a/end-to-end/jest.config.ts b/end-to-end/jest.config.ts new file mode 100644 index 00000000..55b2597f --- /dev/null +++ b/end-to-end/jest.config.ts @@ -0,0 +1,25 @@ +import type { Config } from "jest"; + +const config: Config = { + verbose: true, + projects: [ + { + displayName: "Jest", + preset: "ts-jest", + }, + { + displayName: { + name: "Jazzer.js", + color: "cyan", + }, + preset: "ts-jest", + runner: "@jazzer.js/jest-runner", + testEnvironment: "node", + testMatch: ["/*.fuzz.[jt]s"], + }, + ], + coveragePathIgnorePatterns: ["/node_modules/", "/dist/"], + modulePathIgnorePatterns: ["/node_modules", "/dist/"], +}; + +export default config; diff --git a/end-to-end/package.json b/end-to-end/package.json new file mode 100644 index 00000000..085c2680 --- /dev/null +++ b/end-to-end/package.json @@ -0,0 +1,19 @@ +{ + "name": "jest_typescript_integration", + "version": "1.0.0", + "description": "An example showing how Jazzer.js integrates with Jest and TypeScript", + "scripts": { + "build": "tsc", + "dryRun": "jest", + "fuzz": "JAZZER_FUZZ=1 jest --coverage", + "coverage": "jest --coverage" + }, + "devDependencies": { + "@jazzer.js/jest-runner": "*", + "@types/jest": "^29.4.0", + "jest": "^29.4.1", + "ts-jest": "^29.0.5", + "ts-node": "^10.9.1", + "typescript": "^4.9.5" + } +} diff --git a/end-to-end/target.ts b/end-to-end/target.ts new file mode 100644 index 00000000..662e2c26 --- /dev/null +++ b/end-to-end/target.ts @@ -0,0 +1,58 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function fuzzMe(data: Buffer) { + const s = data.toString(); + if (s.length !== 16) { + return; + } + if ( + s.slice(0, 8) === "Awesome " && + s.slice(8, 15) === "Fuzzing" && + s[15] === "!" + ) { + throw Error("Welcome to Awesome Fuzzing!"); + } +} + +export function callbackFuzzMe(data: Buffer, done: (e?: Error) => void) { + // Use setImmediate here to unblock the event loop but still have better + // performance compared to setTimeout. + setImmediate(() => { + try { + fuzzMe(data); + done(); + } catch (e: unknown) { + if (e instanceof Error) { + done(e); + } else { + done(new Error(`Error: ${e}`)); + } + } + }); +} + +export async function asyncFuzzMe(data: Buffer) { + return new Promise((resolve, reject) => { + callbackFuzzMe(data, (e?: Error) => { + if (e) { + reject(e); + } else { + resolve(null); + } + }); + }); +} diff --git a/end-to-end/tsconfig.json b/end-to-end/tsconfig.json new file mode 100644 index 00000000..7363a7c8 --- /dev/null +++ b/end-to-end/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "node", + "allowJs": true, + "rootDir": ".", + "outDir": "./dist", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "declaration": true, + "composite": true, + "sourceMap": true + } +} From 84f6a672809c9bbb9a628646cfd52a9d187a8c78 Mon Sep 17 00:00:00 2001 From: Brian Lewis Date: Mon, 31 Jul 2023 14:23:38 +0200 Subject: [PATCH 2/9] Switch to a test run on each PR --- .github/workflows/release.yaml | 18 ------------------ .github/workflows/run-all-tests.yaml | 19 +++++++++++++++++++ end-to-end/README.md | 8 ++++---- end-to-end/package-jazzer-js.sh | 20 ++++++++++++++++++++ end-to-end/package.json | 7 ++++++- end-to-end/remove-version.sed | 1 + 6 files changed, 50 insertions(+), 23 deletions(-) create mode 100755 end-to-end/package-jazzer-js.sh create mode 100644 end-to-end/remove-version.sed diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c30478f3..ffeacdfe 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -23,21 +23,3 @@ jobs: run: npm publish --workspaces --access public env: NODE_AUTH_TOKEN: ${{secrets.NPM_ACCESS_TOKEN}} - - end-to-end: - name: end-to-end - runs-on: ubuntu-20.04 - needs: ["release"] - steps: - - name: checkout - uses: actions/checkout@v3 - - name: node - uses: actions/setup-node@v3 - with: - node-version: 16 - cache: "npm" - registry-url: https://registry.npmjs.org - - name: build-example - run: cd end-to-end && npm install && npm run build - - name: run tests - run: cd end-to-end && npx jest diff --git a/.github/workflows/run-all-tests.yaml b/.github/workflows/run-all-tests.yaml index c4a79d7a..688d3682 100644 --- a/.github/workflows/run-all-tests.yaml +++ b/.github/workflows/run-all-tests.yaml @@ -75,11 +75,30 @@ jobs: run: npm run build --workspace=@jazzer.js/fuzzer - name: run all fuzz tests run: node fuzztests/runFuzzTests.js + end-to-end: + name: end-to-end + runs-on: ubuntu-20.04 + steps: + - name: checkout + uses: actions/checkout@v3 + - name: node + uses: actions/setup-node@v3 + with: + node-version: 16 + cache: "npm" + registry-url: https://registry.npmjs.org + - name: pack jazzer.js + run: cd end-to-end && ./package-jazzer-js.sh + - name: build example + run: cd end-to-end && npm install --save-dev *.tgz && npm run build + - name: run tests + run: cd end-to-end && npx jest auto-merge: needs: - linting - unit_tests - fuzz_tests + - end-to-end permissions: pull-requests: write contents: write diff --git a/end-to-end/README.md b/end-to-end/README.md index 5f9d67e1..093f1de9 100644 --- a/end-to-end/README.md +++ b/end-to-end/README.md @@ -1,10 +1,10 @@ # Jazzer End to End Canary Test This is the code from `examples/jest_typescript_integration` with a single -change to `package.json`: the `@jazzer.js/jest-runner` dependency is now set to -version `*`. This project is meant to be run in our release pipeline after the -release has been created to do a final check to make sure that nothing is broken -in our packaging. +change to `package.json`: the Jazzer.js dependencies now come from +`jazzer-js-.tgz` files in this directory. These can be created by +running `./package-jazzer-js.sh` which will call `npm pack` on the Jazzer +packages so that we can test for any packaging errors. The Typescript integration example was chosen as that should exercise more of jazzer.js than the other examples. diff --git a/end-to-end/package-jazzer-js.sh b/end-to-end/package-jazzer-js.sh new file mode 100755 index 00000000..8ea699f4 --- /dev/null +++ b/end-to-end/package-jazzer-js.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +cd .. +npm install +npm run build +npm run build --workspace='@jazzer.js/fuzzer' + +sed_version_and_mv() { + while read data; do + local no_version=$(echo $data | sed -r -f end-to-end/remove-version.sed) + echo "mv $data end-to-end/$no_version" + mv $data end-to-end/$no_version + done +} + +npm pack --workspaces | sed_version_and_mv + +cd examples + +npm install diff --git a/end-to-end/package.json b/end-to-end/package.json index 085c2680..5cba6f74 100644 --- a/end-to-end/package.json +++ b/end-to-end/package.json @@ -9,7 +9,12 @@ "coverage": "jest --coverage" }, "devDependencies": { - "@jazzer.js/jest-runner": "*", + "@jazzer.js/bug-detectors": "file:jazzer.js-bug-detectors.tgz", + "@jazzer.js/core": "file:jazzer.js-core.tgz", + "@jazzer.js/fuzzer": "file:jazzer.js-fuzzer.tgz", + "@jazzer.js/hooking": "file:jazzer.js-hooking.tgz", + "@jazzer.js/instrumentor": "file:jazzer.js-instrumentor.tgz", + "@jazzer.js/jest-runner": "file:jazzer.js-jest-runner.tgz", "@types/jest": "^29.4.0", "jest": "^29.4.1", "ts-jest": "^29.0.5", diff --git a/end-to-end/remove-version.sed b/end-to-end/remove-version.sed new file mode 100644 index 00000000..8ea20285 --- /dev/null +++ b/end-to-end/remove-version.sed @@ -0,0 +1 @@ +s/-[0-9]+\.[0-9]+\.[0-9]+\.tgz/\.tgz/g \ No newline at end of file From 7b11e03ee6c79b940e8d00f8e8586002407b9557 Mon Sep 17 00:00:00 2001 From: Brian Lewis Date: Tue, 1 Aug 2023 09:19:00 +0200 Subject: [PATCH 3/9] Fix some unnecessary duplication of work package-jazzer-js.sh and the github action would both run `npm install` which isn't necessary --- .github/workflows/run-all-tests.yaml | 2 +- end-to-end/package-jazzer-js.sh | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/run-all-tests.yaml b/.github/workflows/run-all-tests.yaml index 688d3682..f2376e8f 100644 --- a/.github/workflows/run-all-tests.yaml +++ b/.github/workflows/run-all-tests.yaml @@ -90,7 +90,7 @@ jobs: - name: pack jazzer.js run: cd end-to-end && ./package-jazzer-js.sh - name: build example - run: cd end-to-end && npm install --save-dev *.tgz && npm run build + run: cd end-to-end && npm install && npm run build - name: run tests run: cd end-to-end && npx jest auto-merge: diff --git a/end-to-end/package-jazzer-js.sh b/end-to-end/package-jazzer-js.sh index 8ea699f4..3994b9bc 100755 --- a/end-to-end/package-jazzer-js.sh +++ b/end-to-end/package-jazzer-js.sh @@ -14,7 +14,3 @@ sed_version_and_mv() { } npm pack --workspaces | sed_version_and_mv - -cd examples - -npm install From 9329a70947dbb7556454e10af920b7634f9ffec1 Mon Sep 17 00:00:00 2001 From: Brian Lewis Date: Tue, 1 Aug 2023 11:27:46 +0200 Subject: [PATCH 4/9] Fix github action and reformat readme --- .github/workflows/run-all-tests.yaml | 2 +- end-to-end/README.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-all-tests.yaml b/.github/workflows/run-all-tests.yaml index f2376e8f..688d3682 100644 --- a/.github/workflows/run-all-tests.yaml +++ b/.github/workflows/run-all-tests.yaml @@ -90,7 +90,7 @@ jobs: - name: pack jazzer.js run: cd end-to-end && ./package-jazzer-js.sh - name: build example - run: cd end-to-end && npm install && npm run build + run: cd end-to-end && npm install --save-dev *.tgz && npm run build - name: run tests run: cd end-to-end && npx jest auto-merge: diff --git a/end-to-end/README.md b/end-to-end/README.md index 093f1de9..8475e43f 100644 --- a/end-to-end/README.md +++ b/end-to-end/README.md @@ -8,3 +8,15 @@ packages so that we can test for any packaging errors. The Typescript integration example was chosen as that should exercise more of jazzer.js than the other examples. + +## Running Locally + +``` +./package-jazzer-js.sh +npm install --save-dev *.tgz +npx jest +``` + +_Note_: running just `npm install` may result in caching issues where the +contents of the tarballs in this directory are ignored and older versions from +somewhere are used instead. From d80368f7cf19c06753ee882c59d089474eb23435 Mon Sep 17 00:00:00 2001 From: Brian Lewis Date: Tue, 1 Aug 2023 13:20:36 +0200 Subject: [PATCH 5/9] Add language to code block --- end-to-end/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end-to-end/README.md b/end-to-end/README.md index 8475e43f..7065e667 100644 --- a/end-to-end/README.md +++ b/end-to-end/README.md @@ -11,7 +11,7 @@ jazzer.js than the other examples. ## Running Locally -``` +```bash ./package-jazzer-js.sh npm install --save-dev *.tgz npx jest From 0cf490dd56259f1e8a4f16701a4b53ffc3417349 Mon Sep 17 00:00:00 2001 From: Brian Lewis Date: Tue, 1 Aug 2023 16:51:18 +0200 Subject: [PATCH 6/9] Remove unnecessary include --- end-to-end/.jazzerjsrc | 1 - 1 file changed, 1 deletion(-) diff --git a/end-to-end/.jazzerjsrc b/end-to-end/.jazzerjsrc index b789cbcb..99d787bd 100644 --- a/end-to-end/.jazzerjsrc +++ b/end-to-end/.jazzerjsrc @@ -1,4 +1,3 @@ { - "includes": ["target"], "excludes": ["node_modules"] } From ef924027f391fa3be9e18a6e29c6fc5587c5a58b Mon Sep 17 00:00:00 2001 From: Brian Lewis Date: Wed, 2 Aug 2023 15:01:25 +0200 Subject: [PATCH 7/9] Remove registry-url --- .github/workflows/run-all-tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run-all-tests.yaml b/.github/workflows/run-all-tests.yaml index 688d3682..7f5c33ba 100644 --- a/.github/workflows/run-all-tests.yaml +++ b/.github/workflows/run-all-tests.yaml @@ -86,7 +86,6 @@ jobs: with: node-version: 16 cache: "npm" - registry-url: https://registry.npmjs.org - name: pack jazzer.js run: cd end-to-end && ./package-jazzer-js.sh - name: build example From cbf97b2098967a486dd5453678f02d56f6b3e9d3 Mon Sep 17 00:00:00 2001 From: Brian Lewis Date: Wed, 2 Aug 2023 15:08:28 +0200 Subject: [PATCH 8/9] Remove jazzerjsrc --- end-to-end/.jazzerjsrc | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 end-to-end/.jazzerjsrc diff --git a/end-to-end/.jazzerjsrc b/end-to-end/.jazzerjsrc deleted file mode 100644 index 99d787bd..00000000 --- a/end-to-end/.jazzerjsrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "excludes": ["node_modules"] -} From bc2c6a8814b6f3235d4c4ce66c22e94e0d0cd19e Mon Sep 17 00:00:00 2001 From: Brian Lewis Date: Mon, 7 Aug 2023 11:08:03 +0200 Subject: [PATCH 9/9] Fix lint error --- end-to-end/jest.config.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/end-to-end/jest.config.ts b/end-to-end/jest.config.ts index 55b2597f..59c643db 100644 --- a/end-to-end/jest.config.ts +++ b/end-to-end/jest.config.ts @@ -1,3 +1,18 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import type { Config } from "jest"; const config: Config = {