Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
83d7e43
feat: add rust container flavor
rjaegers Mar 8, 2024
6903799
ci: fix linter findings
rjaegers Mar 12, 2024
d1de73e
Merge commit '8653908d77e6bf47652fce63ac10c1275999ea21' into feature/…
rjaegers Mar 12, 2024
a8e77e9
ci: ignore some linter findings
rjaegers Mar 12, 2024
0c5ea47
ci: run for all flavours
rjaegers Mar 12, 2024
1ee4e43
ci: fix tests for rust container flavor
rjaegers Mar 13, 2024
51d825d
Merge commit 'e25462503773386be342066d81c04bf25451e0c3' into feature/…
rjaegers Mar 13, 2024
27be8ef
chore: add pkg-config
rjaegers Mar 13, 2024
54e9106
ci: fix linter findings
rjaegers Mar 13, 2024
945ade2
Merge branch 'main' into feature/add-rust-container-flavor
rjaegers Mar 13, 2024
a63dc83
chore: modify build and push workflow for flavors
rjaegers Mar 13, 2024
ddfd6fb
Merge branch 'feature/add-rust-container-flavor' of https://github.co…
rjaegers Mar 13, 2024
1c7ec58
chore: reduce container size
rjaegers Mar 14, 2024
1a4ff0d
chore: add Docker extension to devcontainer.json
rjaegers Mar 15, 2024
b3f060c
Merge commit 'aa28065477dd5a4da08f8ed882382467d362f1b5' into feature/…
rjaegers Apr 15, 2024
87fd45a
chore: update rust from 1.76.0 to 1.77.2
rjaegers Apr 16, 2024
02c8814
ci: publish result of all testsuites
rjaegers Apr 16, 2024
8f82719
ci: fix duplicate artifact
rjaegers Apr 16, 2024
4eff8f7
Update ci.yml
rjaegers Apr 16, 2024
e9ff8ff
ci: different fix for multiple artifacts
rjaegers Apr 16, 2024
963df19
Merge commit '07dcd6647b97d99fafbe3fa71b902b5bd902222b' into feature/…
rjaegers Apr 16, 2024
8358a7f
ci: update all automation to cope with multiple containers
rjaegers Apr 17, 2024
046e847
chore: move update-apt-packages to own action
rjaegers Apr 17, 2024
a8a988e
chore: add flip-link package
rjaegers Apr 17, 2024
70053ce
Merge branch 'main' into feature/add-rust-container-flavor
rjaegers Apr 20, 2024
0cf302e
Merge commit '80383bdc90f68dc545d2923cbbdb33af4ec9be5c' into feature/…
rjaegers Apr 22, 2024
e028650
chore: process review comments
rjaegers Apr 22, 2024
b505db7
chore: merge the -vscode image into the main image
rjaegers Apr 22, 2024
60314f1
Update build-push.yml
rjaegers Apr 22, 2024
7a48e06
Merge commit '111f15a671d35ed3ccbfc57d7312a2a7885aae32' into feature/…
rjaegers May 6, 2024
dead322
chore: minor fix after merge
rjaegers May 6, 2024
5d83980
chore: move Rust container to Ubuntu 24.04
rjaegers May 6, 2024
2d31f95
chore: update Rust to latest version
rjaegers May 6, 2024
a8c6417
chore: update Docker cli from 25.0.3 to 26.1.1
rjaegers May 7, 2024
74d209b
chore: add more test for Rust container
rjaegers May 7, 2024
de80096
chore: reduce duplication in test code
rjaegers May 7, 2024
269b613
chore: add test for coverage
rjaegers May 7, 2024
62add84
chore: remove unused bats tags
rjaegers May 7, 2024
8501790
chore: don't use run in bats tests when not needed
rjaegers May 7, 2024
a5dd48c
chore: add mutation testing support to Rust container
rjaegers May 7, 2024
6011a60
chore: correct test name to beter reflect its actual scope
rjaegers May 7, 2024
4c44ac0
docs: update documentation
rjaegers May 8, 2024
e7e6d74
Merge commit '5804c2b607a69b190cfadc7136a25b31fa4aaab2' into feature/…
rjaegers May 8, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .devcontainer/Dockerfile → .devcontainer/cpp/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ FROM ubuntu:24.04@sha256:3f85b7caad41a95462cf5b787d8a04604c8262cdcdf9a472b8c52ef
ARG BATS_VERSION=1.10.0
ARG CCACHE_VERSION=4.9.1
ARG CLANG_VERSION=17
ARG DOCKER_VERSION=25.0.3
ARG DOCKER_VERSION=26.1.1
ARG MULL_VERSION=main
ARG INCLUDE_WHAT_YOU_USE_VERSION=0.21
ARG XWIN_VERSION=0.5.0
Expand All @@ -15,7 +15,7 @@ HEALTHCHECK NONE
SHELL ["/bin/bash", "-o", "pipefail", "-c"]

# Install the base system with all tool dependencies
COPY .devcontainer/apt-requirements-base.json /tmp/apt-requirements-base.json
COPY .devcontainer/cpp/apt-requirements-base.json /tmp/apt-requirements-base.json
# hadolint ignore=DL3008
RUN apt-get update && apt-get install -y --no-install-recommends jq \
&& jq -r 'to_entries | .[] | .key + "=" + .value' /tmp/apt-requirements-base.json | xargs apt-get install -y --no-install-recommends \
Expand All @@ -27,7 +27,7 @@ RUN wget -qO /usr/local/share/ca-certificates/Cisco_Umbrella_Root_CA.crt https:/
&& update-ca-certificates

# Install some tools via pip to get more recent versions
COPY .devcontainer/requirements.txt /tmp/requirements.txt
COPY .devcontainer/cpp/requirements.txt /tmp/requirements.txt
RUN python3 -m pip install --break-system-packages --require-hashes --no-cache-dir -r /tmp/requirements.txt \
&& rm -rf /tmp/requirements.txt

Expand All @@ -36,7 +36,7 @@ ENV CMAKE_GENERATOR="Ninja"
ENV CMAKE_EXPORT_COMPILE_COMMANDS="On"

# Install clang toolchain
COPY .devcontainer/apt-requirements-clang.json /tmp/apt-requirements-clang.json
COPY .devcontainer/cpp/apt-requirements-clang.json /tmp/apt-requirements-clang.json
# hadolint ignore=SC1091
RUN wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /usr/share/keyrings/llvm-snapshot-keyring.gpg \
&& UBUNTU_CODENAME=$(. /etc/os-release; echo "${UBUNTU_CODENAME/*, /}") \
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
{
"build": {
"dockerfile": "Dockerfile",
"context": ".."
"context": "../.."
},
"remoteEnv": {
"CONTAINER_FLAVOR": "cpp"
},
"mounts": [
"source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
Expand All @@ -14,9 +17,11 @@
"jetmartin.bats@0.1.10",
"matepek.vscode-catch2-test-adapter@4.6.3",
"mhutchie.git-graph@1.30.0",
"ms-azuretools.vscode-docker@1.29.0",
"ms-vscode.cmake-tools@1.16.32",
"ms-vscode.cpptools@1.18.5",
"SonarSource.sonarlint-vscode@4.1.0"
"SonarSource.sonarlint-vscode@4.1.0",
"usernamehw.errorlens@3.16.0"
]
}
}
Expand Down
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@ include(GoogleTest)

add_subdirectory(clang-cl)
add_subdirectory(clang-tools)
add_subdirectory(coverage)
add_subdirectory(fuzzing)
add_subdirectory(gcc)
add_subdirectory(gcc-arm-none-eabi)
add_subdirectory(mutation)
add_subdirectory(sanitizers)
add_subdirectory(test)
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
31 changes: 31 additions & 0 deletions .devcontainer/cpp/test/test/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
option(ENABLE_COVERAGE_TEST Off)
option(ENABLE_MUTATION_TESTING_TEST Off)

include(FetchContent)

FetchContent_Declare(googletest
GIT_REPOSITORY https://github.com/google/googletest
GIT_TAG v1.14.0
)

if (ENABLE_COVERAGE_TEST)
FetchContent_MakeAvailable(googletest)

add_executable(test-coverage EXCLUDE_FROM_ALL test.cpp)
target_compile_options(test-coverage PRIVATE
-g -O0 --coverage -fprofile-arcs -ftest-coverage -fno-inline
)
target_link_libraries(test-coverage PRIVATE gmock_main gcov)
add_test(NAME test-coverage COMMAND test-coverage)
endif()

if (ENABLE_MUTATION_TESTING_TEST)
FetchContent_MakeAvailable(googletest)

add_executable(test-mutation EXCLUDE_FROM_ALL test.cpp)
target_compile_options(test-mutation PRIVATE
-g -O0 -grecord-command-line -fprofile-instr-generate -fcoverage-mapping -fpass-plugin=/usr/lib/mull-ir-frontend
)
target_link_libraries(test-mutation PRIVATE gmock_main)
add_test(NAME test-mutation COMMAND mull-runner $<TARGET_FILE:test-mutation>)
endif()
File renamed without changes.
152 changes: 43 additions & 109 deletions test/testsuite.bats → .devcontainer/cpp/test/testsuite.bats
Original file line number Diff line number Diff line change
Expand Up @@ -32,149 +32,105 @@ teardown() {
rm -rf build crash-*
}

# bats test_tags=tc:1
@test "valid code input should result in working executable using host compiler" {
run cmake --preset gcc
assert_success

run cmake --build --preset gcc
assert_success
cmake --preset gcc
cmake --build --preset gcc

run build/gcc/gcc/test-gcc
assert_success
assert_output "Hello World!"
}

# bats test_tags=tc:2
@test "valid code input should result in elf executable using arm-none-eabi compiler" {
run cmake --preset gcc-arm-none-eabi
assert_success

run cmake --build --preset gcc-arm-none-eabi
assert_success
cmake --preset gcc-arm-none-eabi
cmake --build --preset gcc-arm-none-eabi

run readelf -h build/gcc-arm-none-eabi/gcc-arm-none-eabi/test-gcc-arm-none-eabi
assert_output --partial "Type: EXEC"
assert_output --partial "Machine: ARM"
}

# bats test_tags=tc:3
@test "valid code input should result in working Windows executable using clang-cl compiler" {
run cmake --preset clang-cl
assert_success

run cmake --build --preset clang-cl
assert_success
@test "valid code input should result in Windows executable using clang-cl compiler" {
cmake --preset clang-cl
cmake --build --preset clang-cl
}

# bats test_tags=tc:20
@test "compilation database should be generated on CMake configure" {
run cmake --preset gcc
[ -e build/gcc/compile_commands.json ]
cmake --preset gcc
assert [ -e build/gcc/compile_commands.json ]

run cmake --preset gcc-arm-none-eabi
[ -e build/gcc-arm-none-eabi/compile_commands.json ]
cmake --preset gcc-arm-none-eabi
assert [ -e build/gcc-arm-none-eabi/compile_commands.json ]
}

# bats test_tags=tc:4
@test "invalid code input should result in failing build" {
run cmake --preset gcc
assert_success

run cmake --build --preset gcc-fail
assert_failure
cmake --preset gcc
run ! cmake --build --preset gcc-fail
}

# bats test_tags=tc:5
@test "using ccache as a compiler launcher should result in cached build using gcc compiler" {
run ccache --clear --zero-stats

run cmake --preset gcc -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
assert_success

run cmake --build --preset gcc
assert_success
ccache --clear --zero-stats
cmake --preset gcc -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
cmake --build --preset gcc

run ccache -s
assert_output --partial "Hits: 0"
assert_output --partial "Misses: 1"

run rm -rf build
run ccache --zero-stats

run cmake --preset gcc -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
assert_success

run cmake --build --preset gcc
assert_success
rm -rf build
ccache --zero-stats
cmake --preset gcc -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
cmake --build --preset gcc

run ccache -s
assert_output --partial "Hits: 1"
assert_output --partial "Misses: 0"
}

# bats test_tags=tc:17
@test "using ccache as a compiler launcher should result in cached build using clang-cl compiler" {
run ccache --clear --zero-stats

run cmake --preset clang-cl -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
assert_success

run cmake --build --preset clang-cl
assert_success
ccache --clear --zero-stats
cmake --preset clang-cl -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
cmake --build --preset clang-cl

run ccache -s
assert_output --partial "Hits: 0"
assert_output --partial "Misses: 1"

run rm -rf build
run ccache --zero-stats

run cmake --preset clang-cl -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
assert_success

run cmake --build --preset clang-cl
assert_success
rm -rf build
ccache --zero-stats
cmake --preset clang-cl -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
cmake --build --preset clang-cl

run ccache -s
assert_output --partial "Hits: 1"
assert_output --partial "Misses: 0"
}

# bats test_tags=tc:6
@test "running clang-tidy as part of the build should result in warning diagnostics" {
run cmake --preset clang
assert_success
cmake --preset clang

run cmake --build --preset clang-tidy
assert_success
assert_output --partial "warning: use a trailing return type for this function"
}

# bats test_tags=tc:7
@test "running include-what-you-use as part of the build should result in warning diagnostics" {
run cmake --preset clang
assert_success
cmake --preset clang

run cmake --build --preset clang-iwyu
assert_success
assert_output --partial "Warning: include-what-you-use reported diagnostics:"
}

# bats test_tags=tc:8
@test "running clang-format should result in re-formatted code" {
run clang-format clang-tools/unformatted.cpp
assert_success
assert_output "int main() {}"
}

# bats test_tags=tc:9
@test "coverage information should be generated when running a testsuite" {
run cmake --preset coverage
assert_success

run cmake --build --preset coverage
assert_success
cmake --preset coverage
cmake --build --preset coverage

run ctest --preset coverage
assert_success
Expand All @@ -185,57 +141,40 @@ teardown() {
assert_output --partial "GCC Code Coverage Report"
}

# bats test_tags=tc:10
@test "crashes should be detected when fuzzing an executable" {
run cmake --preset clang
assert_success

run cmake --build --preset fuzzing
assert_success
cmake --preset clang
cmake --build --preset fuzzing

run build/clang/fuzzing/test-fuzzing
assert_failure
assert_output --partial "SUMMARY: libFuzzer: deadly signal"
}

# bats test_tags=tc:11
@test "a mutation score should be calculated when mutation testing a testsuite" {
run cmake --preset mutation
assert_success

run cmake --build --preset mutation
assert_success
cmake --preset mutation
cmake --build --preset mutation

run ctest --preset mutation
assert_output --partial "[info] Mutation score:"
}

# bats test_tags=tc:12
@test "host gdb should be able to start" {
run gdb --version
assert_success
gdb --version
}

# bats test_tags=tc:13
@test "gdb-multiarch should be able to start" {
run gdb-multiarch --version
assert_success
gdb-multiarch --version
}

# bats test_tags=tc:14
@test "clangd should be able to analyze source files" {
run clangd --check=gcc/main.cpp
assert_success
assert_output --partial "All checks completed, 0 errors"
}

# bats test_tags=tc:15
@test "using lld as an alternative linker should result in working host executable" {
run cmake --preset gcc
assert_success

run cmake --build --preset gcc-lld
assert_success
cmake --preset gcc
cmake --build --preset gcc-lld

run readelf --string-dump .comment build/gcc/gcc/test-gcc-lld
assert_output --partial "Linker: Ubuntu LLD"
Expand All @@ -245,18 +184,15 @@ teardown() {
assert_output "Hello World!"
}

# bats test_tags=tc:18
@test "when the docker socket is mounted, using the docker cli should give access to the host docker daemon" {
run docker info
assert_success
assert_output --partial "Server Version:"
}

# bats test_tags=tc:21
@test "sanitizers should detect undefined or suspicious behavior in code compiled with gcc" {
run cmake --preset gcc
run cmake --build --preset gcc-sanitizers
assert_success
cmake --preset gcc
cmake --build --preset gcc-sanitizers

run build/gcc/sanitizers/test-asan
assert_failure
Expand All @@ -267,11 +203,9 @@ teardown() {
assert_output --partial "runtime error: load of null pointer"
}

# bats test_tags=tc:22
@test "sanitizers should detect undefined or suspicious behavior in code compiled with clang" {
run cmake --preset clang
run cmake --build --preset clang-sanitizers
assert_success
cmake --preset clang
cmake --build --preset clang-sanitizers

run build/clang/sanitizers/test-asan
assert_failure
Expand Down
Loading