From 3c28c70572d7c00fe517492c97a40a36d0075fca Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley Date: Thu, 5 Nov 2026 09:04:57 +0000 Subject: [PATCH] Run the badssl.com tests in CI without failing the build The tests that connect to badssl.com fail in most CI runs with TLS handshake errors or timeouts, and different tests fail each time, because the badssl.com servers are unreliable. Tag those tests with "Requires", "badssl", so the test runner skips them unless it is given /badssl, and register httpclient_test_badssl and websocketsclient_test_badssl, labelled badssl, to run only them. In CI, run the other tests with -LE badssl, then run the badssl.com tests in a separate step that does not fail the job. Signed-off-by: Gareth Sylvester-Bradley --- .github/workflows/build-test.yml | 18 +++++-- .../functional/http/client/CMakeLists.txt | 7 +++ .../functional/http/client/outside_tests.cpp | 49 ++++++++++++++----- .../functional/http/client/redirect_tests.cpp | 4 +- .../functional/websockets/CMakeLists.txt | 7 +++ .../client/authentication_tests.cpp | 6 +-- 6 files changed, 71 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 402f0384c9..0782fae58d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -35,7 +35,11 @@ jobs: run: cmake --build build - name: Test - run: ctest --test-dir build --output-on-failure + run: ctest --test-dir build --output-on-failure -LE badssl + + - name: Test with badssl.com + continue-on-error: true + run: ctest --test-dir build --output-on-failure -L badssl macos: name: ${{ matrix.os }} ${{ matrix.build_type }} @@ -67,7 +71,11 @@ jobs: run: cmake --build build - name: Test - run: ctest --test-dir build --output-on-failure + run: ctest --test-dir build --output-on-failure -LE badssl + + - name: Test with badssl.com + continue-on-error: true + run: ctest --test-dir build --output-on-failure -L badssl windows: name: ${{ matrix.os }} ${{ matrix.build_type }} @@ -97,4 +105,8 @@ jobs: run: cmake --build build --config ${{ matrix.build_type }} - name: Test - run: ctest --test-dir build -C ${{ matrix.build_type }} --output-on-failure + run: ctest --test-dir build -C ${{ matrix.build_type }} --output-on-failure -LE badssl + + - name: Test with badssl.com + continue-on-error: true + run: ctest --test-dir build -C ${{ matrix.build_type }} --output-on-failure -L badssl diff --git a/Release/tests/functional/http/client/CMakeLists.txt b/Release/tests/functional/http/client/CMakeLists.txt index 3e1a93635a..e21242a5ca 100644 --- a/Release/tests/functional/http/client/CMakeLists.txt +++ b/Release/tests/functional/http/client/CMakeLists.txt @@ -27,6 +27,13 @@ set(SOURCES ) add_casablanca_test(httpclient_test SOURCES) +if(BUILD_SHARED_LIBS) + add_test(NAME httpclient_test_badssl + WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + COMMAND test_runner /badssl /select:@Requires=badssl $ + ) + set_tests_properties(httpclient_test_badssl PROPERTIES LABELS badssl) +endif() if(TEST_LIBRARY_TARGET_TYPE STREQUAL "OBJECT") target_include_directories(httpclient_test PRIVATE ../utilities/include) else() diff --git a/Release/tests/functional/http/client/outside_tests.cpp b/Release/tests/functional/http/client/outside_tests.cpp index cbb8accdcd..b8147b1f8d 100644 --- a/Release/tests/functional/http/client/outside_tests.cpp +++ b/Release/tests/functional/http/client/outside_tests.cpp @@ -218,16 +218,19 @@ SUITE(outside_tests) } #endif // !defined(__cplusplus_winrt) - TEST(server_selfsigned_cert) { test_failed_ssl_cert(U("https://self-signed.badssl.com/")); } + TEST(server_selfsigned_cert, "Requires", "badssl") { test_failed_ssl_cert(U("https://self-signed.badssl.com/")); } #if !defined(__cplusplus_winrt) - TEST(server_selfsigned_cert_ignored) { test_ignored_ssl_cert(U("https://self-signed.badssl.com/")); } + TEST(server_selfsigned_cert_ignored, "Requires", "badssl") + { + test_ignored_ssl_cert(U("https://self-signed.badssl.com/")); + } #endif // !defined(__cplusplus_winrt) - TEST(server_hostname_mismatch) { test_failed_ssl_cert(U("https://wrong.host.badssl.com/")); } + TEST(server_hostname_mismatch, "Requires", "badssl") { test_failed_ssl_cert(U("https://wrong.host.badssl.com/")); } #if !defined(__cplusplus_winrt) && !defined(CPPREST_FORCE_HTTP_CLIENT_WINHTTPPAL) - TEST(server_hostname_host_override) + TEST(server_hostname_host_override, "Requires", "badssl") { handle_timeout([] { http_client client(U("https://wrong.host.badssl.com/")); @@ -238,7 +241,10 @@ SUITE(outside_tests) }); } - TEST(server_hostname_mismatch_ignored) { test_ignored_ssl_cert(U("https://wrong.host.badssl.com/")); } + TEST(server_hostname_mismatch_ignored, "Requires", "badssl") + { + test_ignored_ssl_cert(U("https://wrong.host.badssl.com/")); + } // 198.35.26.96 no longer accepts connections for en.wikipedia.org TEST(server_hostname_host_override_after_upgrade, "Ignore", "198.35.26.96") @@ -251,29 +257,48 @@ SUITE(outside_tests) } #endif // !defined(__cplusplus_winrt) && !defined(CPPREST_FORCE_HTTP_CLIENT_WINHTTPPAL) - TEST(server_cert_expired) { test_failed_ssl_cert(U("https://expired.badssl.com/")); } + TEST(server_cert_expired, "Requires", "badssl") { test_failed_ssl_cert(U("https://expired.badssl.com/")); } #if !defined(__cplusplus_winrt) - TEST(server_cert_expired_ignored) { test_ignored_ssl_cert(U("https://expired.badssl.com/")); } + TEST(server_cert_expired_ignored, "Requires", "badssl") { test_ignored_ssl_cert(U("https://expired.badssl.com/")); } #endif // !defined(__cplusplus_winrt) - TEST(server_cert_revoked, "Ignore:Android", "229", "Ignore:Apple", "229", "Ignore:Linux", "229") + TEST(server_cert_revoked, + "Requires", + "badssl", + "Ignore:Android", + "229", + "Ignore:Apple", + "229", + "Ignore:Linux", + "229") { test_failed_ssl_cert(U("https://revoked.badssl.com/")); } #if !defined(__cplusplus_winrt) - TEST(server_cert_revoked_ignored) { test_ignored_ssl_cert(U("https://revoked.badssl.com/")); } + TEST(server_cert_revoked_ignored, "Requires", "badssl") { test_ignored_ssl_cert(U("https://revoked.badssl.com/")); } #endif // !defined(__cplusplus_winrt) - TEST(server_cert_untrusted) { test_failed_ssl_cert(U("https://untrusted-root.badssl.com/")); } + TEST(server_cert_untrusted, "Requires", "badssl") { test_failed_ssl_cert(U("https://untrusted-root.badssl.com/")); } #if !defined(__cplusplus_winrt) - TEST(server_cert_untrusted_ignored) { test_ignored_ssl_cert(U("https://untrusted-root.badssl.com/")); } + TEST(server_cert_untrusted_ignored, "Requires", "badssl") + { + test_ignored_ssl_cert(U("https://untrusted-root.badssl.com/")); + } #endif // !defined(__cplusplus_winrt) #if !defined(__cplusplus_winrt) - TEST(ignore_server_cert_invalid, "Ignore:Android", "229", "Ignore:Apple", "229", "Ignore:Linux", "229") + TEST(ignore_server_cert_invalid, + "Requires", + "badssl", + "Ignore:Android", + "229", + "Ignore:Apple", + "229", + "Ignore:Linux", + "229") { handle_timeout([] { http_client_config config; diff --git a/Release/tests/functional/http/client/redirect_tests.cpp b/Release/tests/functional/http/client/redirect_tests.cpp index a9d4179450..50311b5c54 100644 --- a/Release/tests/functional/http/client/redirect_tests.cpp +++ b/Release/tests/functional/http/client/redirect_tests.cpp @@ -160,7 +160,7 @@ SUITE(redirect_tests) } } - TEST(does_not_follow_https_to_http_by_default) + TEST(does_not_follow_https_to_http_by_default, "Requires", "badssl") { handle_timeout([] { http_client_config config; @@ -171,7 +171,7 @@ SUITE(redirect_tests) }); } - TEST(can_follow_https_to_http) + TEST(can_follow_https_to_http, "Requires", "badssl") { handle_timeout([] { http_client_config config; diff --git a/Release/tests/functional/websockets/CMakeLists.txt b/Release/tests/functional/websockets/CMakeLists.txt index c8efb77ca1..9a33589f5f 100644 --- a/Release/tests/functional/websockets/CMakeLists.txt +++ b/Release/tests/functional/websockets/CMakeLists.txt @@ -28,6 +28,13 @@ if (NOT CPPREST_EXCLUDE_WEBSOCKETS) ) add_casablanca_test(websocketsclient_test SOURCES) + if(BUILD_SHARED_LIBS) + add_test(NAME websocketsclient_test_badssl + WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} + COMMAND test_runner /badssl /select:@Requires=badssl $ + ) + set_tests_properties(websocketsclient_test_badssl PROPERTIES LABELS badssl) + endif() if(NOT TEST_LIBRARY_TARGET_TYPE STREQUAL "OBJECT") target_link_libraries(websocketsclient_test PRIVATE websockettest_utilities) endif() diff --git a/Release/tests/functional/websockets/client/authentication_tests.cpp b/Release/tests/functional/websockets/client/authentication_tests.cpp index ab7de2e101..872535e429 100644 --- a/Release/tests/functional/websockets/client/authentication_tests.cpp +++ b/Release/tests/functional/websockets/client/authentication_tests.cpp @@ -149,11 +149,11 @@ SUITE(authentication_tests) } } - TEST(self_signed_cert) { handshake_error_test_impl(U("wss://self-signed.badssl.com/")); } + TEST(self_signed_cert, "Requires", "badssl") { handshake_error_test_impl(U("wss://self-signed.badssl.com/")); } - TEST(hostname_mismatch) { handshake_error_test_impl(U("wss://wrong.host.badssl.com/")); } + TEST(hostname_mismatch, "Requires", "badssl") { handshake_error_test_impl(U("wss://wrong.host.badssl.com/")); } - TEST(cert_expired) { handshake_error_test_impl(U("wss://expired.badssl.com/")); } + TEST(cert_expired, "Requires", "badssl") { handshake_error_test_impl(U("wss://expired.badssl.com/")); } } // SUITE(authentication_tests)