diff --git a/.github/workflows/build-posix-latest.yml b/.github/workflows/build-posix-latest.yml index 13ac881ab..8f9320e57 100644 --- a/.github/workflows/build-posix-latest.yml +++ b/.github/workflows/build-posix-latest.yml @@ -47,3 +47,14 @@ jobs: continue-on-error: true - name: Test ${{ matrix.os }} ${{ matrix.config }} run: ./build-tests.sh ${{ matrix.config }} + + public-headers: + name: Public header gate (GCC/Clang) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install clang + run: sudo apt-get update && sudo apt-get install -y clang + - name: Compile each public header standalone under strict flags + run: bash tests/headers/check_public_headers.sh diff --git a/.github/workflows/build-windows-vs2022.yaml b/.github/workflows/build-windows-vs2022.yaml index b28aa1515..222e32e67 100644 --- a/.github/workflows/build-windows-vs2022.yaml +++ b/.github/workflows/build-windows-vs2022.yaml @@ -29,13 +29,13 @@ jobs: - name: Checkout uses: actions/checkout@v4 - continue-on-error: true - name: Build env: SKIP_ARM_BUILD: 1 SKIP_ARM64_BUILD: 1 + SKIP_NET40_BUILD: 1 PlatformToolset: v143 VSTOOLS_VERSION: vs2022 shell: cmd - run: build-all.bat + run: build-all-windows.bat diff --git a/.github/workflows/test-win-latest.yml b/.github/workflows/test-win-latest.yml index 255868a88..4928fc71f 100644 --- a/.github/workflows/test-win-latest.yml +++ b/.github/workflows/test-win-latest.yml @@ -54,3 +54,13 @@ jobs: - name: Test ${{ matrix.arch }} ${{ matrix.build }} shell: cmd run: build-tests.cmd ${{ matrix.arch }} ${{ matrix.build }} + + public-headers: + name: Public header gate (MSVC) + runs-on: windows-2022 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Compile each public header standalone under /W4 /WX + shell: cmd + run: tests\headers\check_public_headers.cmd diff --git a/.github/workflows/vcpkg-release-bump.yml b/.github/workflows/vcpkg-release-bump.yml new file mode 100644 index 000000000..77ed47444 --- /dev/null +++ b/.github/workflows/vcpkg-release-bump.yml @@ -0,0 +1,218 @@ +name: Vcpkg release bump + +# Opens a version-bump pull request against microsoft/vcpkg for the +# `cpp-client-telemetry` port whenever a new SDK release is published. +# +# It runs ONLY when a new version is cut: +# * automatically on a published, non-draft, non-prerelease GitHub Release +# whose tag looks like a version (vMAJOR.MINOR.PATCH.BUILD), or +# * manually via workflow_dispatch for a specific tag (recovery / re-run). +# It never runs on ordinary pushes, and it opens no PR if the port already +# matches the release (no version change). +# +# One-time setup required in this repository: +# * Variable VCPKG_FORK_REPO -> the vcpkg fork to push branches to, +# e.g. "your-org/vcpkg". +# * Secret VCPKG_BUMP_TOKEN -> a PAT (classic: repo+workflow, or +# fine-grained: Contents+Pull requests RW on +# the fork) able to push to VCPKG_FORK_REPO and +# open pull requests on microsoft/vcpkg. + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "Release tag to bump the vcpkg port to (e.g. v3.10.161.1)" + required: true + type: string + +permissions: + contents: read + +concurrency: + group: vcpkg-release-bump-${{ github.event.release.tag_name || github.event.inputs.tag }} + cancel-in-progress: false + +jobs: + bump: + name: Bump cpp-client-telemetry port + # Skip drafts and pre-releases; always allow manual dispatch. + if: >- + ${{ github.event_name == 'workflow_dispatch' || + (github.event.release.draft == false && github.event.release.prerelease == false) }} + runs-on: ubuntu-latest + env: + UPSTREAM_REPO: ${{ github.repository }} # microsoft/cpp_client_telemetry + VCPKG_UPSTREAM: microsoft/vcpkg + VCPKG_FORK_REPO: ${{ vars.VCPKG_FORK_REPO }} + PORT: cpp-client-telemetry + steps: + - name: Validate configuration + env: + VCPKG_BUMP_TOKEN: ${{ secrets.VCPKG_BUMP_TOKEN }} + run: | + set -euo pipefail + if [ -z "${VCPKG_FORK_REPO}" ]; then + echo "::error::Repository variable VCPKG_FORK_REPO is not set (e.g. 'your-org/vcpkg')." + exit 1 + fi + if [ -z "${VCPKG_BUMP_TOKEN}" ]; then + echo "::error::Secret VCPKG_BUMP_TOKEN is not set. Provide a token that can push to ${VCPKG_FORK_REPO} and open PRs on ${VCPKG_UPSTREAM}." + exit 1 + fi + + - name: Resolve tag and version + id: ver + env: + # Pass untrusted tag values through the environment instead of + # interpolating ${{ ... }} directly into the script body, so a tag + # containing shell metacharacters cannot inject commands into this + # step (which shares a runner with later PAT-bearing steps). + RELEASE_TAG: ${{ github.event.release.tag_name }} + INPUT_TAG: ${{ github.event.inputs.tag }} + run: | + set -euo pipefail + TAG="${RELEASE_TAG:-$INPUT_TAG}" + if [ -z "${TAG}" ]; then echo "::error::No release tag could be resolved."; exit 1; fi + # Only act on version tags: vMAJOR.MINOR.PATCH.BUILD. A non-matching + # tag from the automatic release trigger is a clean no-op (the SDK also + # has historical 3-part tags such as v3.3.8); a non-matching tag from a + # manual workflow_dispatch is user error and fails loudly. + if ! printf '%s' "${TAG}" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'; then + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "::error::Tag '${TAG}' is not a version tag (expected vX.Y.Z.W)." + exit 1 + fi + echo "::notice::Tag '${TAG}' is not a version tag (expected vX.Y.Z.W); nothing to bump." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + VERSION="${TAG#v}" + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + echo "branch=port/${PORT}-${VERSION}" >> "$GITHUB_OUTPUT" + echo "Bumping ${PORT} -> tag=${TAG} version=${VERSION}" + + - name: Compute source archive SHA512 + id: sha + if: ${{ steps.ver.outputs.skip != 'true' }} + run: | + set -euo pipefail + URL="https://github.com/${UPSTREAM_REPO}/archive/${{ steps.ver.outputs.tag }}.tar.gz" + echo "Downloading ${URL}" + curl -fsSL --retry 3 "${URL}" -o source.tar.gz + SHA512="$(sha512sum source.tar.gz | cut -d' ' -f1)" + echo "sha512=${SHA512}" >> "$GITHUB_OUTPUT" + echo "SHA512=${SHA512}" + + - name: Clone vcpkg fork and branch off upstream master + if: ${{ steps.ver.outputs.skip != 'true' }} + env: + GH_TOKEN: ${{ secrets.VCPKG_BUMP_TOKEN }} + run: | + set -euo pipefail + # Authenticate git via gh's credential helper instead of embedding the + # token in the clone URL (which would persist it in .git/config and + # risk leaking it if git echoes the remote). The helper is written to + # the global gitconfig and reused by the later push step. + gh auth setup-git + git clone --depth 1 "https://github.com/${VCPKG_FORK_REPO}.git" vcpkg + cd vcpkg + git remote add upstream "https://github.com/${VCPKG_UPSTREAM}.git" + git fetch --depth 1 upstream master + git checkout -B "${{ steps.ver.outputs.branch }}" upstream/master + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Bootstrap vcpkg + if: ${{ steps.ver.outputs.skip != 'true' }} + run: cd vcpkg && ./bootstrap-vcpkg.sh -disableMetrics + + - name: Update port REF, SHA512 and version + if: ${{ steps.ver.outputs.skip != 'true' }} + run: | + set -euo pipefail + cd vcpkg + PORTFILE="ports/${PORT}/portfile.cmake" + MANIFEST="ports/${PORT}/vcpkg.json" + if [ ! -f "${PORTFILE}" ] || [ ! -f "${MANIFEST}" ]; then + echo "::error::${PORT} port not found in ${VCPKG_UPSTREAM}. The port must already be in the registry before it can be bumped." + exit 1 + fi + sed -i -E "s|^([[:space:]]*REF[[:space:]]+).*$|\1${{ steps.ver.outputs.tag }}|" "${PORTFILE}" + sed -i -E "s|^([[:space:]]*SHA512[[:space:]]+).*$|\1${{ steps.sha.outputs.sha512 }}|" "${PORTFILE}" + jq --arg v "${{ steps.ver.outputs.version }}" '.version = $v | del(."port-version")' "${MANIFEST}" > "${MANIFEST}.tmp" + mv "${MANIFEST}.tmp" "${MANIFEST}" + ./vcpkg format-manifest "${MANIFEST}" + + - name: Validate updated production port + if: ${{ steps.ver.outputs.skip != 'true' }} + run: | + set -euo pipefail + cd vcpkg + MANIFEST="ports/${PORT}/vcpkg.json" + + # Exercise the real production path: MATSDK_VCPKG_SOURCE_DIR must be + # unset so the port downloads the just-updated REF/SHA512 instead of + # accidentally validating this workflow's working tree. This catches + # manifest/portfile changes that require source changes not present in + # the release tag. + unset MATSDK_VCPKG_SOURCE_DIR + + PORT_SPEC="${PORT}" + if jq -e '(.features["minimal-sqlite"] != null) and (.features["curl-openssl"] != null)' "${MANIFEST}" >/dev/null; then + # Use an opt-in feature set when available so release validation covers + # feature wiring as well as the default graph. The default graph is + # still covered by regular vcpkg CI and by consumers. + PORT_SPEC="${PORT}[core,minimal-sqlite,curl-openssl]" + fi + + echo "Validating production port: ${PORT_SPEC}" + ./vcpkg install "${PORT_SPEC}" --triplet x64-linux --clean-after-build + + - name: Detect change + id: diff + if: ${{ steps.ver.outputs.skip != 'true' }} + run: | + set -euo pipefail + cd vcpkg + if git diff --quiet -- "ports/${PORT}"; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "No change: ${PORT} is already at ${{ steps.ver.outputs.version }} with this REF/SHA512. Nothing to do." + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Commit, update version DB, push and open PR + if: ${{ steps.ver.outputs.skip != 'true' && steps.diff.outputs.changed == 'true' }} + env: + GH_TOKEN: ${{ secrets.VCPKG_BUMP_TOKEN }} + run: | + set -euo pipefail + cd vcpkg + # gh auth setup-git ran in the clone step; reuse that credential helper + # so 'git push' authenticates without a token in the remote URL. + BR="${{ steps.ver.outputs.branch }}" + git add "ports/${PORT}" + git commit -m "[${PORT}] Update to ${{ steps.ver.outputs.version }}" + ./vcpkg x-add-version "${PORT}" --overwrite-version + git add versions + git commit -m "[${PORT}] Update version database" + # Ensure a remote-tracking ref exists so --force-with-lease has a lease + # to compare against on reruns: the bump branch may already exist on the + # fork but be absent from this fresh clone. Ignore failure on the first + # run, when the branch does not exist remotely yet. + git fetch origin "+refs/heads/${BR}:refs/remotes/origin/${BR}" || true + git push --force-with-lease origin "${BR}" + if [ -n "$(gh pr list --repo "${VCPKG_UPSTREAM}" --head "$(printf '%s' "${VCPKG_FORK_REPO}" | cut -d/ -f1):${BR}" --state open --json number --jq '.[0].number // empty' 2>/dev/null)" ]; then + echo "An open PR already exists for ${BR}; the force-pushed branch refreshes it." + else + gh pr create \ + --repo "${VCPKG_UPSTREAM}" \ + --base master \ + --head "$(printf '%s' "${VCPKG_FORK_REPO}" | cut -d/ -f1):${BR}" \ + --title "[${PORT}] Update to ${{ steps.ver.outputs.version }}" \ + --body "Automated port bump to [\`${UPSTREAM_REPO}@${{ steps.ver.outputs.tag }}\`](https://github.com/${UPSTREAM_REPO}/releases/tag/${{ steps.ver.outputs.tag }}). Generated by the \`vcpkg-release-bump\` workflow." + fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a0ba0e82..cc36e9da3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,16 @@ else() endif() message(STATUS "MATSDK_USE_VCPKG_DEPS: ${MATSDK_USE_VCPKG_DEPS}") +# Build a private, feature-stripped copy of the vendored SQLite amalgamation +# instead of linking an external SQLite. The SDK uses SQLite only for its offline +# event-storage cache, so the minimal build (see lib/CMakeLists.txt +# MATSDK_SQLITE_MINIMAL_DEFS) omits every optional SQLite subsystem the SDK does +# not use, shrinking the SQLite code ~10% and removing the external sqlite3 +# dependency. Off by default to preserve the existing external/system-SQLite +# behavior; the Android NDK path always bundles SQLite regardless. +option(MATSDK_MINIMAL_SQLITE "Build a feature-stripped vendored SQLite instead of an external one" OFF) +message(STATUS "MATSDK_MINIMAL_SQLITE: ${MATSDK_MINIMAL_SQLITE}") + # Begin Uncomment for i386 build #set(CMAKE_SYSTEM_PROCESSOR i386) #set(CMAKE_C_FLAGS -m32) @@ -149,16 +159,17 @@ else() endif() if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - # Using GCC with -s and -Wl linker flags - set(REL_FLAGS "-s -Wl,--gc-sections -Os ${WARN_FLAGS} -ffunction-sections -fdata-sections -fmerge-all-constants") + # Using GCC with -s and -Wl linker flags. -ffunction-sections/-fdata-sections + # are set once for all dep modes by the global block further below. + set(REL_FLAGS "-s -Wl,--gc-sections -Os ${WARN_FLAGS} -fmerge-all-constants") elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") set(REL_FLAGS "${WARN_FLAGS}") elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") - # AppleClang does not support -ffunction-sections and -fdata-sections with the -fembed-bitcode and -fembed-bitcode-marker set(REL_FLAGS "-Os ${WARN_FLAGS} -fmerge-all-constants") else() - # Using clang - strip unsupported GCC options - set(REL_FLAGS "-Os ${WARN_FLAGS} -ffunction-sections -fmerge-all-constants") + # Using clang - strip unsupported GCC options (-ffunction-sections is set by + # the global block further below). + set(REL_FLAGS "-Os ${WARN_FLAGS} -fmerge-all-constants") endif() ## Uncomment this to reduce the volume of note warnings on RPi4 w/gcc-8 Ref. https://gcc.gnu.org/ml/gcc/2017-05/msg00073.html @@ -206,6 +217,55 @@ endif() endif() # NOT MATSDK_USE_VCPKG_DEPS (compiler flags) +# --- Dead-strip enablement (applies in BOTH vendored and vcpkg modes) --------- +# Deliberate exception to the "let the toolchain manage compiler flags" note +# above (the NOT MATSDK_USE_VCPKG_DEPS block): these flags are NOT optimization +# or dependency choices the vcpkg toolchain owns -- they only split functions and +# data into separate COMDATs/sections so a *consumer's* linker can drop +# unreferenced SDK code (MSVC /OPT:REF + /OPT:ICF, GNU/Clang --gc-sections, Apple +# ld -dead_strip). The toolchain does not set them, and the vcpkg-packaged +# library (and every MSVC build, which never gets /Gy from the block above) would +# otherwise link whole .obj files instead of individual functions. Applying them +# here in both modes closes that gap and matches the MSBuild Release projects, +# which already enable FunctionLevelLinking + OptimizeReferences + COMDATFolding. +if(MSVC) + # /Gy (function-level linking) is supported by both cl.exe and clang-cl. + add_compile_options(/Gy) + # /Gw (whole-program global data) is cl.exe-only; the ClangCL toolset (for + # which MSVC is also true) does not support it. + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + add_compile_options(/Gw) + endif() +elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") + # On Mach-O, clang emits .subsections_via_symbols, so ld64's -dead_strip + # already removes unreferenced code at per-symbol (function) granularity + # without -ffunction-sections; we add it only for cross-toolchain + # consistency. -fdata-sections is omitted because it historically conflicted + # with bitcode on AppleClang. + add_compile_options(-ffunction-sections) +else() + # GCC / Clang (Linux, Android, MinGW) + add_compile_options(-ffunction-sections -fdata-sections) +endif() + +# Hidden symbol visibility (non-Windows): export only the MATSDK_LIBABI-decorated +# public API (classes + the C API), hiding SDK internals and the bundled +# sqlite3/zlib. This shrinks the dynamic symbol table (faster dynamic +# linking/loading, smaller binaries) and enables more inlining + dead-code +# elimination -- the non-Windows analog of what /Gy plus the consumer's /OPT:REF +# achieve on MSVC. All Windows toolchains (MSVC, MinGW, ClangCL) restrict exports +# via __declspec(dllexport) on MATSDK_LIBABI (lib/include/public/ctmacros.hpp), +# so this is gated on NOT WIN32 (not NOT MSVC, which would also catch MinGW/ +# Clang-GNU Windows builds and apply ELF-style visibility that does not belong on +# a PE/COFF target). +if(NOT WIN32) + # -fvisibility=hidden applies to C and C++; -fvisibility-inlines-hidden is a + # C++-only option, so scope it to CXX. (Applying it to C sources -- e.g. the + # bundled sqlite3/zlib on the legacy Android path -- makes Clang emit an + # "unused argument" warning that becomes an error under the project's -Werror.) + add_compile_options(-fvisibility=hidden $<$:-fvisibility-inlines-hidden>) +endif() + include(tools/Utils.cmake) include(GNUInstallDirs) include(CMakePackageConfigHelpers) @@ -277,8 +337,20 @@ message(STATUS "SDK version: ${SDK_VERSION_PREFIX}-${MATSDK_BUILD_VERSION}") option(BUILD_HEADERS "Build API headers" YES) option(BUILD_LIBRARY "Build library" YES) option(BUILD_TEST_TOOL "Build console test tool" YES) -option(BUILD_UNIT_TESTS "Build unit tests" YES) -option(BUILD_FUNC_TESTS "Build functional tests" YES) +# Default the test suites ON only when this repository is the top-level project +# (developer/CI build), and OFF when it is consumed via add_subdirectory()/ +# FetchContent, so downstream projects don't build the tests or require the +# third_party/googletest submodule. PROJECT_IS_TOP_LEVEL exists on CMake >= 3.21; +# fall back to comparing the source dirs on older CMake (floor is 3.15). +if(DEFINED PROJECT_IS_TOP_LEVEL) + set(MATSDK_TESTS_DEFAULT ${PROJECT_IS_TOP_LEVEL}) +elseif(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(MATSDK_TESTS_DEFAULT ON) +else() + set(MATSDK_TESTS_DEFAULT OFF) +endif() +option(BUILD_UNIT_TESTS "Build unit tests" ${MATSDK_TESTS_DEFAULT}) +option(BUILD_FUNC_TESTS "Build functional tests" ${MATSDK_TESTS_DEFAULT}) option(BUILD_JNI_WRAPPER "Build JNI wrapper" NO) option(BUILD_OBJC_WRAPPER "Build Obj-C wrapper" YES) option(BUILD_SWIFT_WRAPPER "Build Swift Wrappers" YES) @@ -290,6 +362,35 @@ option(BUILD_SIGNALS "Build Signals" YES) option(BUILD_SANITIZER "Build Sanitizer" YES) option(LINK_STATIC_DEPENDS "Link dependencies for static build" YES) +set(MATSDK_ANDROID_HTTP_CLIENT "AUTO" CACHE STRING "Android HTTP client: AUTO, JAVA, or CURL") +set_property(CACHE MATSDK_ANDROID_HTTP_CLIENT PROPERTY STRINGS AUTO JAVA CURL) +string(TOUPPER "${MATSDK_ANDROID_HTTP_CLIENT}" MATSDK_ANDROID_HTTP_CLIENT_UPPER) +if(NOT MATSDK_ANDROID_HTTP_CLIENT_UPPER STREQUAL "AUTO" + AND NOT MATSDK_ANDROID_HTTP_CLIENT_UPPER STREQUAL "JAVA" + AND NOT MATSDK_ANDROID_HTTP_CLIENT_UPPER STREQUAL "CURL") + message(FATAL_ERROR + "MATSDK_ANDROID_HTTP_CLIENT must be AUTO, JAVA, or CURL; got " + "'${MATSDK_ANDROID_HTTP_CLIENT}'.") +endif() + +set(MATSDK_ANDROID_HTTP_CLIENT_RESOLVED "") +set(MATSDK_ANDROID_USES_CURL OFF) +set(MATSDK_ANDROID_USES_JAVA_HTTP OFF) +if(CMAKE_SYSTEM_NAME STREQUAL "Android") + if(MATSDK_ANDROID_HTTP_CLIENT_UPPER STREQUAL "AUTO") + set(MATSDK_ANDROID_HTTP_CLIENT_RESOLVED "JAVA") + else() + set(MATSDK_ANDROID_HTTP_CLIENT_RESOLVED "${MATSDK_ANDROID_HTTP_CLIENT_UPPER}") + endif() + + if(MATSDK_ANDROID_HTTP_CLIENT_RESOLVED STREQUAL "CURL") + set(MATSDK_ANDROID_USES_CURL ON) + elseif(MATSDK_ANDROID_HTTP_CLIENT_RESOLVED STREQUAL "JAVA") + set(MATSDK_ANDROID_USES_JAVA_HTTP ON) + endif() + message(STATUS "MATSDK_ANDROID_HTTP_CLIENT: ${MATSDK_ANDROID_HTTP_CLIENT} -> ${MATSDK_ANDROID_HTTP_CLIENT_RESOLVED}") +endif() + # Enable Azure Monitor / Application Insights end-point support option(BUILD_AZMON "Build for Azure Monitor" YES) @@ -309,14 +410,29 @@ endif() set(MATSDK_NEEDS_CURL OFF) if(PAL_IMPLEMENTATION STREQUAL "CPP11" AND NOT BUILD_IOS - AND (NOT CMAKE_SYSTEM_NAME STREQUAL "Android" OR MATSDK_USE_VCPKG_DEPS) + AND (NOT CMAKE_SYSTEM_NAME STREQUAL "Android" OR MATSDK_ANDROID_USES_CURL) AND NOT BUILD_APPLE_HTTP) set(MATSDK_NEEDS_CURL ON) add_definitions(-DHAVE_MAT_CURL_HTTP_CLIENT) - find_package(CURL REQUIRED) if(MATSDK_USE_VCPKG_DEPS) + # The TLS backend (OpenSSL/mbedTLS) is selected by the vcpkg port's + # curl-openssl (default) / curl-mbedtls features; the SDK just links libcurl. + # Force CONFIG mode so the vcpkg-provided CURLConfig (which defines the + # CURL::libcurl imported target) is used rather than the module FindCURL, + # which on some CMake versions does not define that target. + find_package(CURL CONFIG QUIET) + if(NOT TARGET CURL::libcurl) + message(FATAL_ERROR + "libcurl was not found. The vcpkg port provides the curl HTTP client " + "through the curl-openssl (default) or curl-mbedtls feature. Install " + "cpp-client-telemetry with its default features, or, under the [core,...] " + "form (which drops the default curl-openssl and system-sqlite features), " + "re-select a curl backend and a SQLite backend together, e.g. " + "[core,curl-openssl,system-sqlite] or [core,curl-mbedtls,minimal-sqlite].") + endif() list(APPEND LIBS CURL::libcurl) else() + find_package(CURL REQUIRED) # Prefer the imported target, which carries curl's include dirs and link # flags. Fall back to the find-module variables on CMake < 3.12, where # find_package(CURL) does not define CURL::libcurl. @@ -333,13 +449,46 @@ endif() # Dependency resolution (vcpkg mode vs vendored) ################################################################################################ if(MATSDK_USE_VCPKG_DEPS) - find_package(unofficial-sqlite3 CONFIG REQUIRED) - find_package(ZLIB REQUIRED) - find_package(nlohmann_json CONFIG REQUIRED) - message(STATUS "Using vcpkg-provided sqlite3, zlib, nlohmann-json") + if(APPLE) + # macOS/iOS ship libsqlite3 and libz as system libraries (the SDK's SPM + # distribution links them the same way), so the vcpkg sqlite3/zlib packages are + # not pulled there -- find the system ones via CMake's standard find modules. + find_package(SQLite3 REQUIRED) + find_package(ZLIB REQUIRED) + find_package(nlohmann_json CONFIG REQUIRED) + set(MATSDK_APPLE_SYSTEM_DEPS ON) + message(STATUS "Apple: using system SQLite3 + zlib; vcpkg-provided nlohmann-json") + else() + set(MATSDK_APPLE_SYSTEM_DEPS OFF) + # SQLite is provided by the private minimal build when MATSDK_MINIMAL_SQLITE is + # ON, so only require the external vcpkg sqlite3 package otherwise. + if(NOT MATSDK_MINIMAL_SQLITE) + find_package(unofficial-sqlite3 CONFIG QUIET) + if(NOT unofficial-sqlite3_FOUND) + message(FATAL_ERROR + "SQLite was not found and the minimal SQLite is not enabled. The vcpkg " + "port provides SQLite through one of two features: 'system-sqlite' " + "(default, links the external sqlite3 package) or 'minimal-sqlite' " + "(builds a private feature-stripped SQLite). Install " + "cpp-client-telemetry with its default features, or with " + "[core,system-sqlite] or [core,minimal-sqlite]. For a direct CMake build, pass " + "-DMATSDK_MINIMAL_SQLITE=ON or ensure unofficial-sqlite3 is discoverable.") + endif() + endif() + find_package(ZLIB REQUIRED) + find_package(nlohmann_json CONFIG REQUIRED) + if(MATSDK_MINIMAL_SQLITE) + message(STATUS "Using vcpkg-provided zlib, nlohmann-json; private minimal SQLite") + else() + message(STATUS "Using vcpkg-provided sqlite3, zlib, nlohmann-json") + endif() + endif() else() - # Include repo root to allow includes of vendored sqlite, zlib, and nlohmann - include_directories(${CMAKE_SOURCE_DIR}) + # Include repo root to allow includes of vendored sqlite, zlib, and nlohmann. + # Use CMAKE_CURRENT_SOURCE_DIR (this repo's root) rather than CMAKE_SOURCE_DIR + # so the vendored headers still resolve when the SDK is consumed as a subproject + # (add_subdirectory/FetchContent), where CMAKE_SOURCE_DIR is the consumer's root. + include_directories(${CMAKE_CURRENT_SOURCE_DIR}) message(STATUS "Using vendored sqlite3, zlib, nlohmann-json") endif() diff --git a/Solutions/conformance.props b/Solutions/conformance.props new file mode 100644 index 000000000..79e1ae779 --- /dev/null +++ b/Solutions/conformance.props @@ -0,0 +1,23 @@ + + + + + true + + + diff --git a/Solutions/net40/net40.vcxproj b/Solutions/net40/net40.vcxproj index ec55747a0..d21aede17 100644 --- a/Solutions/net40/net40.vcxproj +++ b/Solutions/net40/net40.vcxproj @@ -290,6 +290,7 @@ + diff --git a/Solutions/win32-dll/win32-dll.vcxproj b/Solutions/win32-dll/win32-dll.vcxproj index 026356130..b01b9e690 100644 --- a/Solutions/win32-dll/win32-dll.vcxproj +++ b/Solutions/win32-dll/win32-dll.vcxproj @@ -337,6 +337,7 @@ + diff --git a/Solutions/win32-lib/win32-lib.vcxproj b/Solutions/win32-lib/win32-lib.vcxproj index 90602e232..1b9fb6a7c 100644 --- a/Solutions/win32-lib/win32-lib.vcxproj +++ b/Solutions/win32-lib/win32-lib.vcxproj @@ -537,6 +537,7 @@ + diff --git a/Solutions/win32-mini-dll/win32-mini-dll.vcxproj b/Solutions/win32-mini-dll/win32-mini-dll.vcxproj index ce42cf020..fe923aee2 100644 --- a/Solutions/win32-mini-dll/win32-mini-dll.vcxproj +++ b/Solutions/win32-mini-dll/win32-mini-dll.vcxproj @@ -390,6 +390,7 @@ + diff --git a/Solutions/win32-mini-lib/win32-mini-lib.vcxproj b/Solutions/win32-mini-lib/win32-mini-lib.vcxproj index e720328fd..700623d89 100644 --- a/Solutions/win32-mini-lib/win32-mini-lib.vcxproj +++ b/Solutions/win32-mini-lib/win32-mini-lib.vcxproj @@ -678,6 +678,7 @@ + diff --git a/build-all-v142.bat b/build-all-v142.bat index 4ae0364bc..72776a978 100644 --- a/build-all-v142.bat +++ b/build-all-v142.bat @@ -2,4 +2,4 @@ set VSTOOLS_VERSION=vs2019 set PlatformToolset=v142 -call build-all.bat +call "%~dp0build-all-windows.bat" %* diff --git a/build-all-v143.bat b/build-all-v143.bat index 6ff46738d..8d5ebbfa9 100644 --- a/build-all-v143.bat +++ b/build-all-v143.bat @@ -2,4 +2,5 @@ set VSTOOLS_VERSION=vs2022 set PlatformToolset=v143 -call build-all.bat +set SKIP_NET40_BUILD=1 +call "%~dp0build-all-windows.bat" %* diff --git a/build-all-v145.bat b/build-all-v145.bat new file mode 100644 index 000000000..54f6b4e9a --- /dev/null +++ b/build-all-v145.bat @@ -0,0 +1,6 @@ +@echo off + +set VSTOOLS_VERSION=vs2026 +set PlatformToolset=v145 +set SKIP_NET40_BUILD=1 +call "%~dp0build-all-windows.bat" %* diff --git a/build-all-windows.bat b/build-all-windows.bat new file mode 100644 index 000000000..4ea3808e9 --- /dev/null +++ b/build-all-windows.bat @@ -0,0 +1,111 @@ +@echo off + +cd /d "%~dp0" +@setlocal ENABLEEXTENSIONS + +set CUSTOM_PROPS= +if not "%~1"=="" ( + if not exist "%~f1" ( + goto custom_props_missing + ) + if /I not "%~x1"==".props" ( + if /I not "%~x1"==".targets" ( + goto custom_props_invalid_type + ) + ) + set CUSTOM_PROPS="/p:ForceImportBeforeCppTargets=%~f1" + echo Using custom properties file for the build: + echo "/p:ForceImportBeforeCppTargets=%~f1" +) + +goto after_custom_props_validation + +:custom_props_missing +echo ERROR: Custom build input not found: %~1 +echo Pass an existing MSBuild .props or .targets file to ForceImportBeforeCppTargets. +exit /b 1 + +:custom_props_invalid_type +echo ERROR: Custom build input must be an MSBuild .props or .targets file: %~1 +echo Pass the MSBuild import file, not the CONFIG_CUSTOM_H header. +exit /b 1 + +:after_custom_props_validation +call tools\gen-version.cmd + +set NET40_MD_TARGETS=,net40:Rebuild +set NET40_SAMPLE_TARGETS=,Samples\cs\SampleCsNet40:Rebuild +if DEFINED SKIP_NET40_BUILD ( + echo Skipping legacy .NET Framework 4.0 targets. + set NET40_MD_TARGETS= + set NET40_SAMPLE_TARGETS= +) + +echo Update all public submodules... +git -c submodule."lib/modules".update=none submodule update --init --recursive + +if DEFINED GIT_PULL_TOKEN ( + rd /s /q lib\modules + git clone https://%GIT_PULL_TOKEN%:x-oauth-basic@github.com/microsoft/cpp_client_telemetry_modules.git lib\modules +) + +set GTEST_PATH=third_party\googletest +if NOT EXIST %GTEST_PATH%\CMakeLists.txt ( + git clone --depth 1 --branch release-1.12.1 https://github.com/google/googletest %GTEST_PATH% +) + +if NOT DEFINED SKIP_MD_BUILD ( + REM DLL and static /MD build + REM Release + call tools\RunMsBuild.bat Win32 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild%NET40_MD_TARGETS%,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild%NET40_SAMPLE_TARGETS%" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + call tools\RunMsBuild.bat x64 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild%NET40_MD_TARGETS%,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild%NET40_SAMPLE_TARGETS%" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + REM Debug + if NOT DEFINED SKIP_DEBUG_BUILD ( + call tools\RunMsBuild.bat Win32 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild%NET40_MD_TARGETS%,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + call tools\RunMsBuild.bat x64 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild%NET40_MD_TARGETS%,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + ) +) + +if NOT DEFINED SKIP_MT_BUILD ( + REM Static /MT build + REM Release + call tools\RunMsBuild.bat Win32 Release.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + call tools\RunMsBuild.bat x64 Release.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + REM Debug + if NOT DEFINED SKIP_DEBUG_BUILD ( + call tools\RunMsBuild.bat Win32 Debug.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + call tools\RunMsBuild.bat x64 Debug.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + ) +) + +if NOT DEFINED SKIP_ARM_BUILD ( + REM ARM DLL build + REM Release + call tools\RunMsBuild.bat ARM Release "zlib:Rebuild,sqlite-uwp:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + if NOT DEFINED SKIP_DEBUG_BUILD ( + REM Debug + call tools\RunMsBuild.bat ARM Debug "zlib:Rebuild,sqlite-uwp:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + ) +) + +if NOT DEFINED SKIP_ARM64_BUILD ( + REM ARM64 DLL build + REM Release + call tools\RunMsBuild.bat ARM64 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + if NOT DEFINED SKIP_DEBUG_BUILD ( + REM Debug + call tools\RunMsBuild.bat ARM64 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + ) +) diff --git a/build-all.bat b/build-all.bat index 5b535bc08..547594ff6 100644 --- a/build-all.bat +++ b/build-all.bat @@ -1,91 +1,3 @@ @echo off - -cd %~dp0 -@setlocal ENABLEEXTENSIONS - -set CUSTOM_PROPS= -if not "%~1"=="" ( - if not exist "%~f1" ( - goto custom_props_missing - ) - if /I not "%~x1"==".props" ( - if /I not "%~x1"==".targets" ( - goto custom_props_invalid_type - ) - ) - set CUSTOM_PROPS="/p:ForceImportBeforeCppTargets=%~f1" - echo Using custom properties file for the build: - echo %CUSTOM_PROPS% -) - -goto after_custom_props_validation - -:custom_props_missing -echo ERROR: Custom build input not found: %~1 -echo Pass an existing MSBuild .props or .targets file to ForceImportBeforeCppTargets. -exit /b 1 - -:custom_props_invalid_type -echo ERROR: Custom build input must be an MSBuild .props or .targets file: %~1 -echo Pass the MSBuild import file, not the CONFIG_CUSTOM_H header. -exit /b 1 - -:after_custom_props_validation -call tools\gen-version.cmd - -echo Update all public submodules... -git -c submodule."lib/modules".update=none submodule update --init --recursive - -if DEFINED GIT_PULL_TOKEN ( - rd /s /q lib\modules - git clone https://%GIT_PULL_TOKEN%:x-oauth-basic@github.com/microsoft/cpp_client_telemetry_modules.git lib\modules -) - -set GTEST_PATH=third_party\googletest -if NOT EXIST %GTEST_PATH%\CMakeLists.txt ( - git clone --depth 1 --branch release-1.12.1 https://github.com/google/googletest %GTEST_PATH% -) - -if NOT DEFINED SKIP_MD_BUILD ( - REM DLL and static /MD build - REM Release - call tools\RunMsBuild.bat Win32 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild,Samples\cs\SampleCsNet40:Rebuild" %CUSTOM_PROPS% - call tools\RunMsBuild.bat x64 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild,Samples\cs\SampleCsNet40:Rebuild" %CUSTOM_PROPS% - REM Debug - if NOT DEFINED SKIP_DEBUG_BUILD ( - call tools\RunMsBuild.bat Win32 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild" %CUSTOM_PROPS% - call tools\RunMsBuild.bat x64 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild" %CUSTOM_PROPS% - ) -) - -if NOT DEFINED SKIP_MT_BUILD ( - REM Static /MT build - REM Release - call tools\RunMsBuild.bat Win32 Release.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% - call tools\RunMsBuild.bat x64 Release.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% - REM Debug - if NOT DEFINED SKIP_DEBUG_BUILD ( - call tools\RunMsBuild.bat Win32 Debug.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% - call tools\RunMsBuild.bat x64 Debug.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% - ) -) - -if NOT DEFINED SKIP_ARM_BUILD ( - REM ARM DLL build - REM Release - call tools\RunMsBuild.bat ARM Release "zlib:Rebuild,sqlite-uwp:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild" %CUSTOM_PROPS% - if NOT DEFINED SKIP_DEBUG_BUILD ( - REM Debug - call tools\RunMsBuild.bat ARM Debug "zlib:Rebuild,sqlite-uwp:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild" %CUSTOM_PROPS% - ) -) - -if NOT DEFINED SKIP_ARM64_BUILD ( - REM ARM64 DLL build - REM Release - call tools\RunMsBuild.bat ARM64 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild" %CUSTOM_PROPS% - if NOT DEFINED SKIP_DEBUG_BUILD ( - REM Debug - call tools\RunMsBuild.bat ARM64 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild" %CUSTOM_PROPS% - ) -) +echo build-all.bat is a compatibility wrapper. Use build-all-windows.bat for Windows Visual Studio builds. +call "%~dp0build-all-windows.bat" %* diff --git a/cmake/MSTelemetryConfig.cmake.in b/cmake/MSTelemetryConfig.cmake.in index 5cf00c560..8d63ac1f0 100644 --- a/cmake/MSTelemetryConfig.cmake.in +++ b/cmake/MSTelemetryConfig.cmake.in @@ -2,19 +2,42 @@ include(CMakeFindDependencyMacro) -# Re-find dependencies that consumers need -find_dependency(unofficial-sqlite3 CONFIG) +# Re-find dependencies that consumers need. +# On Apple the SDK links the system libsqlite3 (SQLite::SQLite3); elsewhere it uses +# the vcpkg sqlite3 package unless a private minimal SQLite is bundled. +if(@MATSDK_APPLE_SYSTEM_DEPS@) + find_dependency(SQLite3) +elseif(NOT @MATSDK_BUNDLE_SQLITE@) + find_dependency(unofficial-sqlite3 CONFIG) +endif() find_dependency(ZLIB) find_dependency(nlohmann_json CONFIG) # Curl is re-found only when the SDK was built with the curl HTTP client -# (Linux, Android via vcpkg, and macOS built without Apple HTTP). -# Windows (WinInet), iOS, and macOS-with-Apple-HTTP do not link curl. +# (Linux, explicit Android curl builds, and macOS built without Apple HTTP). +# Windows (WinInet), default Android Java/JNI HTTP, iOS, and +# macOS-with-Apple-HTTP do not link curl. # We bake the build-time decision into a boolean rather than re-deriving it, # because the macOS BUILD_APPLE_HTTP choice can't be inferred from # CMAKE_SYSTEM_NAME alone. if(@MATSDK_NEEDS_CURL@) - find_dependency(CURL) + # Force CONFIG mode so the vcpkg-provided CURLConfig (which defines the + # CURL::libcurl imported target referenced by MSTelemetryTargets.cmake) is + # used, rather than module-mode FindCURL, which on some CMake versions does + # not define that target. + find_dependency(CURL CONFIG) +endif() + +if("@MATSDK_ANDROID_HTTP_CLIENT_RESOLVED@" STREQUAL "") + set(MSTelemetry_ANDROID_HTTP_CLIENT "NONE") +else() + set(MSTelemetry_ANDROID_HTTP_CLIENT "@MATSDK_ANDROID_HTTP_CLIENT_RESOLVED@") +endif() +if(MSTelemetry_ANDROID_HTTP_CLIENT STREQUAL "JAVA") + set(MSTelemetry_ANDROID_JAVA_SOURCE_DIR + "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_DATADIR@/cpp-client-telemetry/android/java") +else() + set(MSTelemetry_ANDROID_JAVA_SOURCE_DIR "") endif() # Pthreads are needed on Linux and Android (POSIX threading) diff --git a/docs/building-custom-SKU.md b/docs/building-custom-SKU.md index 0668fbe3f..a68d6a681 100644 --- a/docs/building-custom-SKU.md +++ b/docs/building-custom-SKU.md @@ -39,12 +39,12 @@ Build recipe must contain the following preprocessor definitions: Command: ```console -build-all.bat %CD%\Solutions\build.compact.props +build-all-windows.bat %CD%\Solutions\build.compact.props ``` produces a custom compact SDK build. -The argument passed to `build-all.bat` must be an MSBuild `.props` or `.targets` file that sets the required preprocessor definitions. Do not pass the `config-*.h` header directly to `ForceImportBeforeCppTargets`. +The argument passed to `build-all-windows.bat` must be an MSBuild `.props` or `.targets` file that sets the required preprocessor definitions. Do not pass the `config-*.h` header directly to `ForceImportBeforeCppTargets`. `build-all.bat` remains as a compatibility wrapper for existing automation. How it works: diff --git a/docs/building-with-vcpkg.md b/docs/building-with-vcpkg.md index b736ba3c5..fed2dbdcf 100644 --- a/docs/building-with-vcpkg.md +++ b/docs/building-with-vcpkg.md @@ -1,6 +1,6 @@ # Building 1DS C++ SDK with vcpkg -[vcpkg](https://vcpkg.io/) is a Microsoft cross-platform open source C++ package manager. Onboarding instructions for Windows, Linux and Mac OS X [available here](https://docs.microsoft.com/en-us/cpp/build/vcpkg). This document assumes that the customer build system is already configured to use vcpkg ([getting started guide](https://learn.microsoft.com/en-us/vcpkg/get_started/overview)). 1DS C++ SDK maintainers provide a build recipe, `cpp-client-telemetry` port or CONTROL file for vcpkg. The mainline vcpkg repo is refreshed to point to latest stable open source release of 1DS C++ SDK. +[vcpkg](https://vcpkg.io/) is a Microsoft cross-platform open source C++ package manager. Onboarding instructions for Windows, Linux and Mac OS X [available here](https://docs.microsoft.com/en-us/cpp/build/vcpkg). This document assumes that the customer build system is already configured to use vcpkg ([getting started guide](https://learn.microsoft.com/en-us/vcpkg/get_started/overview)). The `cpp-client-telemetry` port is published in the official vcpkg registry, so it can be consumed directly with no overlay or extra configuration. Maintainers refresh the registry to point to the latest stable open source release of the 1DS C++ SDK on each release. The port provides the core SDK — the `MSTelemetry::mat` target and its public C++ headers. The optional Microsoft-proprietary modules (Privacy Guard, @@ -16,7 +16,8 @@ git clone --recurse-submodules https://github.com/microsoft/cpp_client_telemetry ### Installing from the vcpkg registry -Once a new port has been accepted into the official vcpkg registry, install with: +The `cpp-client-telemetry` port is available in the [official vcpkg registry](https://github.com/microsoft/vcpkg/tree/master/ports/cpp-client-telemetry), +so you can install it directly — no overlay or extra configuration required: ```console vcpkg install cpp-client-telemetry @@ -26,8 +27,9 @@ That's it! The package should be compiled for the current OS. ### Installing from the overlay port (development / pre-release) -Before the port is published, or to test local changes, use the overlay port -shipped in this repository: +The overlay port shipped in this repository is for **development only** — use it +to test local changes to the port, or a newer SDK revision, before they are +published to the registry: ```console git clone https://github.com/microsoft/cpp_client_telemetry @@ -137,24 +139,94 @@ scaffolding and not part of the published package.) Supported triplets: `arm64-android`, `arm-neon-android`, `x64-android`, `x86-android`. +#### Android HTTP transport + +The CMake option `MATSDK_ANDROID_HTTP_CLIENT` selects the Android HTTP transport: + +| Value | Behavior | +| ----- | -------- | +| `AUTO` | Default. Uses the Android Java/JNI transport. | +| `CURL` | Builds the native libcurl transport. This is an explicit escape hatch and requires one Android curl backend feature. | +| `JAVA` | Builds `HttpClient_Android`, which calls the Android Java bridge via JNI. | + +For vcpkg, Android uses Java transport by default: + +```json +{ + "dependencies": [ + { + "name": "cpp-client-telemetry", + "default-features": false, + "features": ["system-sqlite"] + } + ] +} +``` + +To opt into native curl on Android, select exactly one Android curl backend: + +```json +{ + "dependencies": [ + { + "name": "cpp-client-telemetry", + "default-features": false, + "features": ["android-curl-openssl", "system-sqlite"] + } + ] +} +``` + +Use `android-curl-mbedtls` in place of `android-curl-openssl` for the mbedTLS +backend. + +When Java transport is selected, the package installs the bridge sources under: + +```text +share/cpp-client-telemetry/android/java/com/microsoft/applications/events/ +``` + +The installed bridge contains `HttpClient.java` and `HttpClientRequest.java`. +Consumers are responsible for compiling those Java sources into their Android +application/AAR and constructing `com.microsoft.applications.events.HttpClient` +so it initializes the native `HttpClient_Android` singleton before telemetry is +uploaded. The bridge imports AndroidX annotations (`@Keep`, `@NonNull`, +`@Nullable`, `@RequiresApi`), so ensure `androidx.annotation:annotation` is on +the Java compile classpath, for example as a Gradle `compileOnly` or +`implementation` dependency. + ## Dependencies The vcpkg port automatically resolves the following dependencies: | Dependency | vcpkg Package | CMake Target | Platforms | | -------------- | --------------- | --------------------------------- | ------------------ | -| SQLite3 | `sqlite3` | `unofficial::sqlite3::sqlite3` | All | -| zlib | `zlib` | `ZLIB::ZLIB` | All | +| SQLite3 | `sqlite3` | `unofficial::sqlite3::sqlite3` | Non-Apple (default; see `minimal-sqlite`). **macOS/iOS link the system `libsqlite3`** (`SQLite::SQLite3`) | +| zlib | `zlib` | `ZLIB::ZLIB` | Non-Apple. **macOS/iOS link the system `libz`** | | nlohmann JSON | `nlohmann-json` | `nlohmann_json::nlohmann_json` | All | -| libcurl | `curl[openssl]` | `CURL::libcurl` | Non-Windows, non-Apple | +| libcurl | `curl[openssl]` or `curl[mbedtls]` | `CURL::libcurl` | Linux by default; Android only when `android-curl-openssl` or `android-curl-mbedtls` is selected | + +On **macOS/iOS** the SDK links the OS-provided `libsqlite3` and `libz` (the same +system libraries the SDK's Swift Package links), so the vcpkg `sqlite3` and `zlib` +packages are not pulled there — those binaries carry no bundled SQLite/zlib. +(`minimal-sqlite` therefore has no effect on Apple.) + +The external `sqlite3` package is provided by the default `system-sqlite` +feature. The `minimal-sqlite` feature replaces it with a private, feature-stripped +SQLite built from the SDK's vendored amalgamation — see +[Build a private minimal SQLite](#build-a-private-minimal-sqlite-minimal-sqlite-feature). + +On Linux, libcurl is provided by the default `curl-openssl` feature; +`curl-mbedtls` swaps in the mbedTLS backend — see +[Choose the Linux HTTP client / TLS backend](#choose-the-linux-http-client--tls-backend-largest-lever-on-linux). Windows and macOS/iOS use platform-native HTTP clients (WinInet and -NSURLSession respectively). Android vcpkg consumers use native libcurl because -the Java-backed `HttpClient_Android` singleton is initialized by the repo's -Android Gradle/AAR flow, not by standalone native vcpkg consumers. +NSURLSession respectively). Android defaults to the platform Java/JNI HTTP +bridge; native curl is available only through explicit `android-curl-*` features. > **Note (Windows):** The port targets the MSVC/`WIN32` PAL on Windows, which -> uses WinInet, so `curl` is declared for `linux | android` only. A MinGW / +> uses WinInet, so the default `curl` dependency is declared for Linux only +> (Android has separate explicit `android-curl-*` features). A MinGW / > non-MSVC Windows triplet — or forcing `-DPAL_IMPLEMENTATION=CPP11` on Windows — > selects the curl HTTP client, which the port does not provision on Windows > (broadening `curl` to `windows` would pull an unused curl into every MSVC @@ -190,12 +262,190 @@ will automatically use the optimized zlib-ng build. > zlib. When using `ZLIB_COMPAT=ON`, ensure all dependencies resolve to > zlib-ng rather than mixing stock zlib and zlib-ng. +## Reducing binary footprint + +This section applies when the SDK is linked **statically** into your binary +(the default for the `*-static` vcpkg triplets) — most footprint control then +lives on *your* side of the link. If you instead consume a **dynamic** `mat` +(e.g. the default `x64-windows` triplet, or `BUILD_SHARED_LIBS=ON`), the runtime +ships as its own `mat.dll` / `libmat.so` / `libmat.dylib`; the SDK's own +`-fvisibility=hidden` and `/Gy /Gw` already trim its exported symbol table, and +the consumer-side linker options below are specific to the static-link case. + +### Enable linker dead-stripping (largest lever) + +The SDK is compiled with function-level linking (`/Gy /Gw` on MSVC, +`-ffunction-sections -fdata-sections` on GCC/Clang) so that **your** linker can +discard SDK code you never reference. Make sure your final link enables it: + +- **MSVC:** `/OPT:REF` (drop unreferenced functions/data) and `/OPT:ICF` (fold + identical COMDATs). These are on by default for Release, **but `/DEBUG` flips + their default to off** (`/OPT:NOREF,NOICF`, per the MSVC `/OPT` docs) — so if + you ship PDBs, re-enable them explicitly. `/OPT:REF` is also incompatible with + incremental linking, so set `/INCREMENTAL:NO`: + + ```cmake + target_link_options(your_target PRIVATE + $<$,$>:/OPT:REF> + $<$,$>:/OPT:ICF> + $<$,$>:/INCREMENTAL:NO>) + ``` + +- **GCC / Clang:** link with `-Wl,--gc-sections`. +- **Apple (clang):** link with `-Wl,-dead_strip`. + +This is by far the largest lever — on a static `x64-windows-static` Release link +it can roughly halve the binary. The SDK's `/Gy /Gw` flags only *enable* this; +the stripping happens at your link. Keep the SDK a static dependency linked +*into* your binary: if you re-export its API across your own DLL boundary, the +export table pins its symbols and defeats `/OPT:REF`. + +### Choose the Linux HTTP client / TLS backend (largest lever on Linux) + +On Linux the built-in HTTP client is libcurl, and curl's TLS backend dominates +the SDK's footprint. (Windows uses WinInet, Apple uses NSURLSession, and Android +uses the Java/JNI bridge by default, so this section does not apply there.) The +port exposes the Linux TLS backend as two mutually-exclusive features; pick the +one that matches what your application already has: + +| Feature | Transport | Approx. stripped size¹ | Use when | +| ------- | --------- | ---------------------- | -------- | +| `curl-openssl` (default) | libcurl + OpenSSL | ~10.6 MB | your app already links OpenSSL (share it) | +| `curl-mbedtls` | libcurl + mbedTLS | ~4.4 MB | your app has no HTTP/TLS stack of its own | + +¹ Rough sizes of a minimal Linux consumer **without** consumer-side dead-stripping +(worst case); enabling `-Wl,--gc-sections` at your link reduces them. Your numbers +depend on triplet, dead-stripping, and what else shares those libraries. + +To select **mbedTLS on Linux**, two things are required in *your top-level* +manifest: + +```json +{ + "dependencies": [ + { + "name": "cpp-client-telemetry", + "default-features": false, + "features": [ "minimal-sqlite", "curl-mbedtls" ] + }, + { "name": "curl", "default-features": false, "features": [ "mbedtls" ] } + ] +} +``` + +1. `"default-features": false` (the `[core,...]` form) drops **all** of the SDK's + default features -- both `curl-openssl` *and* `system-sqlite` -- so the SDK no + longer *requests* OpenSSL. Because it also drops `system-sqlite`, you must + re-select a SQLite backend (`minimal-sqlite` above, or `system-sqlite`); + otherwise the SDK configure step fails with no SQLite feature selected. +2. The explicit top-level `curl` entry is also needed because vcpkg honors curl's + own `"default-features": false` **only for top-level dependencies** — curl's + default `ssl` feature (which pulls OpenSSL on Linux) and `non-http` are + installed transitively otherwise. With both, curl resolves to `curl[core,mbedtls]` + and OpenSSL is not built; with only the feature, you get + `curl[mbedtls,ssl,openssl,non-http]` (mbedTLS *and* OpenSSL). This recipe is + verified with `vcpkg install --dry-run`. + +The default install (no features specified) keeps `curl-openssl` and works out of +the box on Linux. Android uses the Java/JNI HTTP bridge by default; use +`android-curl-openssl` or `android-curl-mbedtls` only when you explicitly want +the native curl Android escape hatch. + +### Drop unused SQLite features (json1) + +The SDK uses SQLite only for offline event storage — plain tables and indexes, +with no JSON, FTS, R*Tree, or virtual-table features. This in-repo overlay port +already requests `sqlite3` with `default-features: false` on its dependency edge +(the published registry port will follow once this change is upstreamed). + +vcpkg unions feature requests across the whole dependency graph, and a +transitive opt-out alone is **not** enough: you must **also** request `sqlite3` +with `default-features: false` in your own top-level manifest to actually omit +`json1` (which compiles SQLite with `SQLITE_OMIT_JSON`, ~50 KB smaller on a +static `x64-windows-static` Release build): + +```json +{ + "dependencies": [ + "cpp-client-telemetry", + { "name": "sqlite3", "default-features": false } + ] +} +``` + +If any package in your build (or your own code) needs SQLite's JSON functions, +request `sqlite3[json1]` instead and the extension is restored for the whole +graph. + +### Build a private minimal SQLite (`minimal-sqlite` feature) + +For a larger, self-contained reduction, the port can compile a private, +feature-stripped SQLite directly from the SDK's vendored amalgamation instead of +linking the external `sqlite3` package at all. The SDK uses SQLite only for its +offline event-storage cache (plain tables and indexes, transactions, WAL, +autovacuum/`VACUUM`, a few PRAGMAs, and one custom UTF-8 SQL function), so this +build omits the unused SQLite subsystems — `SQLITE_OMIT_JSON` plus load-extension, +shared-cache, deprecated APIs, authorization, EXPLAIN, introspection pragmas, +deserialize, and more. The result is **~10% smaller SQLite code** (`.text`) and +**~13% smaller** as a stripped object, and it drops the external `sqlite3` +dependency from your graph entirely. + +Enable it through the vcpkg feature: + +```json +{ + "dependencies": [ + { + "name": "cpp-client-telemetry", + "default-features": false, + "features": [ "minimal-sqlite", "curl-openssl" ] + } + ] +} +``` + +Use the `[core,minimal-sqlite]` form (here, `"default-features": false` is the +`[core]` part) so the default `system-sqlite` feature — and its `sqlite3` +dependency — is dropped. Because `[core]` drops **all** defaults, Linux examples +also re-select `curl-openssl`; on Linux the built-in curl client requires a TLS +backend, so omitting it would fail to configure (swap in `curl-mbedtls` for the +smaller mbedTLS backend). Android does not need a curl feature unless you +explicitly opt into `android-curl-openssl` or `android-curl-mbedtls`. +Requesting `minimal-sqlite` *without* `[core]` still pulls in the default +`system-sqlite`; that is harmless (the external `sqlite3` is installed but +unused) but does not save the dependency. + +For a plain (non-vcpkg) CMake build, pass the option directly: + +```bash +cmake -DMATSDK_MINIMAL_SQLITE=ON .. +``` + +The strip is **amalgamation-safe**: it changes no SQLite grammar/parser, so no +code generation is required. All offline storage features the SDK relies on (WAL, +autovacuum, `VACUUM`, PRAGMAs, the custom UTF-8 function, blobs, 64-bit integers, +transactions) are retained, and the SDK's offline-storage unit tests pass +unchanged against the minimal build. + +> **Caveat — symbol visibility when linking statically.** The private SQLite keeps +> SQLite's default `sqlite3_*` symbol names. For a **shared** `mat` +> (`mat.dll` / `libmat.so` / `libmat.dylib`), those symbols are hidden by the +> SDK's `-fvisibility=hidden`, so there is no conflict. For a **static** `mat`, +> the minimal SQLite is installed and exported as a separate +> `MSTelemetry::sqlite3_bundled` archive that links into your binary; if **any** +> part of the final static link — your own code *or another dependency* — also +> pulls in SQLite, the duplicate `sqlite3_*` symbols will collide at link time. In +> that case, prefer the default `system-sqlite` feature so the whole graph shares a +> single SQLite. + ## How It Works: MATSDK_USE_VCPKG_DEPS When the SDK detects it is being built via vcpkg (by checking for `VCPKG_TOOLCHAIN` or `VCPKG_TARGET_TRIPLET`), it automatically sets `MATSDK_USE_VCPKG_DEPS=ON`. This switches dependency resolution from -vendored sources to vcpkg-provided packages via `find_package()`. +vendored sources to vcpkg-provided packages via `find_package()`. Android HTTP +transport selection is controlled separately by `MATSDK_ANDROID_HTTP_CLIENT`, +which defaults to `JAVA` on Android. You can also set this explicitly for custom CMake workflows: diff --git a/docs/cpp-start-android.md b/docs/cpp-start-android.md index 0f979bcda..2281a5e31 100644 --- a/docs/cpp-start-android.md +++ b/docs/cpp-start-android.md @@ -14,6 +14,8 @@ The Gradle wrapper in ```android_build``` builds two modules, ```app``` and ```m On Android, there are two database implementations to choose from. By default (the main branch on Github), the SDK will use the Android-supported androidx.Room database package. This reduces APK size because we don't need to compile and link in a copy of SQLite in native code (SQLite is hundreds of kB per ABI of APK file size). Room does have a slight CPU performance disadvantage since database transactions cross the JNI boundary when native code uses it. If you wish to change from Room to the native SQLite implementation, you should change the two module ```build.gradle``` files (app and maesdk). In those files, you will see an argument to CMake to select Room: ```"-DUSE_ROOM=1"```. Change this to ```"-DUSE_ROOM=0``` to select the native SQLite. +When using the Room implementation, the ```maesdk``` AAR brings ```androidx.room``` as a transitive dependency, pinned in ```lib/android_build/maesdk/build.gradle``` (currently ```2.8.4```). The SDK's native (JNI) code is compiled and tested against this version and the Room-generated schema. Because Gradle resolves a single ```androidx.room``` version for the entire app, if your app (or one of its dependencies) selects a different version, the SDK's native code runs against it. **Do not force ```androidx.room``` below the version the SDK is built against**, and prefer aligning your app on the bundled version (or a compatible newer one). A significantly different Room version can change the shape of query results that cross the JNI boundary and has historically caused native crashes in record retrieval (issue #1227); the SDK now guards against null results defensively, but version alignment avoids subtle behavior differences. + The Room database implementation adds one additional initialization requirement, since it needs a pointer to the JVM and an object reference to the application context. See below (4.5) for the required call to either ```connectContext``` (in Java) or ```ConnectJVM``` (in C++) to set this up. If you are building on Windows, this helper script [build-android.cmd](../build-android.cmd) is provided to illustrate how to deploy the necessary SDK and NDK dependencies. Once you installed the necessary dependencies, you may use Android Studio IDE for local builds. See [ide.cmd](../lib/android_build/ide.cmd) that shows how to build the project from IDE. The `app` project (`maesdktest`) allows to build and run all SDK tests on either emulator or real Android device. While the tests are running, you can monitor the test results in logcat output. diff --git a/docs/cpp-start-windows.md b/docs/cpp-start-windows.md index ef5850d84..6f6189056 100644 --- a/docs/cpp-start-windows.md +++ b/docs/cpp-start-windows.md @@ -16,17 +16,24 @@ If your project requires the Universal Telemetry Client (a.k.a. UTC) to send tel ## **Windows prerequisites and dependencies for building from source** -* Visual Studio 2019 or 2022 (2022 is recommended). +* Visual Studio 2019, 2022, or 2026 (2022 or newer is recommended). * C++ Dev Tools ## **Option 1: Build the SDK from source using Visual Studio** * Open the *cpp_client_telemetry/Solutions/MSTelemetrySDK.sln* solution in Visual Studio. -* Alternatively you can use *build-all.bat* located in workspace root folder to build from command line +* Alternatively, build from the workspace root with the script that matches your Visual Studio toolset: + * Visual Studio 2019: `build-all-v142.bat` + * Visual Studio 2022: `build-all-v143.bat` + * Visual Studio 2026: `build-all-v145.bat` + +The version-specific scripts set `VSTOOLS_VERSION` and `PlatformToolset` before calling `build-all-windows.bat`, which builds the Windows Visual Studio solution matrix. `build-all.bat` remains as a compatibility wrapper for existing automation; if you call either script directly, set both values yourself so `tools\vcvars.cmd` selects the same Visual Studio installation as your requested toolset. + +Visual Studio 2022 and newer may report the legacy .NET Framework 4.0 projects (`net40` and `SampleCsNet40`) as unsupported. They are only needed for the legacy .NET Framework wrapper; the VS2022 and VS2026 command-line wrappers skip those projects, and you can unload them in the IDE when building the native SDK. If your build fails, then you most likely missing the following optional Visual Studio components: -* ATL support +* MFC/ATL support (for example, `SampleCppMini` uses static MFC in its Visual Studio project) * ARM64 support * Spectre mitigation libraries diff --git a/docs/sharing-a-single-sdk-runtime.md b/docs/sharing-a-single-sdk-runtime.md new file mode 100644 index 000000000..10e69746d --- /dev/null +++ b/docs/sharing-a-single-sdk-runtime.md @@ -0,0 +1,164 @@ +# Sharing one SDK runtime across several modules in a process + +When more than one module in a single process links this SDK — for example an +application that loads several plug-ins or libraries, each of which uses 1DS — +the easy default (every module statically embeds the SDK) has two costs: + +1. **Size.** The SDK (plus its bundled SQLite/zlib) is duplicated once per module. +2. **Duplicated global state.** Each static copy has its *own* default + `LogManager`, HTTP transport, offline SQLite cache, and upload threads. They do + not share a pipeline, and multiple writers to the same offline-cache path will + corrupt it. + +This document describes how to ship **one** shared SDK runtime (`mat.dll` / +`libmat.so` / `libmat.dylib`) that every module imports, so there is a single +copy on disk and a single set of process-global state. + +There are two ways to consume the shared runtime. **The C API is strongly +recommended** because it removes the fragile C++/CRT ABI coupling between modules. + +--- + +## Option 1 (recommended): consume the stable C API + +The SDK ships a flat **C ABI** in [`mat.h`](../lib/include/public/mat.h). Every +`evt_*` entry point (`evt_open`, `evt_log`, `evt_flush`, `evt_upload`, +`evt_pause`, `evt_resume`, `evt_close`, `evt_configure`, …) is a `static inline` +wrapper that marshals its arguments into a POD struct and calls through a single +exported `__cdecl` symbol, `evt_api_call_default`. + +Consequences that make this the robust choice: + +* **Only one symbol crosses the module boundary, and no C++/STL type does.** The + request is a plain C struct, so there is *no* requirement that the modules and + the shared runtime agree on the C++ standard library ABI (`/MD`, + `_ITERATOR_DEBUG_LEVEL`, MSVC toolset/STL version, libstdc++ vs libc++, + `_GLIBCXX_USE_CXX11_ABI`, …). A 1DS version bump does not force every module to + rebuild in lockstep against an identical toolchain. +* **It does not *require* `__declspec(dllimport)` to link.** A plain C function + resolves through the shared library's import lib even without `dllimport`, so + the C API works across the boundary regardless. Consumers that link the shared + `MSTelemetry::mat` target do get `dllimport` applied automatically (via the + `MATSDK_IMPORT_LIB` interface define this PR adds); for a C function that is a + harmless calling-convention optimization, not a requirement. + +Each module includes `mat.h`, links the one shared runtime, and uses its own +tenant/source. You still pin the **same SDK version** in every module (so the +request/struct layout matches), but you avoid the C++ ABI lockstep entirely. + +## Option 2: consume the C++ API from a shared library + +All modules `find_package(MSTelemetry CONFIG REQUIRED)` and link +`MSTelemetry::mat` (resolving to the import lib); none statically embed the SDK. + +The C++ public API passes C++ standard-library types (`std::string`, `std::map`, +…) across the module boundary, so **every module and the shared runtime must +share one C++ ABI**. If they do not, you get heap corruption / undefined +behavior. Pin all of the following identically: + +| Axis | Requirement | +|------|-------------| +| **CRT linkage (Windows)** | Dynamic CRT (`/MD`, `/MDd` for Debug) everywhere — never `/MT`, and never mix Debug/Release CRT across the boundary. (vcpkg: `VCPKG_CRT_LINKAGE dynamic`.) | +| **STL / iterator debug** | One compiler + STL, one build config. `_ITERATOR_DEBUG_LEVEL` must match (Release `0` vs Debug `2`) — a Release consumer + Debug runtime is a silent layout mismatch. | +| **Toolset** | One MSVC toolset across all binaries (the v14x toolsets share an STL ABI, but don't mix major versions); or one libstdc++/libc++ with the same `_GLIBCXX_USE_CXX11_ABI`. | +| **Language / model** | Same `/std:c++NN`, same `/EHsc` exception model, same architecture, no overridden struct packing. | +| **SDK build options** | Same SDK feature/version selection in every module's manifest — different features mean different headers, hence a different ABI even at the same version. | + +Because the C++ ABI must match exactly across separately built and separately +versioned modules, this option is materially more brittle than the C API. Prefer +Option 1 unless you specifically need the C++ surface and control all modules' +toolchains. + +--- + +## Building the shared runtime with vcpkg (per-port linkage) + +A common requirement is "share *this* SDK, but keep everything else statically +linked (no DLL forest)". Override the library linkage **per port** in your +triplet so only this SDK goes dynamic: + +```cmake +set(VCPKG_CRT_LINKAGE dynamic) +if(PORT STREQUAL "cpp-client-telemetry") + set(VCPKG_LIBRARY_LINKAGE dynamic) # mat.dll / libmat.so / libmat.dylib + import lib +else() + set(VCPKG_LIBRARY_LINKAGE static) +endif() +``` + +The port honors `VCPKG_LIBRARY_LINKAGE` / `BUILD_SHARED_LIBS` and emits the +shared `mat` plus its import lib and the `MSTelemetry` CMake config package. + +### Pin one version across all modules + +All modules must compile against identical SDK headers. Pin the same +`cpp-client-telemetry` version and `builtin-baseline` (or a shared version +override) in every module's manifest, and ideally build all artifacts in the same +CI job/container with the same toolchain image. Most ABI drift comes from +separate modules quietly building on different agents. + +--- + +## How the SDK decorates its public symbols + +The SDK has no `.def` file; on Windows, exporting/importing is driven entirely by +`MATSDK_LIBABI` in [`ctmacros.hpp`](../lib/include/public/ctmacros.hpp), which the +build ties to the actual linkage: + +* **Shared build:** the SDK is compiled with `MATSDK_SHARED_LIB` + (`__declspec(dllexport)`), and the installed `MSTelemetry::mat` target carries + an `INTERFACE` definition of `MATSDK_IMPORT_LIB`, so consumers that + `find_package` + link automatically get `__declspec(dllimport)` — no + consumer-side configuration required. +* **Static build:** nothing is decorated, so the SDK's public symbols are not + re-exported by a consumer DLL that absorbs the static lib. + +On non-Windows platforms the SDK is built with `-fvisibility=hidden` and the +public API is marked `__attribute__((visibility("default")))`, so only the public +API (including the C API) is exported from the shared object. + +--- + +## Coordinate the single runtime's lifetime + +One shared runtime means **one** set of process-global state. Decide ownership: + +* **Recommended — single owner.** The top-level module initializes and tears down + the SDK (`LogManager::Initialize` / `FlushAndTeardown`, or `evt_open` / + `evt_close`). Other modules obtain loggers (their own tenant/source) but never + initialize or tear down. This avoids teardown-ordering crashes. +* **Alternative — named instances.** `LogManagerProvider::CreateLogManager(id)` + gives each module its own instance/tenant/config sharing the one transport; then + you need a last-one-out teardown refcount and **distinct offline-cache paths** + (one shared path with multiple writers corrupts it). +* Teardown must happen exactly once, **last**, after every module has stopped + logging. + +--- + +## Ship exactly one copy on the loader path + +Place a single runtime where every module finds it: + +* **Windows:** the same directory as the consumers (or side-by-side assembly). +* **Linux:** `RPATH=$ORIGIN` so every module resolves the one copy. +* **macOS:** a stable install name, `@rpath/libmat.dylib`. + +Make exactly one package own and ship the SDK runtime; the others declare a +dependency rather than bundling their own. If several packages each ship their +own copy, which one loads is path-order luck — and if their versions differ, you +are back to an ABI mismatch even with "one" DLL. + +--- + +## Validate + +* **Dependency present, definitions absent.** `dumpbin /dependents` (Windows), + `ldd` (Linux), `otool -L` (macOS) on each consumer should show a dependency on + the one `mat` module; `dumpbin /exports` (or `nm -D`) on a consumer should show + it imports — not defines — the SDK symbols. +* **One copy at runtime.** Process Explorer / `/proc//maps` / `vmmap` should + map the `mat` module exactly once; there should be one offline-cache file. +* **(C++ option) CRT/STL smoke test.** Have a consumer pass a `std::string` event + property into the SDK and read it back. A `/MD` vs `/MT` or `_ITERATOR_DEBUG_LEVEL` + mismatch typically crashes immediately (especially in Debug). diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 584c678ec..13b4d46d4 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -6,8 +6,10 @@ cmake_policy(SET CMP0063 NEW) # to downstream consumers via find_package() (see target_include_directories below). include_directories( . ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/include/public ${CMAKE_CURRENT_SOURCE_DIR}/include/mat ${CMAKE_CURRENT_SOURCE_DIR}/pal ${CMAKE_CURRENT_SOURCE_DIR}/utils ${CMAKE_CURRENT_SOURCE_DIR}/modules/exp ${CMAKE_CURRENT_SOURCE_DIR}/modules/dataviewer ${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard ${CMAKE_CURRENT_SOURCE_DIR}/modules/liveeventinspector ${CMAKE_CURRENT_SOURCE_DIR}/modules/cds ${CMAKE_CURRENT_SOURCE_DIR}/modules/signals ${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer ) -# Legacy builds may need system-installed deps from /usr/local/include -if(NOT MATSDK_USE_VCPKG_DEPS) +# Legacy builds may need system-installed deps from /usr/local/include. Excluded on +# iOS: /usr/local/include is a host (macOS) path, and injecting it into an iOS +# cross-compile's search path can shadow the iOS SDK's own headers. +if(NOT MATSDK_USE_VCPKG_DEPS AND NOT CMAKE_SYSTEM_NAME STREQUAL "iOS") include_directories(/usr/local/include) endif() @@ -195,7 +197,7 @@ if(PAL_IMPLEMENTATION STREQUAL "CPP11") list(APPEND SRCS pal/posix/NetworkInformationImpl_Android.cpp ) - if(MATSDK_USE_VCPKG_DEPS) + if(MATSDK_ANDROID_USES_CURL) list(APPEND SRCS http/HttpClient_Curl.cpp http/HttpClient_Curl.hpp @@ -216,7 +218,7 @@ if(PAL_IMPLEMENTATION STREQUAL "CPP11") endif() if(APPLE AND BUILD_OBJC_WRAPPER) message(STATUS "Include ObjC Wrappers") - list(APPEND SRCS + set(OBJC_WRAPPER_SRCS ../wrappers/obj-c/ODWLogger.mm ../wrappers/obj-c/ODWLogManager.mm ../wrappers/obj-c/ODWEventProperties.mm @@ -227,22 +229,23 @@ if(PAL_IMPLEMENTATION STREQUAL "CPP11") ../wrappers/obj-c/ODWSanitizerInitConfig.mm ) if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/dataviewer/") - list(APPEND SRCS + list(APPEND OBJC_WRAPPER_SRCS ../wrappers/obj-c/ODWDiagnosticDataViewer.mm ) endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard/" AND BUILD_PRIVACYGUARD) set(MATSDK_OBJC_PRIVACYGUARD_AVAILABLE ON) - list(APPEND SRCS + list(APPEND OBJC_WRAPPER_SRCS ../wrappers/obj-c/ODWPrivacyGuard.mm ) endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer/" AND BUILD_SANITIZER) set(MATSDK_OBJC_SANITIZER_AVAILABLE ON) - list(APPEND SRCS + list(APPEND OBJC_WRAPPER_SRCS ../wrappers/obj-c/ODWSanitizer.mm ) endif() + list(APPEND SRCS ${OBJC_WRAPPER_SRCS}) endif() if(APPLE AND BUILD_SWIFT_WRAPPER) @@ -271,7 +274,7 @@ elseif(PAL_IMPLEMENTATION STREQUAL "WIN32") if(NOT MATSDK_USE_VCPKG_DEPS) include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/../zlib ${CMAKE_CURRENT_SOURCE_DIR}/../sqlite) endif() -add_definitions(-D_UNICODE -DUNICODE -DWIN32 -DMATSDK_PLATFORM_WINDOWS=1 -DMATSDK_SHARED_LIB=1 -D_UTC_SDK -DUSE_BOND -D_WINDOWS -D_USRDLL -DWINVER=_WIN32_WINNT_WIN7) +add_definitions(-D_UNICODE -DUNICODE -DWIN32 -DMATSDK_PLATFORM_WINDOWS=1 -D_UTC_SDK -DUSE_BOND -D_WINDOWS -D_USRDLL -DWINVER=_WIN32_WINNT_WIN7) remove_definitions(-D_MBCS) list(APPEND SRCS http/HttpClient_WinInet.cpp @@ -323,14 +326,48 @@ else() add_library(mat STATIC ${SRCS}) endif() +# Public-API export decoration (MATSDK_LIBABI in lib/include/public/ctmacros.hpp). +# The SDK has no .def file, so __declspec(dllexport)/(dllimport) on Windows and +# __attribute__((visibility("default"))) elsewhere are the sole export mechanisms, +# and the decoration must follow the actual linkage: +# * shared: the SDK's own translation units export the public API +# (MATSDK_SHARED_LIB, PRIVATE). On Windows, consumers must additionally import +# it -- the INTERFACE MATSDK_IMPORT_LIB is carried by the installed +# MSTelemetry::mat target, so find_package() + link gives consumers dllimport +# automatically with no consumer-side configuration. Non-Windows consumers need +# nothing: they call symbols exported by libmat.so/.dylib. +# * static: decorate nothing, so the SDK's public symbols are NOT re-exported by +# a consumer DLL/.so that statically absorbs this library. Windows needs an +# explicit (empty) MATSDK_STATIC_LIB; elsewhere the empty MATSDK_LIBABI default +# plus -fvisibility=hidden (root CMakeLists.txt) already hides them. +if(BUILD_SHARED_LIBS) + target_compile_definitions(mat PRIVATE MATSDK_SHARED_LIB=1) + if(WIN32) + target_compile_definitions(mat INTERFACE MATSDK_IMPORT_LIB=1) + endif() +elseif(WIN32) + target_compile_definitions(mat PUBLIC MATSDK_STATIC_LIB=1) +endif() + # Target-based include paths for vcpkg / install workflow. # PUBLIC propagates to consumers; PRIVATE is SDK-internal only. # BUILD_INTERFACE is used during the SDK build; INSTALL_INTERFACE is used # by consumers after cmake --install. +# +# The public headers are added in a separate SYSTEM call: SYSTEM marks them as +# system includes for consumers, so a consumer building with -Wall -Wextra +# -Werror is not broken by warnings originating inside the SDK's headers (e.g. +# -Wpedantic variadic-macro or -Wconversion diagnostics). find_package consumers +# already treat an imported target's includes as system; SYSTEM extends the same +# courtesy to add_subdirectory/FetchContent consumers. The PRIVATE internal +# include dirs are deliberately kept out of this SYSTEM call so the SDK's own +# -Werror build still diagnoses warnings in its internal headers. target_include_directories(mat - PUBLIC + SYSTEM PUBLIC $ $ +) +target_include_directories(mat PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/include @@ -339,6 +376,17 @@ target_include_directories(mat ) if(APPLE AND BUILD_OBJC_WRAPPER) + if(BUILD_SHARED_LIBS AND OBJC_WRAPPER_SRCS) + # The root CMakeLists.txt applies -fvisibility=hidden globally to shrink the + # exported symbol table of the core C++ SDK. For Objective-C that also hides + # the wrapper class symbols (_OBJC_CLASS_$_ODW*), which are public API on + # Apple: a shared libmat.dylib would export no ODW* classes and consumers + # would fail to link (undefined _OBJC_CLASS_$_...). Re-export just the wrapper + # translation units with default visibility; the C++ core stays hidden. + set_source_files_properties(${OBJC_WRAPPER_SRCS} + PROPERTIES COMPILE_FLAGS "-fvisibility=default") + endif() + if(MATSDK_OBJC_PRIVACYGUARD_AVAILABLE) target_compile_definitions(mat PRIVATE MATSDK_OBJC_PRIVACYGUARD_AVAILABLE=1) else() @@ -355,6 +403,97 @@ endif() ################################################################################################ # Link dependencies ################################################################################################ +# --- Minimal SQLite ----------------------------------------------------------- +# The SDK uses SQLite only for its offline event-storage cache: plain tables, +# indexes, transactions, WAL, autovacuum/VACUUM, a handful of PRAGMAs, and one +# custom UTF-8 SQL function. None of SQLite's optional subsystems are needed, so +# when MATSDK_MINIMAL_SQLITE is set the bundled SQLite is compiled with these +# options to strip out everything the SDK does not use (~10% smaller SQLite code). +# They are all amalgamation-safe (no grammar/parser regeneration) and validated +# against the offline-storage unit tests. +# +# Two options are deliberately NOT stripped because the SDK depends on them: +# * SQLITE_OMIT_AUTOINIT: the skipSqliteInitAndShutdown runtime config lets the +# host own SQLite's lifecycle and skip the SDK's explicit sqlite3_initialize() +# (lib/offline/SQLiteWrapper.hpp). With a private bundled SQLite the host cannot +# initialize the SDK's copy, so auto-init must remain on. +# * SQLITE_DEFAULT_MEMSTATUS=0: the SDK arms a soft heap limit via +# sqlite3_soft_heap_limit64(cacheMemorySizeLimitInBytes) on every open, and that +# limit is only enforced while memory statistics are enabled. Disabling them +# would silently turn the configured memory cap into a no-op. +set(MATSDK_SQLITE_MINIMAL_DEFS + SQLITE_DQS=0 + SQLITE_THREADSAFE=1 + SQLITE_DEFAULT_WAL_SYNCHRONOUS=1 + SQLITE_DEFAULT_FOREIGN_KEYS=0 + SQLITE_LIKE_DOESNT_MATCH_BLOBS + SQLITE_MAX_EXPR_DEPTH=0 + SQLITE_MAX_MMAP_SIZE=0 + SQLITE_USE_ALLOCA + SQLITE_OMIT_DEPRECATED + SQLITE_OMIT_PROGRESS_CALLBACK + SQLITE_OMIT_SHARED_CACHE + SQLITE_OMIT_LOAD_EXTENSION + SQLITE_OMIT_DECLTYPE + SQLITE_OMIT_JSON + SQLITE_OMIT_TRACE + SQLITE_OMIT_COMPLETE + SQLITE_OMIT_GET_TABLE + SQLITE_OMIT_TCL_VARIABLE + SQLITE_OMIT_EXPLAIN + SQLITE_OMIT_AUTHORIZATION + SQLITE_OMIT_DESERIALIZE + SQLITE_OMIT_INTROSPECTION_PRAGMAS + SQLITE_UNTESTABLE +) + +# Bundle a vendored SQLite (built from sqlite/sqlite3.c) when MATSDK_MINIMAL_SQLITE +# is requested, or on the Android NDK legacy path (which has no system SQLite and +# has always built the vendored amalgamation). Otherwise an external/system SQLite +# is used. The feature-strip definitions above are applied ONLY when +# MATSDK_MINIMAL_SQLITE is ON, so the default Android legacy build keeps its +# existing (unstripped) bundled SQLite behavior. +set(MATSDK_BUNDLE_SQLITE OFF) +if(MATSDK_MINIMAL_SQLITE AND NOT APPLE) + # On Apple the SDK links the system libsqlite3/libz (see the Apple branch below), + # so MATSDK_MINIMAL_SQLITE has no effect there. + set(MATSDK_BUNDLE_SQLITE ON) +elseif(NOT MATSDK_USE_VCPKG_DEPS AND CMAKE_SYSTEM_NAME STREQUAL "Android") + # Android NDK ships no system SQLite, so the vendored amalgamation is always bundled. + set(MATSDK_BUNDLE_SQLITE ON) +endif() + +if(MATSDK_BUNDLE_SQLITE AND NOT TARGET sqlite3_bundled) + add_library(sqlite3_bundled STATIC "${CMAKE_CURRENT_SOURCE_DIR}/../sqlite/sqlite3.c") + # Consumers of MSTelemetry::mat never include sqlite3.h (it is an internal + # implementation detail), so the header path is only needed while building the + # SDK itself -- wrap it in BUILD_INTERFACE so install(EXPORT) stays valid. + target_include_directories(sqlite3_bundled PUBLIC + "$") + set_target_properties(sqlite3_bundled PROPERTIES POSITION_INDEPENDENT_CODE ON) + if(MATSDK_MINIMAL_SQLITE) + # Feature-stripped build: apply the minimal definitions. + target_compile_definitions(sqlite3_bundled PRIVATE ${MATSDK_SQLITE_MINIMAL_DEFS}) + endif() + if(MSVC) + # Silence the vendored amalgamation's warnings (/w) and turn off + # warning-as-error (/WX-) for this third-party translation unit, so the SDK's + # /WX does not promote any amalgamation warning that survives /w to an error. + target_compile_options(sqlite3_bundled PRIVATE /w /WX-) + elseif(MATSDK_MINIMAL_SQLITE) + # -w disables all warnings for this vendored translation unit so the SDK's + # -Werror does not fire on amalgamation code (the OMIT_* options leave some + # debug-build macros expanding to empty/unused statements). -fno-finite-math-only: + # the amalgamation relies on the INFINITY macro, which -ffast-math / + # -ffinite-math-only would break. + target_compile_options(sqlite3_bundled PRIVATE -w -fno-finite-math-only) + else() + # Unstripped vendored build (Android legacy): keep the existing narrower + # warning suppression. -fno-finite-math-only guards the INFINITY macro. + target_compile_options(sqlite3_bundled PRIVATE -fno-finite-math-only -Wno-unused-function) + endif() +endif() + # TODO: allow adding "${Tcmalloc_LIBRARIES}" to target_link_libraries for memory leak debugging # (USE_TCMALLOC / FindTcmalloc.cmake are configured for Debug builds in the root CMakeLists.txt, # but the library is not yet linked here). @@ -362,25 +501,42 @@ if(MATSDK_USE_VCPKG_DEPS) # vcpkg mode: all deps resolved via find_package() in root CMakeLists.txt # These are PUBLIC so static-library consumers get the transitive link set # through the exported MSTelemetry::mat target. - target_link_libraries(mat - PUBLIC - unofficial::sqlite3::sqlite3 - ZLIB::ZLIB - nlohmann_json::nlohmann_json - ${LIBS} - ) + if(APPLE) + # macOS/iOS link the system libsqlite3 + libz (SQLite::SQLite3 / ZLIB::ZLIB + # resolve to the OS libraries via CMake's find modules), so the vcpkg + # sqlite3/zlib packages are neither pulled nor linked here. + target_link_libraries(mat + PUBLIC + SQLite::SQLite3 + ZLIB::ZLIB + nlohmann_json::nlohmann_json + ${LIBS} + ) + else() + if(MATSDK_BUNDLE_SQLITE) + # Private minimal SQLite instead of the vcpkg sqlite3 package. PRIVATE so its + # include dirs / compile definitions are not propagated as a public usage + # requirement. A static mat still propagates the archive itself for linking + # (via $), so it is added to the export set for static builds + # below; a shared mat absorbs it and propagates nothing. + target_link_libraries(mat PRIVATE sqlite3_bundled) + else() + target_link_libraries(mat PUBLIC unofficial::sqlite3::sqlite3) + endif() + target_link_libraries(mat + PUBLIC + ZLIB::ZLIB + nlohmann_json::nlohmann_json + ${LIBS} + ) + endif() else() # Legacy mode: use vendored or system-installed deps if(CMAKE_SYSTEM_NAME STREQUAL "Android") - # Android NDK has no system sqlite3 or zlib — build from bundled source. - add_library(sqlite3_bundled STATIC "${CMAKE_CURRENT_SOURCE_DIR}/../sqlite/sqlite3.c") - target_include_directories(sqlite3_bundled PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../sqlite") - set_target_properties(sqlite3_bundled PROPERTIES POSITION_INDEPENDENT_CODE ON) - # Guard bundled sqlite3 against toolchain or environment flags that imply finite-math-only (uses INFINITY macro). - # Also suppress warnings treated as errors in vendored code. - target_compile_options(sqlite3_bundled PRIVATE -fno-finite-math-only -Wno-unused-function) - - # Build zlib from bundled source. + # Build zlib from bundled source: the Android NDK ships no system zlib, and the + # vendored zlib renames its exports to act_z_* (via zlib/names.h). SQLite is + # provided by sqlite3_bundled, created above (MATSDK_BUNDLE_SQLITE is ON for + # the Android NDK path). add_library(zlib_bundled STATIC "${CMAKE_CURRENT_SOURCE_DIR}/../zlib/adler32.c" "${CMAKE_CURRENT_SOURCE_DIR}/../zlib/compress.c" @@ -410,26 +566,33 @@ else() elseif(PAL_IMPLEMENTATION STREQUAL "WIN32") # Windows legacy: vendored sqlite/zlib headers are included via # include_directories in the PAL section above; link only ${LIBS} - # (e.g. CURL if needed — sqlite/zlib come from .vcxproj references). - target_link_libraries(mat PRIVATE ${LIBS}) + # (e.g. CURL if needed — sqlite/zlib come from .vcxproj references), plus the + # private minimal SQLite when MATSDK_MINIMAL_SQLITE is enabled. + if(MATSDK_BUNDLE_SQLITE) + target_link_libraries(mat PRIVATE sqlite3_bundled ${LIBS}) + else() + target_link_libraries(mat PRIVATE ${LIBS}) + endif() + elseif(APPLE) + # macOS and iOS both ship system libsqlite3 and libz. Link them by portable + # names -- matching the SDK's own iOS Xcode projects (libsqlite3.tbd + libz.tbd + # from the SDKROOT), Package.swift (.linkedLibrary sqlite3/z), and the vcpkg + # Apple path -- so nothing is bundled and exported static packages stay + # relocatable. On Apple, #include / resolve from the SDK + # sysroot, so no explicit include dir or find_package is needed. + target_link_libraries(mat PRIVATE sqlite3 z ${LIBS}) else() - # Linux/macOS legacy: link system-installed sqlite3 and zlib - if(EXISTS "/usr/local/lib/libsqlite3.a") - set(MATSDK_SQLITE3_LIB "/usr/local/lib/libsqlite3.a") - elseif(EXISTS "/usr/local/opt/sqlite/lib/libsqlite3.a") - set(MATSDK_SQLITE3_LIB "/usr/local/opt/sqlite/lib/libsqlite3.a") - elseif(EXISTS "/opt/homebrew/opt/sqlite/lib/libsqlite3.a") - set(MATSDK_SQLITE3_LIB "/opt/homebrew/opt/sqlite/lib/libsqlite3.a") + # Linux legacy: system zlib + system (or private minimal) sqlite3. ZLIB::ZLIB + # and SQLite::SQLite3 are imported targets that carry their own include dirs. + find_package(ZLIB REQUIRED) + if(MATSDK_BUNDLE_SQLITE) + target_link_libraries(mat PRIVATE sqlite3_bundled ZLIB::ZLIB ${LIBS}) else() # find_package(SQLite3) needs CMake >= 3.14, guaranteed by the project floor; # SQLite::SQLite3 is an imported target carrying its own include dirs. find_package(SQLite3 REQUIRED) - set(MATSDK_SQLITE3_LIB SQLite::SQLite3) + target_link_libraries(mat PRIVATE SQLite::SQLite3 ZLIB::ZLIB ${LIBS}) endif() - - find_package(ZLIB REQUIRED) - target_include_directories(mat PRIVATE ${ZLIB_INCLUDE_DIRS}) - target_link_libraries(mat PRIVATE ${MATSDK_SQLITE3_LIB} ZLIB::ZLIB ${LIBS}) endif() endif() @@ -466,7 +629,21 @@ endif() # consumer that does find_package(MSTelemetry). Legacy (non-vcpkg) builds install # via install.sh or MSBuild output directories and don't need this. if(MATSDK_USE_VCPKG_DEPS) - install(TARGETS mat + # A static libmat propagates its PRIVATE static dependencies through its link + # interface (as $), so the bundled SQLite must be part of the same + # export set and installed alongside mat for downstream find_package() consumers + # to link. A shared libmat absorbs sqlite3_bundled into the .so/.dylib/.dll and + # does not propagate the PRIVATE dep, so exporting the archive there is + # unnecessary (and risks a consumer linking a second SQLite copy) -- only export + # it for a static mat. + set(MATSDK_INSTALL_TARGETS mat) + if(MATSDK_BUNDLE_SQLITE AND TARGET sqlite3_bundled) + get_target_property(_mat_type mat TYPE) + if(_mat_type STREQUAL "STATIC_LIBRARY") + list(APPEND MATSDK_INSTALL_TARGETS sqlite3_bundled) + endif() + endif() + install(TARGETS ${MATSDK_INSTALL_TARGETS} EXPORT MSTelemetryTargets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} @@ -503,6 +680,14 @@ if(MATSDK_USE_VCPKG_DEPS) "${CMAKE_CURRENT_BINARY_DIR}/MSTelemetryConfigVersion.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/MSTelemetry ) + + if(CMAKE_SYSTEM_NAME STREQUAL "Android" AND MATSDK_ANDROID_USES_JAVA_HTTP) + install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/android_build/maesdk/src/main/java/com/microsoft/applications/events/HttpClient.java" + "${CMAKE_CURRENT_SOURCE_DIR}/android_build/maesdk/src/main/java/com/microsoft/applications/events/HttpClientRequest.java" + DESTINATION "${CMAKE_INSTALL_DATADIR}/cpp-client-telemetry/android/java/com/microsoft/applications/events" + ) + endif() else() # Legacy install: just put the library and headers in standard locations install(TARGETS mat diff --git a/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/EventPropertiesStorage.java b/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/EventPropertiesStorage.java index b5b7929e8..9641caa67 100644 --- a/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/EventPropertiesStorage.java +++ b/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/EventPropertiesStorage.java @@ -21,6 +21,7 @@ class EventPropertiesStorage { EventPropertiesStorage() { eventName = ""; + eventType = ""; eventLatency = EventLatency.Normal; eventPersistence = EventPersistence.Normal; eventPopSample = 100; diff --git a/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/LogConfigurationKey.java b/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/LogConfigurationKey.java index 329f17680..0ce2881ef 100644 --- a/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/LogConfigurationKey.java +++ b/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/LogConfigurationKey.java @@ -29,6 +29,9 @@ public enum LogConfigurationKey { /** Enable network detector. */ CFG_BOOL_ENABLE_NET_DETECT("enableNetworkDetector", Boolean.class), + /** Scrub (obfuscate) the client IP address at the collector. Applied unless explicitly set to false (on by default; not present in the default configuration). */ + CFG_BOOL_ENABLE_IP_SCRUBBING("enableIpScrubbing", Boolean.class), + CFG_BOOL_TPM_CLOCK_SKEW_ENABLED("clockSkewEnabled", Boolean.class), /** Parameter that allows to check if the SDK is running on UTC mode */ diff --git a/lib/android_build/maesdk/src/test/java/com/microsoft/applications/events/EventsUnitTest.java b/lib/android_build/maesdk/src/test/java/com/microsoft/applications/events/EventsUnitTest.java index 24bfa90ea..513354682 100644 --- a/lib/android_build/maesdk/src/test/java/com/microsoft/applications/events/EventsUnitTest.java +++ b/lib/android_build/maesdk/src/test/java/com/microsoft/applications/events/EventsUnitTest.java @@ -390,4 +390,17 @@ public void requestException() throws java.io.IOException, PackageManager.NameNo } } + @Test + public void newEventPropertiesStorageEventTypeDefaultsToEmptyString() { + // Regression test for #1329: EventPropertiesStorage previously left + // eventType uninitialized (null), so EventProperties.getType() (which + // returns mStorage.eventType) returned null instead of its documented + // "" default. Exercise the pure-Java storage directly: constructing an + // EventProperties here would call setName() -> native validateEventName(), + // which is not loaded in these JVM (MockitoJUnitRunner) unit tests. + EventPropertiesStorage storage = new EventPropertiesStorage(); + assertNotNull(storage.eventType); + assertEquals("", storage.eventType); + } + } diff --git a/lib/api/LogManagerImpl.cpp b/lib/api/LogManagerImpl.cpp index 2f0e8933d..24215c0cd 100644 --- a/lib/api/LogManagerImpl.cpp +++ b/lib/api/LogManagerImpl.cpp @@ -839,7 +839,7 @@ namespace MAT_NS_BEGIN return; } - auto itDataInspector = std::find_if(m_dataInspectors.begin(), m_dataInspectors.end(), [&dataInspector](const std::shared_ptr& currentInspector) + auto itDataInspector = std::find_if(m_dataInspectors.begin(), m_dataInspectors.end(), [&dataInspector](const std::shared_ptr& currentInspector) noexcept { return strcmp(dataInspector->GetName(), currentInspector->GetName()) == 0; }); @@ -862,7 +862,7 @@ namespace MAT_NS_BEGIN void LogManagerImpl::RemoveDataInspector(const std::string& name) { LOCKGUARD(m_dataInspectorGuard); - auto itDataInspector = std::find_if(m_dataInspectors.begin(), m_dataInspectors.end(), [&name](const std::shared_ptr& inspector){ + auto itDataInspector = std::find_if(m_dataInspectors.begin(), m_dataInspectors.end(), [&name](const std::shared_ptr& inspector) noexcept { return strcmp(inspector->GetName(), name.c_str()) == 0; }); @@ -875,7 +875,7 @@ namespace MAT_NS_BEGIN std::shared_ptr LogManagerImpl::GetDataInspector(const std::string& name) noexcept { LOCKGUARD(m_dataInspectorGuard); - auto it = std::find_if(m_dataInspectors.begin(), m_dataInspectors.end(), [&name](const std::shared_ptr& inspector){ + auto it = std::find_if(m_dataInspectors.begin(), m_dataInspectors.end(), [&name](const std::shared_ptr& inspector) noexcept{ return strcmp(inspector->GetName(), name.c_str()) == 0; }); @@ -944,7 +944,7 @@ namespace MAT_NS_BEGIN if (m_pause_state != PauseState::Pausing) { return; } - m_pause_cv.wait(lock, [this]() -> bool { + m_pause_cv.wait(lock, [this]() noexcept -> bool { return m_pause_state != PauseState::Pausing; }); } diff --git a/lib/api/LogSessionData.cpp b/lib/api/LogSessionData.cpp index 9ea280f6a..34f6538b2 100644 --- a/lib/api/LogSessionData.cpp +++ b/lib/api/LogSessionData.cpp @@ -10,7 +10,7 @@ using namespace std; namespace MAT_NS_BEGIN { - uint64_t LogSessionData::getSessionFirstTime() const + uint64_t LogSessionData::getSessionFirstTime() const noexcept { return m_sessionFirstTimeLaunch; } diff --git a/lib/api/capi.cpp b/lib/api/capi.cpp index 916c4ebda..531d9419c 100644 --- a/lib/api/capi.cpp +++ b/lib/api/capi.cpp @@ -49,7 +49,7 @@ capi_client * MAT::capi_get_client(evt_handle_t handle) /// /// Remove C API handle from active client tracking struct. /// -void remove_client(evt_handle_t handle) +static void remove_client(evt_handle_t handle) { LOCKGUARD(mtx); clients.erase(handle); @@ -66,7 +66,7 @@ void remove_client(evt_handle_t handle) return ENOENT; \ }; -evt_status_t mat_open_core( +static evt_status_t mat_open_core( evt_context_t *ctx, const char* config, http_send_fn_t httpSendFn, @@ -77,7 +77,11 @@ evt_status_t mat_open_core( { if ((config == nullptr) || (config[0] == 0)) { - // Invalid configuration + // Invalid configuration. ctx is guaranteed non-null by the callers + // (mat_open / mat_open_with_params); set result and a known-invalid + // handle so callers don't observe a stale ctx->handle on error. + ctx->result = static_cast(EFAULT); + ctx->handle = 0; return EFAULT; } @@ -91,7 +95,12 @@ evt_status_t mat_open_core( { if (client->ctx_data == config) { - // Guest instance with the same config is already open + // Guest instance with the same config is already open. + // Return its handle so the caller still gets a usable handle + // (rather than leaving ctx->handle uninitialized), and set + // ctx->result to match the returned status like the other paths. + ctx->handle = code; + ctx->result = static_cast(EALREADY); return EALREADY; } // hash code is assigned to another client, increment and retry @@ -143,6 +152,11 @@ evt_status_t mat_open_core( } catch (...) { + // Roll back the partially-populated client so a later open with the + // same config does not find stale half-initialized state. + remove_client(code); + ctx->result = static_cast(EFAULT); + ctx->handle = 0; return EFAULT; } } @@ -158,6 +172,11 @@ evt_status_t mat_open_core( } catch (...) { + // Roll back the partially-populated client so a later open with the + // same config does not find stale half-initialized state. + remove_client(code); + ctx->result = static_cast(EFAULT); + ctx->handle = 0; return EFAULT; } } @@ -175,7 +194,7 @@ evt_status_t mat_open_core( return ctx->result; } -evt_status_t mat_open(evt_context_t *ctx) +static evt_status_t mat_open(evt_context_t *ctx) { if (ctx == nullptr) { @@ -186,7 +205,7 @@ evt_status_t mat_open(evt_context_t *ctx) return mat_open_core(ctx, config, nullptr, nullptr, nullptr, nullptr, nullptr); } -evt_status_t mat_open_with_params(evt_context_t *ctx) +static evt_status_t mat_open_with_params(evt_context_t *ctx) { if (ctx == nullptr) { @@ -231,9 +250,9 @@ evt_status_t mat_open_with_params(evt_context_t *ctx) } /** - * Marashal C struct to C++ API + * Marshal C struct to C++ API */ -evt_status_t mat_log(evt_context_t *ctx) +static evt_status_t mat_log(evt_context_t *ctx) { VERIFY_CLIENT_HANDLE(client, ctx); @@ -285,7 +304,7 @@ evt_status_t mat_log(evt_context_t *ctx) return ctx->result; } -evt_status_t mat_close(evt_context_t *ctx) +static evt_status_t mat_close(evt_context_t *ctx) { VERIFY_CLIENT_HANDLE(client, ctx); const auto result = static_cast(LogManagerProvider::Release(client->logmanager->GetLogConfiguration())); @@ -305,7 +324,7 @@ evt_status_t mat_close(evt_context_t *ctx) return result; } -evt_status_t mat_pause(evt_context_t *ctx) +static evt_status_t mat_pause(evt_context_t *ctx) { VERIFY_CLIENT_HANDLE(client, ctx); const auto result = static_cast(client->logmanager->PauseTransmission()); @@ -313,7 +332,7 @@ evt_status_t mat_pause(evt_context_t *ctx) return result; } -evt_status_t mat_resume(evt_context_t *ctx) +static evt_status_t mat_resume(evt_context_t *ctx) { VERIFY_CLIENT_HANDLE(client, ctx); const auto result = static_cast(client->logmanager->ResumeTransmission()); @@ -321,7 +340,7 @@ evt_status_t mat_resume(evt_context_t *ctx) return result; } -evt_status_t mat_upload(evt_context_t *ctx) +static evt_status_t mat_upload(evt_context_t *ctx) { VERIFY_CLIENT_HANDLE(client, ctx); const auto result = static_cast(client->logmanager->UploadNow()); @@ -329,7 +348,7 @@ evt_status_t mat_upload(evt_context_t *ctx) return result; } -evt_status_t mat_flushAndTeardown(evt_context_t *ctx) +static evt_status_t mat_flushAndTeardown(evt_context_t *ctx) { VERIFY_CLIENT_HANDLE(client, ctx); client->logmanager->FlushAndTeardown(); diff --git a/lib/bond/CompactBinaryProtocolReader.hpp b/lib/bond/CompactBinaryProtocolReader.hpp index 612970421..708aba421 100644 --- a/lib/bond/CompactBinaryProtocolReader.hpp +++ b/lib/bond/CompactBinaryProtocolReader.hpp @@ -69,7 +69,7 @@ class CompactBinaryProtocolReader { return false; } #ifdef HAVE_ONEDS_BOUNDCHECK_METHODS - bool result = MAT::BoundCheckFunctions::oneds_memcpy_s(static_cast(data), size, &(m_input[m_ofs]), size); + bool result = (MAT::BoundCheckFunctions::oneds_memcpy_s(static_cast(data), size, &(m_input[m_ofs]), size) == 0); #else bool result = (memcpy_s(static_cast(data), size, &(m_input[m_ofs]), size) == 0); #endif diff --git a/lib/compression/HttpDeflateCompression.cpp b/lib/compression/HttpDeflateCompression.cpp index f8e2b1779..93605ea89 100644 --- a/lib/compression/HttpDeflateCompression.cpp +++ b/lib/compression/HttpDeflateCompression.cpp @@ -44,7 +44,7 @@ namespace MAT_NS_BEGIN { int result = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, m_windowBits, 8 /*DEF_MEM_LEVEL*/, Z_DEFAULT_STRATEGY); if (result != Z_OK) { - LOG_WARN("HTTP request compressing failed, error=%u/%u (%s)", 1, result, stream.msg); + LOG_WARN("HTTP request compressing failed, error=%d/%d (%s)", 1, result, (stream.msg ? stream.msg : "(null)")); compressionFailed(ctx); return false; } @@ -80,7 +80,7 @@ namespace MAT_NS_BEGIN { deflateEnd(&stream); if (result != Z_STREAM_END) { - LOG_WARN("HTTP request compressing failed, error=%u/%u (%s)", 2, result, stream.msg); + LOG_WARN("HTTP request compressing failed, error=%d/%d (%s)", 2, result, (stream.msg ? stream.msg : "(null)")); compressionFailed(ctx); return false; } diff --git a/lib/decoder/PayloadDecoder.cpp b/lib/decoder/PayloadDecoder.cpp index 5789d5939..6d86a7057 100644 --- a/lib/decoder/PayloadDecoder.cpp +++ b/lib/decoder/PayloadDecoder.cpp @@ -563,7 +563,12 @@ namespace MAT_NS_BEGIN { if (result) { - out = j.dump(2); + // Use error_handler_t::replace so that malformed UTF-8 in the + // decoded telemetry payload is replaced with U+FFFD instead of + // throwing nlohmann::json::type_error (id 316). Telemetry event + // strings can legitimately contain non-UTF-8 bytes; without this + // the unhandled C++ exception terminates the hosting process. + out = j.dump(2, ' ', false, json::error_handler_t::replace); } return result; @@ -580,7 +585,9 @@ namespace MAT_NS_BEGIN { nlohmann::json j; to_json(j, in); - std::string s = j.dump(4); + // See DecodeRequest above: replace malformed UTF-8 rather than + // throwing so a bad record cannot terminate the process. + std::string s = j.dump(4, ' ', false, json::error_handler_t::replace); out.assign(s.begin(), s.end()); return true; diff --git a/lib/decorators/EventPropertiesDecorator.hpp b/lib/decorators/EventPropertiesDecorator.hpp index 5bb3e927a..91c5f74b4 100644 --- a/lib/decorators/EventPropertiesDecorator.hpp +++ b/lib/decorators/EventPropertiesDecorator.hpp @@ -6,6 +6,8 @@ #define EVENTPROPERTIESDECORATOR_HPP #include "IDecorator.hpp" +#include "ILogManager.hpp" +#include "RecordFlagConstants.hpp" #include "EventProperties.hpp" #include "CorrelationVector.hpp" #include "utils/Utils.hpp" @@ -13,18 +15,10 @@ #include #include #include +#include namespace MAT_NS_BEGIN { -// Bit remapping has to happen on bits passed via API surface. -// Ref CS2.1+ : https://osgwiki.com/wiki/CommonSchema/flags -// #define MICROSOFT_EVENTTAG_MARK_PII 0x08000000 -#define RECORD_FLAGS_EVENTTAG_MARK_PII 0x00080000 -// #define MICROSOFT_EVENTTAG_HASH_PII 0x04000000 -#define RECORD_FLAGS_EVENTTAG_HASH_PII 0x00100000 -// #define MICROSOFT_EVENTTAG_DROP_PII 0x02000000 -#define RECORD_FLAGS_EVENTTAG_DROP_PII 0x00200000 - class EventPropertiesDecorator : public IDecorator { protected: @@ -125,6 +119,14 @@ namespace MAT_NS_BEGIN { int64_t tags = eventProperties.GetPolicyBitFlags(); int64_t flags = 0; + // Scrub/obfuscate the client IP address at the collector by default. + // Hosts that require the client IP (e.g. for geo-location enrichment) + // can opt out by setting CFG_BOOL_ENABLE_IP_SCRUBBING = false. + ILogConfiguration& config = m_owner.GetLogConfiguration(); + if (!config.HasConfig(CFG_BOOL_ENABLE_IP_SCRUBBING) || config[CFG_BOOL_ENABLE_IP_SCRUBBING]) + { + flags |= RECORD_FLAGS_EVENTTAG_SCRUB_IP; + } // We must remap from one bitfield set to another, no way to bit-shift :( // At the moment 1DS SDK in direct upload mode supports DROP and MARK tags only: flags |= (tags & MICROSOFT_EVENTTAG_MARK_PII) ? RECORD_FLAGS_EVENTTAG_MARK_PII : 0; @@ -187,11 +189,11 @@ namespace MAT_NS_BEGIN { temp.stringValue = v.to_string(); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } } @@ -209,11 +211,11 @@ namespace MAT_NS_BEGIN { temp.stringValue = v.to_string(); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } #if 0 /* v2 code */ if (v.piiKind != PiiKind_None) @@ -251,11 +253,11 @@ namespace MAT_NS_BEGIN { temp.stringValue = v.to_string(); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -266,11 +268,11 @@ namespace MAT_NS_BEGIN { temp.longValue = v.as_int64; if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -281,11 +283,11 @@ namespace MAT_NS_BEGIN { temp.doubleValue = v.as_double; if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -296,11 +298,11 @@ namespace MAT_NS_BEGIN { temp.longValue = v.as_time_ticks.ticks; if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -311,11 +313,11 @@ namespace MAT_NS_BEGIN { temp.longValue = v.as_bool; if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -345,11 +347,11 @@ namespace MAT_NS_BEGIN { temp.longArray.push_back(*v.as_longArray); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -360,11 +362,11 @@ namespace MAT_NS_BEGIN { temp.doubleArray.push_back(*v.as_doubleArray); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -375,11 +377,11 @@ namespace MAT_NS_BEGIN { temp.stringArray.push_back(*v.as_stringArray); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -398,11 +400,11 @@ namespace MAT_NS_BEGIN { temp.guidArray.push_back(values); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -413,11 +415,11 @@ namespace MAT_NS_BEGIN { temp.stringValue = v.to_string(); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } } } @@ -427,8 +429,8 @@ namespace MAT_NS_BEGIN { if (extPartB.size() > 0) { ::CsProtocol::Data partBdata; - partBdata.properties = extPartB; - record.baseData.push_back(partBdata); + partBdata.properties = std::move(extPartB); + record.baseData.push_back(std::move(partBdata)); } // special case of CorrelationVector value diff --git a/lib/decorators/RecordFlagConstants.hpp b/lib/decorators/RecordFlagConstants.hpp new file mode 100644 index 000000000..8c0fe56d3 --- /dev/null +++ b/lib/decorators/RecordFlagConstants.hpp @@ -0,0 +1,30 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +#ifndef RECORDFLAGCONSTANTS_HPP +#define RECORDFLAGCONSTANTS_HPP + +#include "ctmacros.hpp" + +#include + +namespace MAT_NS_BEGIN { + + // On-wire CS protocol record.flags bits. These are distinct from the + // API-surface MICROSOFT_EVENTTAG_* policy flags and are remapped onto + // record.flags by EventPropertiesDecorator. Kept in a dedicated header so + // other components (e.g. the stats pipeline) can reference a single bit + // definition without depending on the decorator's inline implementation. + // Ref CS2.1+: https://osgwiki.com/wiki/CommonSchema/flags + // (API-surface MICROSOFT_EVENTTAG_MARK_PII 0x08000000) + static constexpr std::int64_t RECORD_FLAGS_EVENTTAG_MARK_PII = 0x00080000; + // (API-surface MICROSOFT_EVENTTAG_HASH_PII 0x04000000) + static constexpr std::int64_t RECORD_FLAGS_EVENTTAG_HASH_PII = 0x00100000; + // (API-surface MICROSOFT_EVENTTAG_DROP_PII 0x02000000) + static constexpr std::int64_t RECORD_FLAGS_EVENTTAG_DROP_PII = 0x00200000; + static constexpr std::int64_t RECORD_FLAGS_EVENTTAG_SCRUB_IP = 0x00400000; + +} MAT_NS_END + +#endif // RECORDFLAGCONSTANTS_HPP diff --git a/lib/http/HttpClient_Apple.mm b/lib/http/HttpClient_Apple.mm index 449b4c9af..b7d6646a4 100644 --- a/lib/http/HttpClient_Apple.mm +++ b/lib/http/HttpClient_Apple.mm @@ -15,6 +15,111 @@ #include "utils/StringUtils.hpp" #include "utils/Utils.hpp" +// Streams the response body in bounded chunks and enforces MAX_HTTP_RESPONSE_SIZE. +// The completionHandler-based NSURLSession APIs fully materialize the response body +// as an NSData before handing it over, so an attacker-controlled collector could force +// a large allocation. This delegate instead accumulates data incrementally in +// didReceiveData: and cancels the transfer as soon as the cap would be exceeded, so no +// more than the cap is ever buffered. Delegate callbacks may arrive on the session's +// delegate queue while a request thread registers a task, so shared state is guarded. +@interface MATStreamingSessionDelegate : NSObject +- (void)registerTask:(NSURLSessionTask*)task + handler:(void (^)(NSData* data, NSURLResponse* response, NSError* error))handler; +@end + +@implementation MATStreamingSessionDelegate { + NSMutableDictionary* _buffers; + NSMutableDictionary* _handlers; + NSMutableSet* _overCap; +} + +- (instancetype)init +{ + self = [super init]; + if (self) + { + _buffers = [NSMutableDictionary new]; + _handlers = [NSMutableDictionary new]; + _overCap = [NSMutableSet new]; + } + return self; +} + +- (void)registerTask:(NSURLSessionTask*)task + handler:(void (^)(NSData*, NSURLResponse*, NSError*))handler +{ + NSNumber* key = @(task.taskIdentifier); + @synchronized(self) + { + _buffers[key] = [NSMutableData new]; + _handlers[key] = [handler copy]; + } +} + +- (void)URLSession:(NSURLSession*)session + dataTask:(NSURLSessionDataTask*)dataTask + didReceiveData:(NSData*)data +{ + NSNumber* key = @(dataTask.taskIdentifier); + @synchronized(self) + { + if ([_overCap containsObject:key]) + { + return; + } + NSMutableData* buffer = _buffers[key]; + if (buffer == nil) + { + return; + } + if (buffer.length + data.length > MAT::MAX_HTTP_RESPONSE_SIZE) + { + // Refuse the over-large response: stop buffering and cancel the transfer. + [_overCap addObject:key]; + [dataTask cancel]; + return; + } + [buffer appendData:data]; + } +} + +- (void)URLSession:(NSURLSession*)session + task:(NSURLSessionTask*)task +didCompleteWithError:(NSError*)error +{ + NSNumber* key = @(task.taskIdentifier); + void (^handler)(NSData*, NSURLResponse*, NSError*) = nil; + NSData* body = nil; + BOOL overCap = NO; + @synchronized(self) + { + handler = (void (^)(NSData*, NSURLResponse*, NSError*))_handlers[key]; + body = _buffers[key]; + overCap = [_overCap containsObject:key]; + [_handlers removeObjectForKey:key]; + [_buffers removeObjectForKey:key]; + [_overCap removeObject:key]; + } + if (handler == nil) + { + return; + } + if (overCap) + { + // Surface a non-cancellation error so the request maps to NetworkFailure + // (retried), not Aborted (which is reserved for caller-initiated cancels). + NSError* capError = [NSError errorWithDomain:@"MATResponseCap" + code:-1 + userInfo:@{ NSLocalizedDescriptionKey : @"HTTP response exceeds max buffered size" }]; + handler(nil, task.response, capError); + } + else + { + handler(body, task.response, error); + } +} +@end + namespace MAT_NS_BEGIN { static std::string NextReqId() @@ -31,6 +136,7 @@ static dispatch_once_t once; static NSURLSession* session; +static MATStreamingSessionDelegate* sessionDelegate; class HttpRequestApple : public SimpleHttpRequest { @@ -42,7 +148,10 @@ m_parent->Add(static_cast(this)); dispatch_once(&once, ^{ NSURLSessionConfiguration* sessionConfig = [NSURLSessionConfiguration defaultSessionConfiguration]; - session = [NSURLSession sessionWithConfiguration:sessionConfig]; + sessionDelegate = [MATStreamingSessionDelegate new]; + session = [NSURLSession sessionWithConfiguration:sessionConfig + delegate:sessionDelegate + delegateQueue:nil]; }); } @@ -75,15 +184,18 @@ void SendAsync(IHttpResponseCallback* callback) if(equalsIgnoreCase(m_method, "get")) { [m_urlRequest setHTTPMethod:@"GET"]; - m_dataTask = [session dataTaskWithRequest:m_urlRequest completionHandler:m_completionMethod]; + m_dataTask = [session dataTaskWithRequest:m_urlRequest]; } else { [m_urlRequest setHTTPMethod:@"POST"]; NSData* postData = [NSData dataWithBytes:m_body.data() length:m_body.size()]; - m_dataTask = [session uploadTaskWithRequest:m_urlRequest fromData:postData completionHandler:m_completionMethod]; + m_dataTask = [session uploadTaskWithRequest:m_urlRequest fromData:postData]; } + // Register before resume so the streaming delegate has the buffer and + // completion handler in place before any response data arrives. + [sessionDelegate registerTask:m_dataTask handler:m_completionMethod]; [m_dataTask resume]; } } @@ -120,10 +232,18 @@ void HandleResponse(NSData* data, NSURLResponse* response, NSError* error) } else { + // The streaming delegate has already enforced MAX_HTTP_RESPONSE_SIZE + // (an over-cap response arrives here as a cap error, handled above), so + // data is bounded. Guard against a nil/empty body to avoid pointer + // arithmetic on a null [data bytes]. simpleResponse->m_result = HttpResult_OK; - auto body = static_cast([data bytes]); - simpleResponse->m_body.reserve(data.length); - std::copy(body, body + data.length, std::back_inserter(simpleResponse->m_body)); + const size_t length = static_cast(data.length); + if (length > 0) + { + auto body = static_cast([data bytes]); + simpleResponse->m_body.reserve(length); + std::copy(body, body + length, std::back_inserter(simpleResponse->m_body)); + } } m_callback->OnHttpResponse(simpleResponse); } diff --git a/lib/http/HttpClient_Curl.hpp b/lib/http/HttpClient_Curl.hpp index b1bb5344c..533c522e3 100644 --- a/lib/http/HttpClient_Curl.hpp +++ b/lib/http/HttpClient_Curl.hpp @@ -484,6 +484,14 @@ class CurlHttpOperation { return poll(&pfd, 1, static_cast(timeout)); } + // SECURITY: upper bound on the collector response the client will buffer. The + // OneCollector protocol responses (status, kill-switch tokens, retry-after, small + // config) are tiny, so this generous cap never rejects a legitimate response but + // stops a hostile or MITM'd collector from driving unbounded memory growth by + // returning an oversized body (a memory-amplification DoS of the embedding process). + // Exceeding it aborts the transfer, so the upload is treated as failed and retried. + static constexpr size_t kMaxResponseBytes = 16 * 1024 * 1024; // 16 MB + // Raw response buffer struct MemoryStruct { char *memory; @@ -501,9 +509,21 @@ class CurlHttpOperation { */ static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) { + // Guard the size * nmemb product against size_t overflow before using it. + if (nmemb != 0 && size > static_cast(-1) / nmemb) { + return 0; + } size_t realsize = size * nmemb; struct MemoryStruct *mem = (struct MemoryStruct *)userp; + // SECURITY: bound the buffered response (see kMaxResponseBytes). Compare + // overflow-safely (mem->size is always <= kMaxResponseBytes here). Returning a + // short count aborts the transfer with CURLE_WRITE_ERROR. + if (realsize > kMaxResponseBytes - mem->size) { + TRACE("Response exceeds max buffered size (%zu bytes); aborting transfer\n", kMaxResponseBytes); + return 0; + } + auto* memory = static_cast(realloc(mem->memory, mem->size + realsize + 1)); if(memory == nullptr) { /* out of memory! */ @@ -533,9 +553,21 @@ class CurlHttpOperation { */ static size_t WriteVectorCallback(void *ptr, size_t size, size_t nmemb, std::vector* data) { + // Guard the size * nmemb product against size_t overflow before using it. + if (nmemb != 0 && size > static_cast(-1) / nmemb) { + return 0; + } if (data != nullptr) { + size_t realsize = size * nmemb; + // SECURITY: bound the buffered response (see kMaxResponseBytes). Compare + // overflow-safely (data->size() is always <= kMaxResponseBytes here). + // Returning a short count aborts the transfer with CURLE_WRITE_ERROR. + if (realsize > kMaxResponseBytes - data->size()) { + TRACE("Response exceeds max buffered size (%zu bytes); aborting transfer\n", kMaxResponseBytes); + return 0; + } const auto* begin = static_cast(ptr); - const auto* end = begin + size * nmemb; + const auto* end = begin + realsize; data->insert( data->end(), begin, end); } return size * nmemb; diff --git a/lib/http/HttpClient_WinInet.cpp b/lib/http/HttpClient_WinInet.cpp index eaefb2318..b1d3b4013 100644 --- a/lib/http/HttpClient_WinInet.cpp +++ b/lib/http/HttpClient_WinInet.cpp @@ -324,27 +324,41 @@ class WinInetRequestWrapper // It might potentially be another async operation which will // trigger INTERNET_STATUS_REQUEST_COMPLETE again. - m_bodyBuffer.insert(m_bodyBuffer.end(), m_buffer, m_buffer + m_bufferUsed); - while (!m_readingData || m_bufferUsed != 0) { - BOOL bResult = ::InternetReadFile(m_hWinInetRequest, m_buffer, sizeof(m_buffer), &m_bufferUsed); - m_readingData = true; - if (!bResult) { - dwError = GetLastError(); - if (dwError == ERROR_IO_PENDING) { - // Do not touch anything from this thread anymore. - // The buffer passed to InternetReadFile() and the - // read count will be filled asynchronously, so they - // must stay valid and writable until the next - // INTERNET_STATUS_REQUEST_COMPLETE callback comes - // (that's why those are member variables). - LOG_TRACE("InternetReadFile() failed: ERROR_IO_PENDING. Waiting for INTERNET_STATUS_REQUEST_COMPLETE to be called again"); - return; + // SECURITY: refuse an over-large response instead of buffering it (see + // MAX_HTTP_RESPONSE_SIZE) so a hostile/MITM'd collector cannot exhaust + // process memory. Checked before every append so the buffer never exceeds + // the cap; reported as an invalid server response -> NetworkFailure (retried). + if (m_bodyBuffer.size() + m_bufferUsed > MAX_HTTP_RESPONSE_SIZE) { + LOG_WARN("HTTP response exceeds max buffered size (%zu bytes); aborting", MAX_HTTP_RESPONSE_SIZE); + dwError = ERROR_HTTP_INVALID_SERVER_RESPONSE; + } else { + m_bodyBuffer.insert(m_bodyBuffer.end(), m_buffer, m_buffer + m_bufferUsed); + while (!m_readingData || m_bufferUsed != 0) { + BOOL bResult = ::InternetReadFile(m_hWinInetRequest, m_buffer, sizeof(m_buffer), &m_bufferUsed); + m_readingData = true; + if (!bResult) { + dwError = GetLastError(); + if (dwError == ERROR_IO_PENDING) { + // Do not touch anything from this thread anymore. + // The buffer passed to InternetReadFile() and the + // read count will be filled asynchronously, so they + // must stay valid and writable until the next + // INTERNET_STATUS_REQUEST_COMPLETE callback comes + // (that's why those are member variables). + LOG_TRACE("InternetReadFile() failed: ERROR_IO_PENDING. Waiting for INTERNET_STATUS_REQUEST_COMPLETE to be called again"); + return; + } + LOG_WARN("InternetReadFile() failed: %d", dwError); + break; } - LOG_WARN("InternetReadFile() failed: %d", dwError); - break; - } - m_bodyBuffer.insert(m_bodyBuffer.end(), m_buffer, m_buffer + m_bufferUsed); + if (m_bodyBuffer.size() + m_bufferUsed > MAX_HTTP_RESPONSE_SIZE) { + LOG_WARN("HTTP response exceeds max buffered size (%zu bytes); aborting", MAX_HTTP_RESPONSE_SIZE); + dwError = ERROR_HTTP_INVALID_SERVER_RESPONSE; + break; + } + m_bodyBuffer.insert(m_bodyBuffer.end(), m_buffer, m_buffer + m_bufferUsed); + } } } diff --git a/lib/http/HttpClient_WinRt.cpp b/lib/http/HttpClient_WinRt.cpp index e46e4c49c..1efc1bb22 100644 --- a/lib/http/HttpClient_WinRt.cpp +++ b/lib/http/HttpClient_WinRt.cpp @@ -144,7 +144,7 @@ namespace MAT_NS_BEGIN { void SendHttpAsyncRequest(HttpRequestMessage ^req) { - IAsyncOperationWithProgress^ operation = m_parent.getHttpClient()->SendRequestAsync(req, HttpCompletionOption::ResponseContentRead); + IAsyncOperationWithProgress^ operation = m_parent.getHttpClient()->SendRequestAsync(req, HttpCompletionOption::ResponseHeadersRead); m_cancellationTokenSource = cancellation_token_source(); create_task(operation, m_cancellationTokenSource.get_token()). @@ -202,37 +202,98 @@ namespace MAT_NS_BEGIN { index++; } - auto operation = m_httpResponseMessage->Content->ReadAsBufferAsync(); - auto task = create_task(operation); - if (task.wait() == task_status::completed) + // Read content headers before streaming the body. + IMapView^ contentHeadersView = m_httpResponseMessage->Content->Headers->GetView(); + auto contentHeadersiterator = contentHeadersView->First(); + unsigned int contentHeadersIndex = 0; + while (contentHeadersIndex < contentHeadersView->Size) { - IMapView^ contentHeadersView = m_httpResponseMessage->Content->Headers->GetView(); + String^ Key = contentHeadersiterator->Current->Key; + String^ Value = contentHeadersiterator->Current->Value; - auto contentHeadersiterator = contentHeadersView->First(); - unsigned int contentHeadersIndex = 0; - while (contentHeadersIndex < contentHeadersView->Size) - { - String^ Key = contentHeadersiterator->Current->Key; - String^ Value = contentHeadersiterator->Current->Value; + response->m_headers.add(from_platform_string(Key), from_platform_string(Value)); + contentHeadersiterator->MoveNext(); + contentHeadersIndex++; + } - response->m_headers.add(from_platform_string(Key), from_platform_string(Value)); - contentHeadersiterator->MoveNext(); - contentHeadersIndex++; + // SECURITY: stream the body in bounded chunks and enforce + // MAX_HTTP_RESPONSE_SIZE. SendRequestAsync uses ResponseHeadersRead, so + // the framework does not pre-buffer the whole body; reading it here in + // chunks ensures an oversized response is never fully materialized in + // memory (a hostile/MITM'd collector cannot exhaust process memory). + // task::wait()/get() rethrow if a read faults, so guard the whole stream. + try + { + IInputStream^ inputStream = nullptr; + { + auto streamOp = m_httpResponseMessage->Content->ReadAsInputStreamAsync(); + auto streamTask = create_task(streamOp, m_cancellationTokenSource.get_token()); + auto status = streamTask.wait(); + if (status == task_status::completed) + { + inputStream = streamTask.get(); + } + else + { + // Caller-initiated cancel maps to Aborted; anything else is a failure. + response->m_result = (status == task_status::canceled) ? HttpResult_Aborted : HttpResult_NetworkFailure; + } } - auto buffer = task.get(); - size_t length = buffer->Length; - - if (length > 0) + if (inputStream != nullptr) { - response->m_body.reserve(length); - response->m_body.resize(length); - DataReader^ dataReader = DataReader::FromBuffer(buffer); - dataReader->ReadBytes((Platform::ArrayReference(reinterpret_cast(response->m_body.data()), (DWORD)length))); - dataReader->DetachBuffer(); - delete dataReader; + const unsigned int chunkSize = 64 * 1024; + for (;;) + { + Buffer^ chunk = ref new Buffer(chunkSize); + auto readOp = inputStream->ReadAsync(chunk, chunkSize, InputStreamOptions::Partial); + auto readTask = create_task(readOp, m_cancellationTokenSource.get_token()); + auto status = readTask.wait(); + if (status != task_status::completed) + { + // Drop any partial body; caller cancel -> Aborted, else failure. + response->m_result = (status == task_status::canceled) ? HttpResult_Aborted : HttpResult_NetworkFailure; + response->m_body.clear(); + break; + } + + IBuffer^ readBuffer = readTask.get(); + unsigned int readLength = (readBuffer != nullptr) ? readBuffer->Length : 0; + if (readLength == 0) + { + break; // end of stream + } + + if (response->m_body.size() + readLength > MAX_HTTP_RESPONSE_SIZE) + { + LOG_WARN("HTTP response exceeds max buffered size (%zu bytes); aborting", MAX_HTTP_RESPONSE_SIZE); + response->m_result = HttpResult_NetworkFailure; + response->m_body.clear(); + break; + } + + const size_t oldSize = response->m_body.size(); + response->m_body.resize(oldSize + readLength); + DataReader^ dataReader = DataReader::FromBuffer(readBuffer); + dataReader->ReadBytes((Platform::ArrayReference(reinterpret_cast(response->m_body.data() + oldSize), readLength))); + dataReader->DetachBuffer(); + delete dataReader; + } + delete inputStream; } } + catch (Platform::Exception^ ex) + { + // A faulted read rethrows here; drop any partial body and fail the request. + LOG_WARN("Reading HTTP response body failed: 0x%08x", ex->HResult); + response->m_result = HttpResult_NetworkFailure; + response->m_body.clear(); + } + catch (...) + { + response->m_result = HttpResult_NetworkFailure; + response->m_body.clear(); + } } else { diff --git a/lib/http/HttpResponseDecoder.cpp b/lib/http/HttpResponseDecoder.cpp index 11e9d4096..6014cec19 100644 --- a/lib/http/HttpResponseDecoder.cpp +++ b/lib/http/HttpResponseDecoder.cpp @@ -91,6 +91,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = DebugEventType::EVT_HTTP_OK; evt.param1 = response.GetStatusCode(); + evt.param2 = ctx->recordIdsAndTenantIds.size(); evt.data = static_cast(request.GetBody().data()); evt.size = request.GetBody().size(); DispatchEvent(evt); @@ -112,6 +113,7 @@ namespace MAT_NS_BEGIN { // This is to be addressed with ETW trace API that can send // a detailed error context to ETW provider. evt.param1 = response.GetStatusCode(); + evt.param2 = ctx->recordIdsAndTenantIds.size(); evt.data = static_cast(request.GetBody().data()); evt.size = request.GetBody().size(); DispatchEvent(evt); @@ -127,6 +129,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = DebugEventType::EVT_HTTP_FAILURE; evt.param1 = 0; // response.GetStatusCode(); + evt.param2 = ctx->recordIdsAndTenantIds.size(); DispatchEvent(evt); } ctx->httpResponse = nullptr; @@ -144,6 +147,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = DebugEventType::EVT_HTTP_FAILURE; evt.param1 = response.GetStatusCode(); + evt.param2 = ctx->recordIdsAndTenantIds.size(); DispatchEvent(evt); } temporaryServerFailure(ctx); @@ -157,6 +161,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = DebugEventType::EVT_HTTP_FAILURE; evt.param1 = response.GetStatusCode(); + evt.param2 = ctx->recordIdsAndTenantIds.size(); DispatchEvent(evt); } temporaryNetworkFailure(ctx); @@ -253,4 +258,3 @@ namespace MAT_NS_BEGIN { } } MAT_NS_END - diff --git a/lib/include/public/CompliantByDefaultFilterApi.hpp b/lib/include/public/CompliantByDefaultFilterApi.hpp index 642aa4c9e..85e779c3f 100644 --- a/lib/include/public/CompliantByDefaultFilterApi.hpp +++ b/lib/include/public/CompliantByDefaultFilterApi.hpp @@ -7,6 +7,7 @@ #include "ctmacros.hpp" +#include #include namespace MAT_NS_BEGIN { namespace Modules { namespace Filtering diff --git a/lib/include/public/EventProperties.hpp b/lib/include/public/EventProperties.hpp index a2aa3ecda..ef7f3b705 100644 --- a/lib/include/public/EventProperties.hpp +++ b/lib/include/public/EventProperties.hpp @@ -60,6 +60,18 @@ namespace MAT_NS_BEGIN /// EventProperties& operator=(EventProperties const& copy); + /// + /// The EventProperties move constructor. Transfers ownership of the + /// underlying storage (O(1)); the moved-from object is left empty and is + /// only valid to destroy or reassign. + /// + EventProperties(EventProperties&& move) noexcept; + + /// + /// The EventProperties move-assignment operator. + /// + EventProperties& operator=(EventProperties&& move) noexcept; + /// /// Constructs an EventProperties object from a map of string to EventProperty.
/// You must supply a non-empty name whenever you supply any custom properties for the event via EventProperties. diff --git a/lib/include/public/IHttpClient.hpp b/lib/include/public/IHttpClient.hpp index 89e5e6cf0..7a8678ceb 100644 --- a/lib/include/public/IHttpClient.hpp +++ b/lib/include/public/IHttpClient.hpp @@ -14,11 +14,24 @@ #include #include #include +#include ///@cond INTERNAL_DOCS namespace MAT_NS_BEGIN { class ILogConfiguration; + + /// + /// SECURITY: upper bound (in bytes) on an HTTP response body that a transport + /// will buffer. OneCollector protocol responses are small (status, kill-switch + /// tokens, retry-after, small config), so this generous cap never rejects a + /// legitimate response, but it stops a hostile or MITM'd collector from driving + /// unbounded memory growth by returning an oversized body (a memory-amplification + /// DoS of the embedding process). A transport that would exceed it refuses the + /// response and reports the request as a network failure so it is retried. + /// + static constexpr std::size_t MAX_HTTP_RESPONSE_SIZE = 16u * 1024u * 1024u; // 16 MB + /// /// The HttpHeaders class contains a set of HTTP headers. /// diff --git a/lib/include/public/ILogConfiguration.hpp b/lib/include/public/ILogConfiguration.hpp index 1cb8103b8..af1bc44c2 100644 --- a/lib/include/public/ILogConfiguration.hpp +++ b/lib/include/public/ILogConfiguration.hpp @@ -104,6 +104,16 @@ namespace MAT_NS_BEGIN /// static constexpr const char* const CFG_BOOL_ENABLE_NET_DETECT = "enableNetworkDetector"; + /// + /// Request collector-side scrubbing (obfuscation) of the client IP address. + /// Applied unless explicitly set to false (on by default; the key is not + /// present in the default configuration). Opt out when the client IP is + /// needed, e.g. for geo-location enrichment. Honored by the OneCollector + /// direct-upload path; in UTC mode client privacy is governed by the OS UTC + /// pipeline instead. + /// + static constexpr const char* const CFG_BOOL_ENABLE_IP_SCRUBBING = "enableIpScrubbing"; + /// /// Parameter that allows to check if the SDK is running on UTC mode /// diff --git a/lib/include/public/ISemanticContext.hpp b/lib/include/public/ISemanticContext.hpp index 76d9f991b..28d9d97b4 100644 --- a/lib/include/public/ISemanticContext.hpp +++ b/lib/include/public/ISemanticContext.hpp @@ -141,7 +141,7 @@ namespace MAT_NS_BEGIN break; default: - assert(!"Unknown NetworkCost enum value"); + assert(false && "Unknown NetworkCost enum value"); value = ""; break; } @@ -180,7 +180,7 @@ namespace MAT_NS_BEGIN break; default: - assert(!"Unknown NetworkType enum value"); + assert(false && "Unknown NetworkType enum value"); value = ""; break; } diff --git a/lib/include/public/LogSessionData.hpp b/lib/include/public/LogSessionData.hpp index 024f2272f..0a6737f36 100644 --- a/lib/include/public/LogSessionData.hpp +++ b/lib/include/public/LogSessionData.hpp @@ -29,7 +29,7 @@ namespace MAT_NS_BEGIN /// Gets the time that this session began. /// /// A 64-bit integer that contains the time. - uint64_t getSessionFirstTime() const; + uint64_t getSessionFirstTime() const noexcept; /// /// Gets the SDK unique identifier. diff --git a/lib/include/public/Version.hpp b/lib/include/public/Version.hpp index bb114f5a5..cf7ec9b99 100644 --- a/lib/include/public/Version.hpp +++ b/lib/include/public/Version.hpp @@ -6,8 +6,8 @@ #define MAT_VERSION_HPP // WARNING: DO NOT MODIFY THIS FILE! // This file has been automatically generated, manual changes will be lost. -#define BUILD_VERSION_STR "3.10.161.1" -#define BUILD_VERSION 3,10,161,1 +#define BUILD_VERSION_STR "3.10.173.1" +#define BUILD_VERSION 3,10,173,1 #ifndef RESOURCE_COMPILER_INVOKED #include "ctmacros.hpp" @@ -18,7 +18,7 @@ namespace MAT_NS_BEGIN { uint64_t const Version = ((uint64_t)3 << 48) | ((uint64_t)10 << 32) | - ((uint64_t)161 << 16) | + ((uint64_t)173 << 16) | ((uint64_t)1); } MAT_NS_END diff --git a/lib/include/public/ctmacros.hpp b/lib/include/public/ctmacros.hpp index 42547e41d..026176a04 100644 --- a/lib/include/public/ctmacros.hpp +++ b/lib/include/public/ctmacros.hpp @@ -28,9 +28,14 @@ #define MATSDK_LIBABI_CDECL __cdecl # if defined(MATSDK_SHARED_LIB) # define MATSDK_LIBABI __declspec(dllexport) +# elif defined(MATSDK_IMPORT_LIB) +// Consumer importing the public API from a shared mat.dll. The installed +// MSTelemetry::mat CMake target propagates this automatically when the SDK was +// built shared (see lib/CMakeLists.txt). +# define MATSDK_LIBABI __declspec(dllimport) # elif defined(MATSDK_STATIC_LIB) # define MATSDK_LIBABI -# else // Header file included by client +# else // Header file included by client; linkage unspecified # ifndef MATSDK_LIBABI # define MATSDK_LIBABI # endif @@ -47,13 +52,28 @@ #define MATSDK_LIBABI_CDECL #endif -#ifndef MATSDK_LIBABI -#define MATSDK_LIBABI +#ifndef MATSDK_LIBABI +// Mark the public API as default-visibility ONLY in shared builds, so it stays +// exported when the SDK is compiled with -fvisibility=hidden (see CMakeLists.txt). +// This mirrors the __declspec(dllexport) gating above: in a static build the +// attribute is omitted, so the public symbols inherit -fvisibility=hidden and are +// NOT re-exported when a consumer .so/.dylib statically absorbs libmat. (When a +// consumer includes this header, MATSDK_SHARED_LIB is not defined either, which +// is fine: the symbols are exported by the shared libmat they link against.) +# if (defined(__GNUC__) || defined(__clang__)) && defined(MATSDK_SHARED_LIB) +# define MATSDK_LIBABI __attribute__((visibility("default"))) +# else +# define MATSDK_LIBABI +# endif #endif -// TODO: [MG] - ideally we'd like to use __attribute__((unused)) with gcc/clang +// Cast the argument(s) to void so the parameter is genuinely referenced. An empty +// expansion left the parameter unused under -Wunused-parameter, which broke +// consumers compiling the SDK headers with -Wextra -Werror. On Windows the Win32 +// SDK provides its own UNREFERENCED_PARAMETER, so this definition only applies +// where that macro is not already defined. #ifndef UNREFERENCED_PARAMETER -#define UNREFERENCED_PARAMETER(...) +#define UNREFERENCED_PARAMETER(...) (void)(__VA_ARGS__) #endif #define OACR_USE_PTR(...) diff --git a/lib/jni/JniConvertors.cpp b/lib/jni/JniConvertors.cpp index 87b7cb15b..d944ddff8 100644 --- a/lib/jni/JniConvertors.cpp +++ b/lib/jni/JniConvertors.cpp @@ -159,8 +159,17 @@ EventProperties GetEventProperties(JNIEnv* env, const jstring& jstrEventName, co const jobjectArray& jEventPropertyStringKeyArray, const jobjectArray& jEventPropertyValueArray) { EventProperties eventProperties; eventProperties.SetName(JStringToStdString(env, jstrEventName)); - if (jstrEventType != NULL) - eventProperties.SetType(JStringToStdString(env, jstrEventType)); + if (jstrEventType != NULL) { + // An empty type means "unset" (the native default). Before #1329 the + // Java getType() returned null for a default EventProperties, so this + // branch was skipped. getType() now returns "" to fix a Java-side NPE; + // forwarding SetType("") here would fail native event-name validation + // and broadcast a spurious EVT_REJECTED for every typeless event, so + // only set a non-empty type. + std::string eventType = JStringToStdString(env, jstrEventType); + if (!eventType.empty()) + eventProperties.SetType(eventType); + } eventProperties.SetLatency(static_cast(jEventLatency)); eventProperties.SetPersistence(static_cast(jEventPersistence)); eventProperties.SetPopsample(static_cast(jEventPopSample)); diff --git a/lib/modules b/lib/modules index c637015fb..7bd8b516e 160000 --- a/lib/modules +++ b/lib/modules @@ -1 +1 @@ -Subproject commit c637015fbbe904ed556d27e3b9072f6f2a5ee401 +Subproject commit 7bd8b516e2d93d1704834e0895733ae7bc2d1f43 diff --git a/lib/offline/LogSessionDataProvider.cpp b/lib/offline/LogSessionDataProvider.cpp index 1615d84e7..68e152d0e 100644 --- a/lib/offline/LogSessionDataProvider.cpp +++ b/lib/offline/LogSessionDataProvider.cpp @@ -49,7 +49,7 @@ namespace MAT_NS_BEGIN } } - LogSessionData* LogSessionDataProvider::GetLogSessionData() + LogSessionData* LogSessionDataProvider::GetLogSessionData() noexcept { return m_logSessionData.get(); } @@ -69,7 +69,7 @@ namespace MAT_NS_BEGIN sessionSDKUid = PAL::generateUuidString(); if (!m_offlineStorage->StoreSetting(sessionFirstLaunchTimeName, std::to_string(sessionFirstTimeLaunch))) { - LOG_WARN("Unable to save session analytics to DB for %d", sessionFirstLaunchTimeName); + LOG_WARN("Unable to save session analytics to DB for %s", sessionFirstLaunchTimeName); } if (!m_offlineStorage->StoreSetting(sessionSdkUidName, sessionSDKUid)) { LOG_WARN("Unable to save session analytics to DB for %s", sessionSDKUid.c_str()); @@ -87,11 +87,11 @@ namespace MAT_NS_BEGIN } if (!m_offlineStorage->DeleteSetting(sessionFirstLaunchTimeName)) { - LOG_WARN("Unable to delete session analytics from DB for %d", sessionFirstLaunchTimeName); + LOG_WARN("Unable to delete session analytics from DB for %s", sessionFirstLaunchTimeName); } if (!m_offlineStorage->DeleteSetting(sessionSdkUidName)) { - LOG_WARN("Unable to delete session analytics from DB for %d", sessionSdkUidName); + LOG_WARN("Unable to delete session analytics from DB for %s", sessionSdkUidName); } } @@ -153,25 +153,32 @@ namespace MAT_NS_BEGIN return true; } - uint64_t LogSessionDataProvider::convertStrToLong(const std::string& s) + uint64_t LogSessionDataProvider::convertStrToLong(const std::string& s) noexcept { uint64_t res = 0ull; char *endptr = nullptr; - res = std::strtoll(s.c_str(), &endptr, 10); - if (errno == ERANGE && (res == LONG_MAX || res == 0 )) + // strtoull silently wraps a leading '-' into a large value, so reject + // negative input explicitly before parsing. + size_t firstNonSpace = s.find_first_not_of(" \t\n\r\f\v"); + if (firstNonSpace != std::string::npos && s[firstNonSpace] == '-') { - LOG_WARN ("Converted value falls out of uint64_t range."); - res = 0; - } - else if ( 0 != errno && 0 == res ) + LOG_WARN ("Converted value is negative; rejecting."); + return 0; + } + errno = 0; + unsigned long long parsed = std::strtoull(s.c_str(), &endptr, 10); + if (errno == ERANGE) { - LOG_WARN("Conversion cannot be performed."); + LOG_WARN ("Converted value falls out of range."); } - else if (std::strlen(endptr) > 0) + else if (endptr == s.c_str() || std::strlen(endptr) > 0) { - LOG_WARN ("Conversion cannot be performed. Alphanumeric characters present"); - res = 0; - } + LOG_WARN ("Conversion cannot be performed."); + } + else + { + res = static_cast(parsed); + } return res; } @@ -193,7 +200,7 @@ namespace MAT_NS_BEGIN } } - void LogSessionDataProvider::remove_eol(std::string& result) + void LogSessionDataProvider::remove_eol(std::string& result) noexcept { if (!result.empty() && result[result.length() - 1] == '\n') { diff --git a/lib/offline/LogSessionDataProvider.hpp b/lib/offline/LogSessionDataProvider.hpp index be782095e..3b45a9fc8 100644 --- a/lib/offline/LogSessionDataProvider.hpp +++ b/lib/offline/LogSessionDataProvider.hpp @@ -21,7 +21,7 @@ namespace MAT_NS_BEGIN { public: LogSessionDataProvider( - IOfflineStorage* offlineStorage) + IOfflineStorage* offlineStorage) noexcept : m_offlineStorage(offlineStorage), m_storageType(SessionStorageType::DatabaseStore), @@ -41,7 +41,7 @@ namespace MAT_NS_BEGIN void CreateLogSessionData(); void ResetLogSessionData(); void DeleteLogSessionData(); - LogSessionData *GetLogSessionData(); + LogSessionData *GetLogSessionData() noexcept; protected: void CreateLogSessionDataFromFile(); @@ -55,9 +55,9 @@ namespace MAT_NS_BEGIN std::string const m_cacheFilePath; SessionStorageType m_storageType; std::unique_ptr m_logSessionData; - static uint64_t convertStrToLong(const std::string&); + static uint64_t convertStrToLong(const std::string&) noexcept; static void writeFileContents(const std::string&, uint64_t, const std::string&); - void remove_eol(std::string& ); + void remove_eol(std::string& ) noexcept; }; } MAT_NS_END diff --git a/lib/offline/MemoryStorage.hpp b/lib/offline/MemoryStorage.hpp index 8a378dc5d..32dc82bdf 100644 --- a/lib/offline/MemoryStorage.hpp +++ b/lib/offline/MemoryStorage.hpp @@ -24,7 +24,7 @@ namespace MAT_NS_BEGIN { - class MemoryStorage : public IOfflineStorage + class MemoryStorage final : public IOfflineStorage { public: @@ -69,7 +69,7 @@ namespace MAT_NS_BEGIN { virtual size_t GetRemainingRecordCountForShutdown() const override; - virtual size_t GetReservedCount(); + size_t GetReservedCount(); virtual std::vector GetRecords(bool shutdown = false, EventLatency minLatency = EventLatency_Unspecified, unsigned maxCount = 0) override; diff --git a/lib/offline/OfflineStorageHandler.cpp b/lib/offline/OfflineStorageHandler.cpp index 9049339c4..52ce15515 100644 --- a/lib/offline/OfflineStorageHandler.cpp +++ b/lib/offline/OfflineStorageHandler.cpp @@ -33,6 +33,7 @@ namespace MAT_NS_BEGIN { m_shutdownStarted(false), m_memoryDbSize(0), m_queryDbSize(0), + m_cacheMemorySizeLimitInBytes(0), m_isStorageFullNotificationSend(false) { // TODO: [MG] - OfflineStorage_SQLite.cpp is performing similar checks @@ -83,7 +84,7 @@ namespace MAT_NS_BEGIN { void OfflineStorageHandler::Initialize(IOfflineStorageObserver& observer) { m_observer = &observer; - uint32_t cacheMemorySizeLimitInBytes = m_config[CFG_INT_RAM_QUEUE_SIZE]; + m_cacheMemorySizeLimitInBytes = m_config[CFG_INT_RAM_QUEUE_SIZE]; m_offlineStorageDisk = OfflineStorageFactory::Create(m_logManager, m_config); if (m_offlineStorageDisk) @@ -94,7 +95,7 @@ namespace MAT_NS_BEGIN { // TODO: [MG] - consider passing m_offlineStorageDisk to m_offlineStorageMemory, // so that the Flush() op on memory storage leads to saving unflushed events to // disk. - if (cacheMemorySizeLimitInBytes > 0) + if (m_cacheMemorySizeLimitInBytes > 0) { m_offlineStorageMemory.reset(new MemoryStorage(m_logManager, m_config)); m_offlineStorageMemory->Initialize(*this); @@ -174,7 +175,7 @@ namespace MAT_NS_BEGIN { // than the handle gets replaced by nullptr in this DeferredCallbackHandle obj. m_flushHandle.Cancel(); - size_t dbSizeBeforeFlush = m_offlineStorageMemory->GetSize(); + size_t dbSizeBeforeFlush = (m_offlineStorageMemory != nullptr) ? m_offlineStorageMemory->GetSize() : 0; if ((m_offlineStorageMemory) && (dbSizeBeforeFlush > 0) && (m_offlineStorageDisk)) { // This will block on and then take a lock for the duration of this move, and @@ -233,8 +234,10 @@ namespace MAT_NS_BEGIN { return false; } - // Check cache size only once at start - static uint32_t cacheMemorySizeLimitInBytes = m_config[CFG_INT_RAM_QUEUE_SIZE]; + // Cache size limit is per-instance config computed once in Initialize(); + // it must NOT be a function-local static, which would share the first + // LogManager's value with every other LogManager instance. + uint32_t cacheMemorySizeLimitInBytes = m_cacheMemorySizeLimitInBytes; if (nullptr != m_offlineStorageMemory && !m_shutdownStarted) { diff --git a/lib/offline/OfflineStorageHandler.hpp b/lib/offline/OfflineStorageHandler.hpp index e7bdce4cb..9a1131aff 100644 --- a/lib/offline/OfflineStorageHandler.hpp +++ b/lib/offline/OfflineStorageHandler.hpp @@ -23,7 +23,7 @@ namespace MAT_NS_BEGIN { - class OfflineStorageHandler : public IOfflineStorage, public IOfflineStorageObserver + class OfflineStorageHandler final : public IOfflineStorage, public IOfflineStorageObserver { public: OfflineStorageHandler(ILogManager& logManager, IRuntimeConfig& runtimeConfig, ITaskDispatcher& taskDispatcher); @@ -64,7 +64,7 @@ namespace MAT_NS_BEGIN { virtual void OnStorageRecordsSaved(size_t numRecords) override; protected: - virtual void DeleteRecordsByKeys(const std::list & keys); + void DeleteRecordsByKeys(const std::list & keys); IOfflineStorageObserver * m_observer; ILogManager & m_logManager; @@ -75,7 +75,7 @@ namespace MAT_NS_BEGIN { KillSwitchManager m_killSwitchManager; ClockSkewManager m_clockSkewManager; - virtual bool isKilled(StorageRecord const& record); + bool isKilled(StorageRecord const& record); std::mutex m_flushLock; bool m_flushPending; @@ -92,6 +92,7 @@ namespace MAT_NS_BEGIN { unsigned m_memoryDbSize; unsigned m_memoryDbSizeNotificationLimit; unsigned m_queryDbSize; + uint32_t m_cacheMemorySizeLimitInBytes; bool m_isStorageFullNotificationSend; protected: diff --git a/lib/offline/OfflineStorage_Room.cpp b/lib/offline/OfflineStorage_Room.cpp index 72a04d0ed..d052e7a9d 100644 --- a/lib/offline/OfflineStorage_Room.cpp +++ b/lib/offline/OfflineStorage_Room.cpp @@ -240,6 +240,14 @@ namespace MAT_NS_BEGIN MATSDK_THROW(std::logic_error("whereFilter not implemented")); } + if (!env) + { + return; + } + if (!m_room) + { + return; + } auto room_class = env->GetObjectClass(m_room); auto deleteByToken = env->GetMethodID(room_class, "deleteByToken", @@ -274,6 +282,10 @@ namespace MAT_NS_BEGIN { return; } + if (!m_room) + { + return; + } auto room_class = env->GetObjectClass(m_room); auto method = env->GetMethodID(room_class, "deleteById", "([J)J"); ThrowLogic(env, "Unable to get deleteById method"); @@ -377,6 +389,10 @@ namespace MAT_NS_BEGIN { return false; } + if (!m_room) + { + return false; + } auto room_class = env->GetObjectClass(m_room); auto reserve = env->GetMethodID(room_class, "getAndReserve", "(IJJJ)[Lcom/microsoft/applications/events/StorageRecord;"); @@ -424,11 +440,26 @@ namespace MAT_NS_BEGIN int persist_lb = static_cast(EventPersistence_Normal); int persist_ub = static_cast(EventPersistence_DoNotStoreOnDisk); + // Set if a null array element is hit below, so the early-release + // path skips releaseUnconsumed (which would index into the null). + bool sawNullElement = false; for (index = 0; index < limit; ++index) { env.pushLocalFrame(32); auto record = env->GetObjectArrayElement(selected, index); ThrowLogic(env, "getAndReserve element"); + if (!record) + { + // Null array element (observed with some androidx.room + // versions): pop this frame and stop rather than + // dereferencing null in GetObjectClass. We cannot safely + // release the tail here (it contains this null and Java + // releaseUnconsumed indexes from 0), so leave the + // remaining reservations to expire and be retried. + sawNullElement = true; + env.popLocalFrame(); + break; + } if (!record_class) { // Promote to a global ref so it survives popLocalFrame on @@ -518,11 +549,14 @@ namespace MAT_NS_BEGIN if (index < limit) { // we did not consume all these events - auto release = env->GetMethodID(room_class, "releaseUnconsumed", - "([Lcom/microsoft/applications/events/StorageRecord;I)V"); - ThrowLogic(env, "releaseUnconsumed"); - env->CallVoidMethod(m_room, release, selected, static_cast(index)); - ThrowRuntime(env, "call ru"); + if (!sawNullElement) + { + auto release = env->GetMethodID(room_class, "releaseUnconsumed", + "([Lcom/microsoft/applications/events/StorageRecord;I)V"); + ThrowLogic(env, "releaseUnconsumed"); + env->CallVoidMethod(m_room, release, selected, static_cast(index)); + ThrowRuntime(env, "call ru"); + } break; // break out of the request > collected loop--end early by request } } @@ -633,6 +667,14 @@ namespace MAT_NS_BEGIN try { ConnectedEnv env(s_vm); + if (!env) + { + return; + } + if (!m_room) + { + return; + } auto room_class = env->GetObjectClass(m_room); ThrowLogic(env, "GetObjectClass for m_room"); auto release = env->GetMethodID(room_class, @@ -700,6 +742,12 @@ namespace MAT_NS_BEGIN env.pushLocalFrame(8); auto byTenant = env->GetObjectArrayElement(results, index); ThrowRuntime(env, "Exception fetching element from results"); + if (!byTenant) + { + // Skip a null array element rather than dereference null. + env.popLocalFrame(); + continue; + } if (!bt_class) { // Promote to a global ref so it survives popLocalFrame. @@ -794,6 +842,10 @@ namespace MAT_NS_BEGIN static constexpr char newRecordSignature[] = "(JIIJIJ[B)Lcom/microsoft/applications/events/StorageRecord;"; + if (!m_room) + { + return 0; + } auto room_class = env->GetObjectClass(m_room); size_t count = std::min(records.size(), INT32_MAX); @@ -924,6 +976,14 @@ namespace MAT_NS_BEGIN try { ConnectedEnv env(s_vm); + if (!env) + { + return false; + } + if (!m_room) + { + return false; + } auto room_class = env->GetObjectClass(m_room); auto delete_method = env->GetMethodID(room_class, "deleteSetting", "(Ljava/lang/String;)V"); @@ -964,6 +1024,14 @@ namespace MAT_NS_BEGIN try { ConnectedEnv env(s_vm); + if (!env) + { + return false; + } + if (!m_room) + { + return false; + } auto room_class = env->GetObjectClass(m_room); jmethodID store_setting = env->GetMethodID( room_class, @@ -1081,6 +1149,10 @@ namespace MAT_NS_BEGIN size_t OfflineStorage_Room::GetSizeInternal(ConnectedEnv& env) const { + if (!m_room) + { + return 0; + } auto room_class = env->GetObjectClass(m_room); auto method = env->GetMethodID(room_class, "totalSize", "()J"); if (!method) @@ -1107,6 +1179,10 @@ namespace MAT_NS_BEGIN { return 0; } + if (!m_room) + { + return 0; + } auto room_class = env->GetObjectClass(m_room); auto count_id = env->GetMethodID(room_class, "getRecordCount", "(I)J"); ThrowLogic(env, "getRecordCount"); @@ -1162,6 +1238,10 @@ namespace MAT_NS_BEGIN { return false; } + if (!m_room) + { + return false; + } auto room_class = env->GetObjectClass(m_room); auto trim_id = env->GetMethodID(room_class, "trim", "(J)J"); ThrowLogic(env, "trim"); @@ -1192,6 +1272,14 @@ namespace MAT_NS_BEGIN { ConnectedEnv env(s_vm); + if (!env) + { + return records; + } + if (!m_room) + { + return records; + } auto room_class = env->GetObjectClass(m_room); auto method = env->GetMethodID(room_class, "getRecords", "(ZIJ)[Lcom/microsoft/applications/events/StorageRecord;"); diff --git a/lib/offline/SQLiteWrapper.hpp b/lib/offline/SQLiteWrapper.hpp index 3f4f998e3..2a5f0d108 100644 --- a/lib/offline/SQLiteWrapper.hpp +++ b/lib/offline/SQLiteWrapper.hpp @@ -16,6 +16,11 @@ #include #include +#if !defined(_WIN32) +#include +#include +#endif + namespace MAT_NS_BEGIN { using SQLRecord = std::vector; @@ -249,15 +254,36 @@ namespace MAT_NS_BEGIN { // We cannot call plain ::remove() here, filename is in UTF-8. Rather // than adding a new set of functions to PAL, let's use SQLite VFS. sqlite3_vfs* vfs = g_sqlite3Proxy->sqlite3_vfs_find(NULL); - result = (vfs != NULL) ? vfs->xDelete(vfs, filename.c_str(), 0) : SQLITE_ERROR; - if (result == SQLITE_OK) { - LOG_INFO("Unusable existing database file was successfully deleted"); - } - else if (result != SQLITE_IOERR_DELETE_NOENT) { - LOG_WARN("Failed to delete unusable database file (%d)", result); + if (vfs == NULL) { + LOG_ERROR("Failed to delete unusable database file: no SQLite VFS"); shutdown_sqlite(); return false; } + // Delete the main database file plus any SQLite companion files + // (-journal/-wal/-shm) left behind by the failed open. A stale + // rollback journal or WAL can otherwise prevent the freshly created + // database below from opening cleanly (observed on iOS, where leaving + // the companions behind made the recreate() open fail). xDelete + // returns SQLITE_IOERR_DELETE_NOENT when a file is already absent, + // which is expected and not an error. + static const char* const companionSuffixes[] = { "", "-journal", "-wal", "-shm" }; + for (const char* suffix : companionSuffixes) { + const std::string companion = filename + suffix; + result = vfs->xDelete(vfs, companion.c_str(), 0); + if (result == SQLITE_OK) { + LOG_INFO("Deleted unusable database file \"%s\"", suffix); + } + else if (result != SQLITE_IOERR_DELETE_NOENT) { + LOG_WARN("Failed to delete database file \"%s\" (%d)", suffix, result); + // Only the main database file is fatal here; a leftover + // companion that cannot be removed must not by itself abort + // the recreate, since the open below may still succeed. + if (suffix[0] == '\0') { + shutdown_sqlite(); + return false; + } + } + } } // Take basename only, potential PII like profile name in the path must not be logged @@ -279,6 +305,31 @@ namespace MAT_NS_BEGIN { g_sqlite3Proxy->sqlite3_extended_result_codes(m_db, 1); + // SECURITY: the offline cache buffers pending telemetry/audit events + // (tenant ids, user identifiers, serialized event payloads). SQLite creates + // the database file with SQLITE_DEFAULT_FILE_PERMISSIONS -- 0644, i.e. + // world-readable -- so restrict it to owner read/write only (0600). This runs + // before WAL is enabled: SQLite derives the -wal/-journal permissions from the + // main database file (findCreateFileMode), so companions it creates inherit + // 0600. A cache created by an older SDK (before this fix) may already have + // companion files on disk with the old 0644 mode, so tighten any pre-existing + // ones too. POSIX only -- on Windows the Unix mode bits are meaningless (access + // is governed by NTFS ACLs). Best-effort: a failure (e.g. a filesystem that + // ignores chmod) must not fail the open, and a missing file -- ENOENT, e.g. an + // in-memory ":memory:" database, which has no file to secure -- is expected and + // silently ignored. +#if !defined(_WIN32) + if (::chmod(filename.c_str(), S_IRUSR | S_IWUSR) != 0 && errno != ENOENT) { + LOG_WARN("Could not restrict database file permissions to 0600 (errno %d)", errno); + } + for (const char* suffix : { "-wal", "-shm", "-journal" }) { + std::string companion = filename + suffix; + if (::chmod(companion.c_str(), S_IRUSR | S_IWUSR) != 0 && errno != ENOENT) { + LOG_WARN("Could not restrict %s file permissions to 0600 (errno %d)", suffix, errno); + } + } +#endif + if (!registerTokenizeFunction()) { shutdown(); return false; diff --git a/lib/pal/PAL.cpp b/lib/pal/PAL.cpp index 01c6e6f75..0fc28abfb 100644 --- a/lib/pal/PAL.cpp +++ b/lib/pal/PAL.cpp @@ -369,19 +369,32 @@ namespace PAL_NS_BEGIN { std::transform(uuidStr.begin(), uuidStr.end(), uuidStr.begin(), ::tolower); return uuidStr; #else - static std::once_flag flag; - std::call_once(flag, [](){ - auto now = std::chrono::high_resolution_clock::now(); - auto nanos = std::chrono::duration_cast(now.time_since_epoch()).count(); - std::srand(static_cast(std::time(0) ^ nanos)); - }); + // Use std::random_device -- a non-deterministic, CSPRNG-backed source on + // the platforms we target (glibc/bionic/libc++ draw from getrandom or + // /dev/urandom) -- instead of std::rand()/srand(time(0)), so the session + // and event identifiers built from it are not predictable. It is + // thread_local so the backing source is opened once per thread rather than + // on every call (generateUuidString is on the event logging hot path), and + // the 128 bits are drawn with four operator() calls instead of eleven + // (random_device::max() is guaranteed to span the full unsigned int range). + thread_local std::random_device rd; GUID_t uuid; - uuid.Data1 = (static_cast(std::rand()) << 16) | static_cast(std::rand()); - uuid.Data2 = static_cast(std::rand()); - uuid.Data3 = static_cast(std::rand()); - for (size_t i = 0; i < sizeof(uuid.Data4); i++) - uuid.Data4[i] = static_cast(std::rand()); + const uint32_t r0 = rd(); + const uint32_t r1 = rd(); + const uint32_t r2 = rd(); + const uint32_t r3 = rd(); + uuid.Data1 = r0; + uuid.Data2 = static_cast(r1); + uuid.Data3 = static_cast(r1 >> 16); + uuid.Data4[0] = static_cast(r2); + uuid.Data4[1] = static_cast(r2 >> 8); + uuid.Data4[2] = static_cast(r2 >> 16); + uuid.Data4[3] = static_cast(r2 >> 24); + uuid.Data4[4] = static_cast(r3); + uuid.Data4[5] = static_cast(r3 >> 8); + uuid.Data4[6] = static_cast(r3 >> 16); + uuid.Data4[7] = static_cast(r3 >> 24); // TODO: [MG] - replace this sprintf by more robust GUID to string converter char buf[40] = { 0 }; @@ -417,7 +430,26 @@ namespace PAL_NS_BEGIN { { #ifdef _WIN32 FILETIME tocks; - ::GetSystemTimeAsFileTime(&tocks); + // Resolve the precise API dynamically so the SDK retains its Windows 7 + // runtime compatibility and falls back when the API is unavailable. + using GetSystemTimePreciseAsFileTimeProc = VOID (WINAPI*)(LPFILETIME); + static const GetSystemTimePreciseAsFileTimeProc getSystemTimePreciseAsFileTime = + []() -> GetSystemTimePreciseAsFileTimeProc + { + HMODULE kernel32 = ::GetModuleHandleW(L"kernel32.dll"); + return kernel32 + ? reinterpret_cast( + ::GetProcAddress(kernel32, "GetSystemTimePreciseAsFileTime")) + : nullptr; + }(); + if (getSystemTimePreciseAsFileTime) + { + getSystemTimePreciseAsFileTime(&tocks); + } + else + { + ::GetSystemTimeAsFileTime(&tocks); + } ULONGLONG ticks = (ULONGLONG(tocks.dwHighDateTime) << 32) | tocks.dwLowDateTime; // number of days from beginning to 1601 multiplied by ticks per day return ticks + 0x701ce1722770000ULL; @@ -427,10 +459,9 @@ namespace PAL_NS_BEGIN { // This UTC epoch contract has been signed in blood since C++20 std::chrono::time_point now = std::chrono::system_clock::now(); auto duration = now.time_since_epoch(); - auto millis = std::chrono::duration_cast(duration).count(); - uint64_t ticks = millis; - ticks *= 10000; // convert millis to ticks (1 tick = 100ns) - ticks += 0x89F7FF5F7B58000ULL; // UTC time 0 in .NET ticks + auto nanos = std::chrono::duration_cast(duration).count(); + int64_t ticks = nanos / 100; // convert nanoseconds to .NET ticks (1 tick = 100ns) + ticks += static_cast(0x89F7FF5F7B58000ULL); // UTC time 0 in .NET ticks return ticks; #endif } diff --git a/lib/pal/TaskDispatcher_CAPI.cpp b/lib/pal/TaskDispatcher_CAPI.cpp index f8e432c74..e75ee1924 100644 --- a/lib/pal/TaskDispatcher_CAPI.cpp +++ b/lib/pal/TaskDispatcher_CAPI.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -37,7 +38,18 @@ namespace PAL_NS_BEGIN { void OnCallback() { if (m_task) { - (*m_task)(); + // The task is host/user code running on the external dispatcher's + // thread; an exception escaping here would terminate the process. + // Log it (mirroring WorkerThread) instead of swallowing silently. + try { + (*m_task)(); + } + catch (const std::exception& ex) { + LOG_ERROR("Unhandled exception in CAPI task: %s", ex.what()); + } + catch (...) { + LOG_ERROR("Unhandled non-standard exception in CAPI task"); + } } ReleaseItem(); } diff --git a/lib/pal/WorkerThread.cpp b/lib/pal/WorkerThread.cpp index 2bdbf6c67..3adfb9e61 100644 --- a/lib/pal/WorkerThread.cpp +++ b/lib/pal/WorkerThread.cpp @@ -6,6 +6,8 @@ #include "pal/WorkerThread.hpp" #include "pal/PAL.hpp" +#include + #if defined(MATSDK_PAL_CPP11) || defined(MATSDK_PAL_WIN32) /* Maximum scheduler interval for SDK is 1 hour required for clamping in case of monotonic clock drift */ @@ -238,7 +240,19 @@ namespace PAL_NS_BEGIN { // Item wasn't cancelled before it could be executed if (self->m_itemInProgress != nullptr) { LOG_TRACE("%10llu Execute item=%p type=%s\n", wakeupCount, item.get(), item.get()->TypeName.c_str() ); - (*item)(); + // A task can run arbitrary work (storage I/O, HTTP encode, and + // user DebugEventListener callbacks). An exception escaping here + // would unwind out of the thread entry function and call + // std::terminate, killing the host process. Contain it. + try { + (*item)(); + } + catch (const std::exception& ex) { + LOG_ERROR("Unhandled exception in worker task: %s", ex.what()); + } + catch (...) { + LOG_ERROR("Unhandled non-standard exception in worker task"); + } self->m_itemInProgress = nullptr; } diff --git a/lib/pal/posix/DeviceInformationImpl_Android.cpp b/lib/pal/posix/DeviceInformationImpl_Android.cpp index 61746d664..fc132629d 100644 --- a/lib/pal/posix/DeviceInformationImpl_Android.cpp +++ b/lib/pal/posix/DeviceInformationImpl_Android.cpp @@ -52,8 +52,10 @@ namespace PAL_NS_BEGIN { ///// IDeviceInformation API DeviceInformationImpl::DeviceInformationImpl(IRuntimeConfig& configuration) : + m_os_architecture(OsArchitectureType_Unknown), + m_powerSource(PowerSource_Battery), m_info_helper(), - m_powerSource(PowerSource_Battery) + m_registeredCount(0) {} std::string DeviceInformationImpl::GetDeviceTicket() const @@ -260,4 +262,3 @@ Java_com_microsoft_applications_events_HttpClient_onPowerChange(JNIEnv* env, PAL::AndroidDeviceInformationConnector::setModel(std::string(start, end)); env->ReleaseStringUTFChars(model, start); } - diff --git a/lib/pal/posix/NetworkInformationImpl_Android.cpp b/lib/pal/posix/NetworkInformationImpl_Android.cpp index 0c12464b4..04f1960f5 100644 --- a/lib/pal/posix/NetworkInformationImpl_Android.cpp +++ b/lib/pal/posix/NetworkInformationImpl_Android.cpp @@ -42,8 +42,10 @@ namespace PAL_NS_BEGIN { NetworkCost AndroidNetcostConnector::s_cost = NetworkCost_Unknown; NetworkInformationImpl::NetworkInformationImpl(IRuntimeConfig& configuration) : - m_info_helper(), + m_type(NetworkType_Unknown), m_cost(NetworkCost_Unknown), + m_info_helper(), + m_registeredCount(0), m_isNetDetectEnabled(configuration[CFG_BOOL_ENABLE_NET_DETECT]){}; NetworkInformationImpl::~NetworkInformationImpl() {}; @@ -156,4 +158,3 @@ Java_com_microsoft_applications_events_HttpClient_onCostChange(JNIEnv* env, { PAL::AndroidNetcostConnector::UpdateCost(isMetered ? NetworkCost_Metered : NetworkCost_Unmetered); } - diff --git a/lib/pal/posix/SystemInformationImpl_Android.cpp b/lib/pal/posix/SystemInformationImpl_Android.cpp index 15e0bb9b1..b1911f8ae 100644 --- a/lib/pal/posix/SystemInformationImpl_Android.cpp +++ b/lib/pal/posix/SystemInformationImpl_Android.cpp @@ -163,8 +163,8 @@ namespace PAL_NS_BEGIN { std::string AndroidSystemInformationConnector::s_device_class; SystemInformationImpl::SystemInformationImpl(IRuntimeConfig& configuration) : - m_info_helper(), - m_os_name("Android") + m_os_name("Android"), + m_info_helper() { if (configuration.HasConfig(CFG_PTR_ANDROID_JVM) && configuration.HasConfig(CFG_JOBJECT_ANDROID_ACTIVITY)) { @@ -245,4 +245,3 @@ extern "C" JNIEXPORT void JNICALL Java_com_microsoft_applications_events_HttpCli PAL::AndroidSystemInformationConnector::s_device_class, deviceClass); } - diff --git a/lib/shared/EventPropertiesCX.cpp b/lib/shared/EventPropertiesCX.cpp index 9c212d919..889965f6d 100644 --- a/lib/shared/EventPropertiesCX.cpp +++ b/lib/shared/EventPropertiesCX.cpp @@ -26,7 +26,7 @@ namespace Microsoft { FromPlatformMap(propertiesMap, properties); FromPlatformMap(this->PIITags, piiTags); - for (map::iterator it = properties.begin(); it != properties.end(); ++it) + for (typename map::iterator it = properties.begin(); it != properties.end(); ++it) { MAT::PiiKind piiType = MAT::PiiKind_None; auto tag = piiTags.find(it->first); diff --git a/lib/shared/PlatformHelpers.h b/lib/shared/PlatformHelpers.h index 47d1e723d..222ee0024 100644 --- a/lib/shared/PlatformHelpers.h +++ b/lib/shared/PlatformHelpers.h @@ -105,6 +105,11 @@ namespace Microsoft { void ThrowPlatformInvalidArgumentException(String^ message); void ThrowPlatformException(String^ message); + // Forward declaration so the FromPlatformMap templates below can + // resolve this helper under /permissive- two-phase name lookup + // (the definition appears later in this header). + std::string FromPlatformString(String^ platformString); + // Defining the template function in the header file eliminates the need in additional linker definitions. // platformmaptype can be read-only or editable platform map. template class platformmaptype> diff --git a/lib/shared/dllmain.cpp b/lib/shared/dllmain.cpp index 4cfa868c6..d584c56c6 100644 --- a/lib/shared/dllmain.cpp +++ b/lib/shared/dllmain.cpp @@ -18,7 +18,7 @@ #ifdef _MANAGED #pragma unmanaged #endif -unsigned thread_count = 0; +static unsigned thread_count = 0; BOOL APIENTRY DllMain(HMODULE /* hModule */, DWORD ul_reason_for_call, LPVOID /* lpReserved */) { diff --git a/lib/stats/Statistics.cpp b/lib/stats/Statistics.cpp index 773dd4d13..a1377ac37 100644 --- a/lib/stats/Statistics.cpp +++ b/lib/stats/Statistics.cpp @@ -9,6 +9,7 @@ #include "ILogManager.hpp" #include "mat/config.h" #include "utils/Utils.hpp" +#include "decorators/RecordFlagConstants.hpp" #include namespace MAT_NS_BEGIN { @@ -83,6 +84,13 @@ namespace MAT_NS_BEGIN { result &= m_baseDecorator.decorate(record); // Allow stats to capture Part A common properties, but not the custom result &= m_semanticContextDecorator.decorate(record, true); + // Stats events bypass EventPropertiesDecorator, so apply the same + // collector-side client-IP scrub here (on by default; opt out via + // CFG_BOOL_ENABLE_IP_SCRUBBING = false). + if (!m_config.HasConfig(CFG_BOOL_ENABLE_IP_SCRUBBING) || m_config[CFG_BOOL_ENABLE_IP_SCRUBBING]) + { + record.flags |= RECORD_FLAGS_EVENTTAG_SCRUB_IP; + } if (result) { IncomingEventContext evt(PAL::generateUuidString(), tenantToken, EventLatency_Normal, EventPersistence_Normal, &record); diff --git a/lib/system/EventProperties.cpp b/lib/system/EventProperties.cpp index ebe62ff58..2ade77741 100644 --- a/lib/system/EventProperties.cpp +++ b/lib/system/EventProperties.cpp @@ -90,8 +90,33 @@ namespace MAT_NS_BEGIN { EventProperties& EventProperties::operator=(EventProperties const& copy) { - *m_storage = *copy.m_storage; + // m_storage may be null if this object was moved-from; reallocate then. + if (m_storage == nullptr) + { + m_storage = new EventPropertiesStorage(*copy.m_storage); + } + else + { + *m_storage = *copy.m_storage; + } + + return *this; + } + + EventProperties::EventProperties(EventProperties&& move) noexcept + : m_storage(move.m_storage) + { + move.m_storage = nullptr; + } + EventProperties& EventProperties::operator=(EventProperties&& move) noexcept + { + if (this != &move) + { + delete m_storage; + m_storage = move.m_storage; + move.m_storage = nullptr; + } return *this; } diff --git a/lib/system/EventProperty.cpp b/lib/system/EventProperty.cpp index 3bf9c3f3c..6d0582440 100644 --- a/lib/system/EventProperty.cpp +++ b/lib/system/EventProperty.cpp @@ -307,10 +307,13 @@ namespace MAT_NS_BEGIN { // How to sort 2 objects (needed for maps) bool GUID_t::operator<(GUID_t const& other) const { - return Data1 < other.Data1 || - Data2 < other.Data2 || - Data3 == other.Data3 || - (memcmp(Data4, other.Data4, sizeof(Data4)) < 0); + if (Data1 != other.Data1) + return Data1 < other.Data1; + if (Data2 != other.Data2) + return Data2 < other.Data2; + if (Data3 != other.Data3) + return Data3 < other.Data3; + return memcmp(Data4, other.Data4, sizeof(Data4)) < 0; } void EventProperty::copydata(EventProperty const* source) diff --git a/lib/system/TelemetrySystemBase.hpp b/lib/system/TelemetrySystemBase.hpp index 30fd1d5af..fba193b1b 100644 --- a/lib/system/TelemetrySystemBase.hpp +++ b/lib/system/TelemetrySystemBase.hpp @@ -36,11 +36,11 @@ namespace MAT_NS_BEGIN { m_isPaused(false), stats(*this, taskDispatcher) { - onStart = []() { return true; }; - onStop = []() { return true; }; - onPause = []() { return true; }; - onResume = []() { return true; }; - onCleanup = []() { return true; }; + onStart = []() noexcept { return true; }; + onStop = []() noexcept { return true; }; + onPause = []() noexcept { return true; }; + onResume = []() noexcept { return true; }; + onCleanup = []() noexcept { return true; }; } /// diff --git a/lib/utils/StringUtils.cpp b/lib/utils/StringUtils.cpp index 7ee3318f3..d47d3c0b2 100644 --- a/lib/utils/StringUtils.cpp +++ b/lib/utils/StringUtils.cpp @@ -31,7 +31,7 @@ namespace MAT_NS_BEGIN } } - bool StringUtils::AreAllCharactersAllowlisted(const string& stringToTest, const string& allowlist) + bool StringUtils::AreAllCharactersAllowlisted(const string& stringToTest, const string& allowlist) noexcept { return (stringToTest.find_first_not_of(allowlist) == string::npos); } @@ -132,7 +132,7 @@ namespace MAT_NS_BEGIN { std::string result = str; std::transform(str.begin(), str.end(), result.begin(), - [](unsigned char c) { return (char)::tolower(c); }); + [](unsigned char c) noexcept { return (char)::tolower(c); }); return result; } @@ -140,7 +140,7 @@ namespace MAT_NS_BEGIN { std::string result = str; std::transform(str.begin(), str.end(), result.begin(), - [](unsigned char c) { return (char)::toupper(c); }); + [](unsigned char c) noexcept { return (char)::toupper(c); }); return result; } @@ -158,7 +158,7 @@ namespace MAT_NS_BEGIN return str; } - const char* priorityToStr(EventPriority priority) + const char* priorityToStr(EventPriority priority) noexcept { switch (priority) { @@ -185,7 +185,7 @@ namespace MAT_NS_BEGIN } } - const char* latencyToStr(EventLatency latency) + const char* latencyToStr(EventLatency latency) noexcept { switch (latency) { diff --git a/lib/utils/StringUtils.hpp b/lib/utils/StringUtils.hpp index 464f551f1..dc7c42bc8 100644 --- a/lib/utils/StringUtils.hpp +++ b/lib/utils/StringUtils.hpp @@ -17,7 +17,7 @@ namespace MAT_NS_BEGIN namespace StringUtils { void SplitString(const std::string& s, const char separator, std::vector& parts); - bool AreAllCharactersAllowlisted(const std::string& stringToTest, const std::string& allowlist); + bool AreAllCharactersAllowlisted(const std::string& stringToTest, const std::string& allowlist) noexcept; } std::string toString(char const* value); @@ -44,9 +44,9 @@ namespace MAT_NS_BEGIN std::string sanitizeIdentifier(const std::string& str); - const char* priorityToStr(EventPriority priority); + const char* priorityToStr(EventPriority priority) noexcept; - const char* latencyToStr(EventLatency latency); + const char* latencyToStr(EventLatency latency) noexcept; bool replace(std::string& str, const std::string& from, const std::string& to); diff --git a/lib/utils/Utils.cpp b/lib/utils/Utils.cpp index e2360ca18..22a48d87f 100644 --- a/lib/utils/Utils.cpp +++ b/lib/utils/Utils.cpp @@ -74,7 +74,7 @@ namespace MAT_NS_BEGIN { #endif } - bool IsRunningInApp() + bool IsRunningInApp() noexcept { #ifdef _WINRT_DLL // Win 10 UWP typedef LONG (*LPFN_GPFN)(UINT32*, PWSTR); diff --git a/lib/utils/Utils.hpp b/lib/utils/Utils.hpp index 78c47201a..ce249b9ab 100644 --- a/lib/utils/Utils.hpp +++ b/lib/utils/Utils.hpp @@ -67,7 +67,7 @@ namespace MAT_NS_BEGIN { long GetCurrentProcessId(); /* Detects if current process is running in a packaged app*/ - bool IsRunningInApp(); + bool IsRunningInApp() noexcept; std::string GetTempDirectory(); std::string GetAppLocalTempDirectory(); diff --git a/lib/utils/ZlibUtils.cpp b/lib/utils/ZlibUtils.cpp index d091ab3fa..993aad2a8 100644 --- a/lib/utils/ZlibUtils.cpp +++ b/lib/utils/ZlibUtils.cpp @@ -48,7 +48,7 @@ namespace MAT_NS_BEGIN } while (ret == Z_OK); if (ret != Z_STREAM_END) { - LOG_WARN("Inflate failed, error=%u/%u (%s)", 2, ret, zs.msg); + LOG_WARN("Inflate failed, error=%d/%d (%s)", 2, ret, (zs.msg ? zs.msg : "(null)")); result = false; } inflateEnd(&zs); diff --git a/lib/utils/annex_k.hpp b/lib/utils/annex_k.hpp index 4e9fef9ed..5aa4b73af 100644 --- a/lib/utils/annex_k.hpp +++ b/lib/utils/annex_k.hpp @@ -45,7 +45,7 @@ namespace MAT_NS_BEGIN class BoundCheckFunctions { private: -static bool oneds_buffer_region_overlap(const char *buffer1, size_t buffer1_len, const char *buffer2, size_t buffer2_len) +static bool oneds_buffer_region_overlap(const char *buffer1, size_t buffer1_len, const char *buffer2, size_t buffer2_len) noexcept { if (buffer2 >= buffer1) { @@ -70,7 +70,7 @@ static bool oneds_buffer_region_overlap(const char *buffer1, size_t buffer1_len, // - returns zero if str is a null pointer // - returns strsz if the null character was not found in the first strsz bytes of str. -static size_t oneds_strnlen_s(const char *str, size_t strsz) +static size_t oneds_strnlen_s(const char *str, size_t strsz) noexcept { if ( str == NULL) { @@ -89,7 +89,7 @@ static size_t oneds_strnlen_s(const char *str, size_t strsz) // - count is greater than RSIZE_MAX // - count is greater or equal destsz, but destsz is less or equal strnlen_s(src, count), in other words, truncation would occur // - overlap would occur between the source and the destination strings -static errno_t oneds_strncpy_s(char * restrict dest, rsize_t destsz, const char *restrict src, rsize_t count) +static errno_t oneds_strncpy_s(char * restrict dest, rsize_t destsz, const char *restrict src, rsize_t count) noexcept { #if (defined __STDC_LIB_EXT1__) || ( defined _MSC_VER) return strncpy_s(dest, destsz, src, count); @@ -148,7 +148,7 @@ static errno_t oneds_strncpy_s(char * restrict dest, rsize_t destsz, const char // (if both dest and destsz are valid)) static errno_t oneds_memcpy_s( void *restrict dest, rsize_t destsz, - const void *restrict src, rsize_t count ) + const void *restrict src, rsize_t count ) noexcept { #if (defined __STDC_LIB_EXT1__) || ( defined _MSC_VER) return memcpy_s(dest, destsz, src, count); diff --git a/tests/common/Common.cpp b/tests/common/Common.cpp index f6023f656..0200402b9 100644 --- a/tests/common/Common.cpp +++ b/tests/common/Common.cpp @@ -268,7 +268,7 @@ namespace testing { return fname; } - void LogMemUsage(const char* label) + void LogMemUsage(const char* label) noexcept { #ifdef DEBUG_PERF #ifdef _WIN32 @@ -295,7 +295,7 @@ namespace testing { #endif } - void LogCpuUsage(const char* label) + void LogCpuUsage(const char* label) noexcept { #ifdef DEBUG_PERF static int64_t lastTime = GetUptimeMs(); diff --git a/tests/common/Common.hpp b/tests/common/Common.hpp index 754040ff8..da7f2903e 100644 --- a/tests/common/Common.hpp +++ b/tests/common/Common.hpp @@ -75,9 +75,9 @@ namespace testing { LogMemUsage(label); \ LogCpuUsage(label); - void LogMemUsage(const char* label); + void LogMemUsage(const char* label) noexcept; - void LogCpuUsage(const char* label); + void LogCpuUsage(const char* label) noexcept; void InflateVector(std::vector &in, std::vector &out, bool isGzip = false); } // namespace testing diff --git a/tests/common/MockIRuntimeConfig.hpp b/tests/common/MockIRuntimeConfig.hpp index 4a7509c84..a52ef8e8d 100644 --- a/tests/common/MockIRuntimeConfig.hpp +++ b/tests/common/MockIRuntimeConfig.hpp @@ -19,7 +19,7 @@ namespace testing { class MockIRuntimeConfig : public MAT::RuntimeConfig_Default /* MAT::IRuntimeConfig */ { protected: - std::unique_ptr& GetStaticConfig() + std::unique_ptr& GetStaticConfig() noexcept { static std::unique_ptr staticConfig; return staticConfig; diff --git a/tests/functests/MultipleLogManagersTests.cpp b/tests/functests/MultipleLogManagersTests.cpp index eac2bfd00..7a9027b9b 100644 --- a/tests/functests/MultipleLogManagersTests.cpp +++ b/tests/functests/MultipleLogManagersTests.cpp @@ -22,6 +22,10 @@ #include "NullObjects.hpp" +#if defined(__APPLE__) +#include +#endif + #if defined __has_include && defined(HAVE_MAT_PRIVACYGUARD) #if __has_include("modules/privacyguard/PrivacyGuard.hpp") #include "modules/privacyguard/PrivacyGuard.hpp" @@ -36,6 +40,15 @@ using namespace testing; using namespace MAT; +// MultipleLogManagersTests stand up an in-process HttpServer on a loopback port +// and run multiple concurrent LogManager instances against it. That pattern +// hangs/fails inside the iOS simulator sandbox (the loopback uploads stall), +// which previously left the iOS CI job to sit until its 60-minute timeout. The +// behavior is still exercised on the desktop and macOS targets; exclude the +// whole suite from the iOS build. (GTEST_SKIP in SetUp is not honored by the +// iOS xctest gtest wrapper, so the exclusion must be at compile time.) +#if !defined(TARGET_OS_IPHONE) || !TARGET_OS_IPHONE + class RequestHandler : public HttpServer::Callback { public: @@ -292,5 +305,7 @@ TEST_F(MultipleLogManagersTests, PrivacyGuardSharedWithTwoInstancesCoexist) } #endif //END HAVE_MAT_PRIVACYGUARD +#endif // !TARGET_OS_IPHONE (suite excluded on iOS; see note above) + #endif // HAVE_MAT_DEFAULT_HTTP_CLIENT diff --git a/tests/headers/check_public_headers.cmd b/tests/headers/check_public_headers.cmd new file mode 100644 index 000000000..53860422c --- /dev/null +++ b/tests/headers/check_public_headers.cmd @@ -0,0 +1,129 @@ +@echo off +REM Copyright (c) Microsoft Corporation. All rights reserved. +REM SPDX-License-Identifier: Apache-2.0 +REM +REM Public header gate (MSVC). Compiles each public SDK header on its own under +REM /W4 /WX, mirroring how ONNX Runtime / Foundry Local compile their own C++ +REM translation units on Windows. STL/Windows SDK headers are treated as external +REM (/external:W0) so only the SDK's headers are gated. Exits non-zero if any +REM header fails to compile or emits a warning. Also compiles mat.h as C11 (/TC). +setlocal enabledelayedexpansion + +set "SCRIPT_DIR=%~dp0" +set "REPO_ROOT=%SCRIPT_DIR%..\.." +set "PUB=%REPO_ROOT%\lib\include\public" +set "C_API_HEADER=mat.h" + +REM Fail fast if the public header directory is missing or miscomputed, otherwise +REM the header loop below would run zero times and the gate would silently "pass" +REM without compiling anything -- a false negative. +if not exist "%PUB%" ( + echo error: public header directory not found: %PUB% 1>&2 + exit /b 2 +) + +REM Enter the MSVC x64 developer environment via vswhere (portable across runners). +set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" +if not exist "%VSWHERE%" ( + echo error: vswhere.exe not found 1>&2 + exit /b 2 +) +set "VSPATH=" +for /f "usebackq tokens=*" %%i in (`"%VSWHERE%" -latest -products * -property installationPath`) do set "VSPATH=%%i" +if not defined VSPATH ( + echo error: no Visual Studio installation found 1>&2 + exit /b 2 +) +call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" >nul +if errorlevel 1 ( + echo error: failed to initialize the MSVC environment 1>&2 + exit /b 2 +) + +REM Unique work directory under the repository so concurrent invocations on the +REM same machine do not clobber each other's temporary translation units. +set "WORK=%REPO_ROOT%\.public-header-gate_%RANDOM%_%RANDOM%" +if exist "%WORK%" rmdir /s /q "%WORK%" +mkdir "%WORK%" +if errorlevel 1 ( + echo error: failed to create work directory %WORK% 1>&2 + exit /b 2 +) + +REM /W4 /WX matches ORT; /external:W0 suppresses platform/STL warnings so only our headers gate. +set "CXX_COMMON=/nologo /permissive- /W4 /WX /EHsc /experimental:external /external:anglebrackets /external:W0" +set "C_COMMON=/nologo /std:c11 /TC /W4 /WX /experimental:external /external:anglebrackets /external:W0" +set "FAIL=0" +set "TOTAL=0" + +REM MSVC does not expose a /std:c++11 switch; /std:c++14 is its lowest selectable mode. +call :RunCxxHeaders c++14 /std:c++14 "cl (c++14, /W4 /WX)" +call :RunCxxHeaders c++17 /std:c++17 "cl (c++17, /W4 /WX)" +call :RunCHeader + +rmdir /s /q "%WORK%" 2>nul + +if "%FAIL%"=="1" ( + echo Public header gate FAILED. + exit /b 1 +) +echo Public header gate passed. ^(!TOTAL! checks^) +exit /b 0 + +:RunCxxHeaders +set "STD_NAME=%~1" +set "STD_FLAG=%~2" +set "LABEL=%~3" +set "OKC=0" +set "FAILC=0" +echo == %LABEL% == +for %%h in ("%PUB%\*.hpp" "%PUB%\*.h") do ( + set "NAME=%%~nxh" + REM Skip implementation-fragment headers not meant to be included standalone + REM (VariantType.hpp is included by Variant.hpp, which defines VariantMap/VariantArray first). + if /I not "!NAME!"=="VariantType.hpp" ( + > "%WORK%\tu_!STD_NAME!.cpp" echo #include "!NAME!" + >> "%WORK%\tu_!STD_NAME!.cpp" echo int main^(^){return 0;} + cl %CXX_COMMON% %STD_FLAG% /I "%PUB%" /I "%REPO_ROOT%\lib\include" /Zs "%WORK%\tu_!STD_NAME!.cpp" > "%WORK%\err.txt" 2>&1 + if errorlevel 1 ( + echo FAIL: !NAME! + type "%WORK%\err.txt" + set "FAIL=1" + set /a FAILC+=1 + ) else ( + set /a OKC+=1 + set /a TOTAL+=1 + ) + ) +) +REM No headers compiled means PUB matched nothing -- treat it as a failure rather +REM than a silent pass. +if "!OKC!"=="0" if "!FAILC!"=="0" ( + echo error: no public headers found under %PUB% 1>&2 + set "FAIL=1" +) +echo %LABEL%: !OKC! passed, !FAILC! failed +exit /b 0 + +:RunCHeader +set "LABEL=cl (C11 mat.h, /TC, /W4 /WX)" +echo == !LABEL! == +if not exist "%PUB%\%C_API_HEADER%" ( + echo error: C API header not found: %PUB%\%C_API_HEADER% 1>&2 + set "FAIL=1" + echo !LABEL!: 0 passed, 1 failed + exit /b 0 +) +> "%WORK%\tu_mat_c11.c" echo #include "%C_API_HEADER%" +>> "%WORK%\tu_mat_c11.c" echo int main^(void^){return 0;} +cl %C_COMMON% /I "%PUB%" /I "%REPO_ROOT%\lib\include" /Zs "%WORK%\tu_mat_c11.c" > "%WORK%\err.txt" 2>&1 +if errorlevel 1 ( + echo FAIL: %C_API_HEADER% + type "%WORK%\err.txt" + set "FAIL=1" + echo !LABEL!: 0 passed, 1 failed +) else ( + set /a TOTAL+=1 + echo !LABEL!: 1 passed, 0 failed +) +exit /b 0 diff --git a/tests/headers/check_public_headers.sh b/tests/headers/check_public_headers.sh new file mode 100644 index 000000000..79724e588 --- /dev/null +++ b/tests/headers/check_public_headers.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Public header gate (GCC/Clang). +# +# Verifies that every public SDK header is self-contained (compiles on its own, +# in any include order) and warning-clean under strict, consumer-representative +# warning flags. Downstream consumers such as ONNX Runtime / Foundry Local +# compile their own translation units -- which include these headers -- with +# -Wall -Wextra -Werror (plus -Wshorten-64-to-32 on Clang). This gate compiles +# each public header on its own as both C++11 and C++17, with no -isystem +# suppression, so any header issue surfaces here instead of at integration time. +# It also compiles the ABI-stable C API header (mat.h) as C11. +# +# Exits non-zero if any header fails to compile or emits a warning. + +set -uo pipefail +shopt -s nullglob + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PUB="$REPO_ROOT/lib/include/public" + +# Fail fast if the public header directory is missing or miscomputed. With +# nullglob on, a bad PUB would make the header globs below expand to nothing and +# the gate would silently "pass" without compiling anything -- a false negative. +if [ ! -d "$PUB" ]; then + echo "error: public header directory not found: $PUB" >&2 + exit 2 +fi + +# Implementation-fragment headers: intentionally included by another public +# header (which supplies their dependencies first) and not meant to be included +# standalone. They are exercised through their public entry point instead. +EXCLUDES=( + "VariantType.hpp" # included by Variant.hpp, which defines VariantMap/VariantArray first +) + +# Mirror the strict warning flags third-party consumers build with +# (-Wall -Wextra -Werror, matching this repo's pipeline and ORT). Unused +# parameters are deliberately NOT suppressed: intentional ones use the +# UNREFERENCED_PARAMETER macro, which expands to (void)(...) on GCC/Clang, so the +# headers stay warning-clean without a blanket -Wno-unused-parameter that would +# hide a real consumer break (e.g. an override that leaves a parameter unused). +C_API_HEADER="mat.h" + +is_excluded() { + local n="$1" e + for e in "${EXCLUDES[@]}"; do [ "$e" = "$n" ] && return 0; done + return 1 +} + +fail=0 +tmp="" +for _ in 1 2 3 4 5; do + candidate="$REPO_ROOT/.public-header-gate.$$.$RANDOM" + if mkdir "$candidate" 2>/dev/null; then + tmp="$candidate" + break + fi +done +if [ -z "$tmp" ]; then + echo "error: failed to create work directory under $REPO_ROOT" >&2 + exit 2 +fi +trap 'rm -rf "$tmp"' EXIT + +run_cxx_compiler() { + local cc="$1" std="$2" label="$3" + shift 3 + local n_ok=0 n_fail=0 h name base tu out + echo "== $label ==" + for h in "$PUB"/*.hpp "$PUB"/*.h; do + name="$(basename "$h")" + is_excluded "$name" && continue + base="${name%.*}" + tu="$tmp/tu_${base}_${std}.cpp" + printf '#include "%s"\nint main() { return 0; }\n' "$name" > "$tu" + if out="$("$cc" "-std=$std" -Wall -Wextra -Werror "$@" -I"$PUB" -I"$REPO_ROOT/lib/include" -fsyntax-only "$tu" 2>&1)"; then + n_ok=$((n_ok + 1)) + else + n_fail=$((n_fail + 1)); fail=1 + echo " FAIL: $name" + echo "$out" | grep -E 'error:|warning:' | head -4 | sed 's/^/ /' + fi + done + if [ $((n_ok + n_fail)) -eq 0 ]; then + echo " ERROR: no public headers found under $PUB" + fail=1 + fi + echo " $label: $n_ok passed, $n_fail failed" +} + +run_c_compiler() { + local cc="$1" label="$2" + shift 2 + local tu out + echo "== $label ==" + if [ ! -f "$PUB/$C_API_HEADER" ]; then + echo " ERROR: C API header not found: $PUB/$C_API_HEADER" + fail=1 + echo " $label: 0 passed, 1 failed" + return + fi + tu="$tmp/tu_${C_API_HEADER%.h}_c11.c" + printf '#include "%s"\nint main(void) { return 0; }\n' "$C_API_HEADER" > "$tu" + if out="$("$cc" -std=c11 -Wall -Wextra -Werror "$@" -I"$PUB" -I"$REPO_ROOT/lib/include" -fsyntax-only "$tu" 2>&1)"; then + echo " $label: 1 passed, 0 failed" + else + fail=1 + echo " FAIL: $C_API_HEADER" + echo "$out" | grep -E 'error:|warning:' | head -4 | sed 's/^/ /' + echo " $label: 0 passed, 1 failed" + fi +} + +cxx_ran=0 +if command -v g++ >/dev/null 2>&1; then + run_cxx_compiler g++ c++11 "g++ (c++11, -Wall -Wextra -Werror)" + run_cxx_compiler g++ c++17 "g++ (c++17, -Wall -Wextra -Werror)" + cxx_ran=1 +fi +if command -v clang++ >/dev/null 2>&1; then + run_cxx_compiler clang++ c++11 "clang++ (c++11, + -Wshorten-64-to-32)" -Wshorten-64-to-32 + run_cxx_compiler clang++ c++17 "clang++ (c++17, + -Wshorten-64-to-32)" -Wshorten-64-to-32 + cxx_ran=1 +fi + +if [ "$cxx_ran" -eq 0 ]; then + echo "error: neither g++ nor clang++ was found" >&2 + exit 2 +fi + +c_ran=0 +if command -v gcc >/dev/null 2>&1; then + run_c_compiler gcc "gcc (c11 mat.h, -Wall -Wextra -Werror)" + c_ran=1 +fi +if command -v clang >/dev/null 2>&1; then + run_c_compiler clang "clang (c11 mat.h, + -Wshorten-64-to-32)" -Wshorten-64-to-32 + c_ran=1 +fi +if [ "$c_ran" -eq 0 ] && command -v cc >/dev/null 2>&1; then + run_c_compiler cc "cc (c11 mat.h, -Wall -Wextra -Werror)" + c_ran=1 +fi + +if [ "$c_ran" -eq 0 ]; then + echo "error: no C compiler (gcc, clang, or cc) was found" >&2 + exit 2 +fi + +if [ "$fail" -ne 0 ]; then + echo "Public header gate FAILED." + exit 1 +fi +echo "Public header gate passed." diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index 945fc23df..7233d2920 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -40,6 +40,7 @@ set(SRCS OfflineStorageTests_Room.cpp OfflineStorageTests_SQLite.cpp PackagerTests.cpp + PayloadDecoderTests.cpp PalTests.cpp RouteTests.cpp StringUtilsTests.cpp diff --git a/tests/unittests/EventPropertiesDecoratorTests.cpp b/tests/unittests/EventPropertiesDecoratorTests.cpp index f38444fd0..348354604 100644 --- a/tests/unittests/EventPropertiesDecoratorTests.cpp +++ b/tests/unittests/EventPropertiesDecoratorTests.cpp @@ -28,6 +28,19 @@ class TestEventPropertiesDecorator : public EventPropertiesDecorator } }; +// NullLogManager hands out a single shared static ILogConfiguration, which would +// leak configuration across tests. This subclass owns a per-instance configuration +// so the IP-scrubbing opt-out can be exercised in isolation. +class ConfigurableLogManager : public NullLogManager +{ +public: + ILogConfiguration localConfig; + ILogConfiguration& GetLogConfiguration() override + { + return localConfig; + } +}; + static std::unique_ptr PopulateRecordForDropPii() { auto record = std::unique_ptr(new Record{}); @@ -545,3 +558,41 @@ TEST(EventPropertiesDecoratorTests, DropPiiPartA_StripsValues) EXPECT_THAT(record->extSdk[0].installId, Eq("")); EXPECT_THAT(record->cV, Eq("")); } + +TEST(EventPropertiesDecoratorTests, Decorate_ScrubIp_EnabledByDefault) +{ + ConfigurableLogManager logManager; // CFG_BOOL_ENABLE_IP_SCRUBBING not set + EventPropertiesDecorator decorator(logManager); + Record record; + EventProperties props {"TestEvent"}; + EventLatency latency = EventLatency::EventLatency_Normal; + + EXPECT_TRUE(decorator.decorate(record, latency, props)); + EXPECT_TRUE(record.flags & RECORD_FLAGS_EVENTTAG_SCRUB_IP); +} + +TEST(EventPropertiesDecoratorTests, Decorate_ScrubIp_OptOutViaConfig) +{ + ConfigurableLogManager logManager; + logManager.localConfig[CFG_BOOL_ENABLE_IP_SCRUBBING] = false; + EventPropertiesDecorator decorator(logManager); + Record record; + EventProperties props {"TestEvent"}; + EventLatency latency = EventLatency::EventLatency_Normal; + + EXPECT_TRUE(decorator.decorate(record, latency, props)); + EXPECT_FALSE(record.flags & RECORD_FLAGS_EVENTTAG_SCRUB_IP); +} + +TEST(EventPropertiesDecoratorTests, Decorate_ScrubIp_ExplicitlyEnabled) +{ + ConfigurableLogManager logManager; + logManager.localConfig[CFG_BOOL_ENABLE_IP_SCRUBBING] = true; + EventPropertiesDecorator decorator(logManager); + Record record; + EventProperties props {"TestEvent"}; + EventLatency latency = EventLatency::EventLatency_Normal; + + EXPECT_TRUE(decorator.decorate(record, latency, props)); + EXPECT_TRUE(record.flags & RECORD_FLAGS_EVENTTAG_SCRUB_IP); +} diff --git a/tests/unittests/GuidTests.cpp b/tests/unittests/GuidTests.cpp index a0fc9fdb5..4a45d880c 100644 --- a/tests/unittests/GuidTests.cpp +++ b/tests/unittests/GuidTests.cpp @@ -7,6 +7,8 @@ #include "utils/Utils.hpp" #include "EventProperties.hpp" +#include + using namespace testing; using namespace MAT; @@ -84,4 +86,23 @@ TEST(GuidTests, MoveAssignment_ValidInput_MovesCorrectly) GUID_t second{"BEE391C8-72B0-464F-93C3-1B27879AD103"}; second = std::move(first); ASSERT_EQ("9D016D64-372E-4DCE-9FA3-0D0772217C54", second.to_string()); -} \ No newline at end of file +} +TEST(GuidTests, OperatorLess_IsStrictWeakOrdering) +{ + // a and b differ in Data1/Data2 such that a non-lexicographic chained-|| operator + // reported BOTH a < b and b < a (antisymmetry violation). + GUID_t a{ "00000001-0005-0000-0000-000000000000" }; + GUID_t b{ "00000002-0003-0000-0000-000000000000" }; + EXPECT_TRUE(a < b); + EXPECT_FALSE(b < a); + + // c and d differ ONLY in Data3; a '==' in that position made them compare equivalent. + GUID_t c{ "00000001-0001-0001-0000-000000000000" }; + GUID_t d{ "00000001-0001-0002-0000-000000000000" }; + EXPECT_TRUE(c < d); + EXPECT_FALSE(d < c); + + // A std::set keyed on operator< must keep four distinct GUIDs distinct. + std::set s{ a, b, c, d }; + EXPECT_EQ(static_cast(4), s.size()); +} diff --git a/tests/unittests/HttpClientCurlTests.cpp b/tests/unittests/HttpClientCurlTests.cpp index c9894b90d..d494ba2fc 100644 --- a/tests/unittests/HttpClientCurlTests.cpp +++ b/tests/unittests/HttpClientCurlTests.cpp @@ -9,6 +9,7 @@ && !defined(__APPLE__) && !defined(ANDROID) #include "common/Common.hpp" +#include "common/HttpServer.hpp" #include "http/HttpClient_Curl.hpp" #include "config/RuntimeConfig_Default.hpp" @@ -126,4 +127,118 @@ TEST_F(HttpClientCurlTests, SetSslVerification_ConcurrentCallsNoRace) SUCCEED(); } +// --- Response-size cap (memory-amplification DoS hardening) --- + +class HttpClientCurlResponseCapTests : public ::testing::Test, + public HttpServer::Callback, + public IHttpResponseCallback +{ +protected: + HttpServer m_server; + HttpClient_Curl m_client; + // The client never takes ownership of the request (it only stores a raw pointer + // and erases it); the fixture owns it and frees it in TearDown -- on the main + // thread, after the transfer has completed. Freeing it inside OnHttpResponse + // would destroy the CurlHttpOperation from within its own async task, whose + // destructor waits on that task (a self-join deadlock). + std::unique_ptr m_request; + std::string m_hostname; + size_t m_responseBodySize {0}; + + std::mutex m_lock; + bool m_received {false}; + HttpResult m_result {}; + unsigned int m_statusCode {0}; + size_t m_bodySize {0}; + + void SetUp() override + { + int port = m_server.addListeningPort(0); + std::ostringstream os; + os << "127.0.0.1:" << port; + m_hostname = os.str(); + m_server.setServerName(m_hostname); + m_server.addHandler("/huge/", *this); + m_server.start(); + } + + void TearDown() override + { + m_server.stop(); + m_request.reset(); + } + + // HttpServer::Callback -- returns a body of m_responseBodySize bytes. + int onHttpRequest(HttpServer::Request const& /*request*/, HttpServer::Response& response) override + { + size_t bodySize; + { + std::lock_guard lock(m_lock); + bodySize = m_responseBodySize; + } + response.headers["Content-Type"] = "application/octet-stream"; + response.content = std::string(bodySize, 'A'); + return 200; + } + + // IHttpResponseCallback -- the SDK hands over ownership of the response. + void OnHttpResponse(IHttpResponse* response) override + { + std::unique_ptr owned(response); + std::lock_guard lock(m_lock); + m_result = owned->GetResult(); + m_statusCode = owned->GetStatusCode(); + m_bodySize = owned->GetBody().size(); + m_received = true; + } + + bool responseReceived() + { + std::lock_guard lock(m_lock); + return m_received; + } + + void sendAndWait(size_t bodySize) + { + { + std::lock_guard lock(m_lock); + m_received = false; + m_result = HttpResult{}; + m_statusCode = 0; + m_bodySize = 0; + m_responseBodySize = bodySize; // read under the same lock by onHttpRequest + } + m_request.reset(m_client.CreateRequest()); + m_request->SetUrl("http://" + m_hostname + "/huge/"); + m_client.SendRequestAsync(m_request.get(), this); + for (int i = 0; i < 300 && !responseReceived(); i++) + PAL::sleep(100); + } +}; + +TEST_F(HttpClientCurlResponseCapTests, AbortsOversizedResponseBody) +{ + // A response body larger than the client's response-size cap (kMaxResponseBytes, + // 16 MB) must be refused, not buffered in full, so a hostile/MITM'd collector + // cannot exhaust process memory. + sendAndWait(17u * 1024u * 1024u); + ASSERT_TRUE(responseReceived()); + // curl aborts the transfer (CURLE_WRITE_ERROR) once the cap is hit -> NetworkFailure. + EXPECT_EQ(m_result, HttpResult_NetworkFailure); + // The oversized body is never fully buffered. + EXPECT_LE(m_bodySize, static_cast(16u * 1024u * 1024u)); +} + +TEST_F(HttpClientCurlResponseCapTests, AcceptsLargeResponseUnderCap) +{ + // A large-but-legitimate response (well under the cap) must still be received + // in full: the cap must not regress normal responses. + const size_t bodySize = 4u * 1024u * 1024u; + sendAndWait(bodySize); + ASSERT_TRUE(responseReceived()); + EXPECT_EQ(m_result, HttpResult_OK); + EXPECT_EQ(m_statusCode, 200u); + EXPECT_EQ(m_bodySize, bodySize); +} + #endif // MATSDK_PAL_CPP11 && !_MSC_VER && HAVE_MAT_DEFAULT_HTTP_CLIENT diff --git a/tests/unittests/OfflineStorageTests_SQLite.cpp b/tests/unittests/OfflineStorageTests_SQLite.cpp index e90b0a9ae..d5aa6808a 100644 --- a/tests/unittests/OfflineStorageTests_SQLite.cpp +++ b/tests/unittests/OfflineStorageTests_SQLite.cpp @@ -12,6 +12,9 @@ #include "offline/OfflineStorage_SQLite.hpp" #include #include +#if !defined(_WIN32) +#include +#endif #include "NullObjects.hpp" @@ -89,6 +92,12 @@ struct OfflineStorageTests_SQLite : public Test EXPECT_THAT(fileExists(storageFilename), true); ::remove(storageFilename.c_str()); EXPECT_THAT(fileExists(storageFilename), false); + // WAL mode can leave -wal/-shm/-journal companions behind; remove them too + // so they don't leak into other tests that reuse the same storage filename. + for (const char* suffix : { "-wal", "-shm", "-journal" }) + { + ::remove((storageFilename + suffix).c_str()); + } } } @@ -839,4 +848,67 @@ TEST_F(OfflineStorageTests_SQLite, SqliteDbInstancesAreCounted) shutdownAndRemoveFile(); EXPECT_EQ(offlineStorage->GetDbInstanceCount(), 0); } + +#if !defined(_WIN32) +// SECURITY: the offline cache buffers pending telemetry/audit events, so it must +// not be world-readable. SQLite creates the file 0644 by default; SQLiteWrapper +// tightens it to 0600 after open, and the -wal/-journal companions inherit that +// mode from the main database file. POSIX-only (mode bits are meaningless on +// Windows, where access is governed by NTFS ACLs). +TEST_F(OfflineStorageTests_SQLite, CacheFileCreatedOwnerReadWriteOnly) +{ + initializeStorage(); + + struct stat st; + ASSERT_EQ(0, ::stat(storageFilename.c_str(), &st)) << "cache database file was not created"; + EXPECT_EQ(static_cast(S_IRUSR | S_IWUSR), static_cast(st.st_mode & 0777)) + << "offline cache database must be created 0600, not world-readable"; + + // Any WAL/journal/shm companion that exists must not grant group or other access + // (SQLite derives their permissions from the main database file's mode). + for (const char* suffix : { "-wal", "-journal", "-shm" }) + { + struct stat cst; + const std::string companion = storageFilename + suffix; + if (::stat(companion.c_str(), &cst) == 0) + { + EXPECT_EQ(0, static_cast(cst.st_mode & (S_IRWXG | S_IRWXO))) + << "companion file " << suffix << " must not be group/world accessible"; + } + } +} + +// A cache written by an older SDK (or left behind after a crash) can have the +// database and companion files already on disk with the old world-readable 0644 +// mode; SQLite only derives 0600 for companions it creates itself. Opening the +// storage must re-tighten both the database and any pre-existing companion. +TEST_F(OfflineStorageTests_SQLite, ExistingFilesAreTightenedOnOpen) +{ + initializeStorage(); + offlineStorage->Shutdown(); + storageInitialized = false; + + // Simulate an old cache: loosen the database and plant a leftover -wal at 0644. + const mode_t loose = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; // 0644 + ASSERT_EQ(0, ::chmod(storageFilename.c_str(), loose)); + const std::string wal = storageFilename + "-wal"; + std::ofstream(wal, std::ios::binary); // empty leftover companion + ASSERT_EQ(0, ::chmod(wal.c_str(), loose)); + + // Reopen -- the open path must re-tighten both files. + initializeStorage(); + + struct stat st; + ASSERT_EQ(0, ::stat(storageFilename.c_str(), &st)); + EXPECT_EQ(0, static_cast(st.st_mode & (S_IRWXG | S_IRWXO))) + << "reopened database must be tightened to 0600"; + + struct stat wst; + if (::stat(wal.c_str(), &wst) == 0) + { + EXPECT_EQ(0, static_cast(wst.st_mode & (S_IRWXG | S_IRWXO))) + << "pre-existing -wal companion must be tightened to 0600"; + } +} +#endif #endif diff --git a/tests/unittests/PalTests.cpp b/tests/unittests/PalTests.cpp index 0bf8a06a3..c931ff376 100644 --- a/tests/unittests/PalTests.cpp +++ b/tests/unittests/PalTests.cpp @@ -5,10 +5,15 @@ #include "common/Common.hpp" #include "pal/PseudoRandomGenerator.hpp" +#include "pal/TaskDispatcher.hpp" +#include "pal/WorkerThread.hpp" #include "Version.hpp" +#include #include +#include #include +#include #ifdef HAVE_MAT_LOGGING #include "pal/PAL.hpp" @@ -42,9 +47,14 @@ class PalTests : public Test {}; TEST_F(PalTests, UuidGeneration) { + // Canonical UUID string length ("8-4-4-4-12") and the number of UUIDs + // generated for the uniqueness check below. + constexpr size_t UuidStringLength = 36; + constexpr size_t UuidBatchSize = 1000; + std::string uuid0 = PAL::generateUuidString(); - EXPECT_THAT(uuid0.length(), 36u); + EXPECT_THAT(uuid0.length(), UuidStringLength); std::string mask = uuid0; for (char& ch : mask) { @@ -61,21 +71,31 @@ TEST_F(PalTests, UuidGeneration) std::string uuid1 = PAL::generateUuidString(); - EXPECT_THAT(uuid1.length(), 36u); + EXPECT_THAT(uuid1.length(), UuidStringLength); size_t diff = 0; - for (size_t i = 0; i < 36; i++) { + for (size_t i = 0; i < UuidStringLength; i++) { diff += (uuid0[i] != uuid1[i]); } EXPECT_THAT(diff, Gt(20u)); + + // A batch of generated UUIDs must all be distinct (guards against a stuck + // or low-entropy generator). + std::set uuids; + for (size_t i = 0; i < UuidBatchSize; i++) { + std::string u = PAL::generateUuidString(); + EXPECT_THAT(u.length(), UuidStringLength); + uuids.insert(u); + } + EXPECT_THAT(uuids.size(), UuidBatchSize); } TEST_F(PalTests, PseudoRandomGenerator) { PAL::PseudoRandomGenerator prg; - size_t const NumQueries = 1000; - size_t const NumBuckets = 11; + constexpr size_t NumQueries = 1000; + constexpr size_t NumBuckets = 11; size_t buckets[NumBuckets] = {}; for (size_t i = 0; i < NumQueries; i++) { @@ -102,6 +122,22 @@ TEST_F(PalTests, SystemTime) EXPECT_THAT(t1, Lt(t0 + 1000)); } +#if !defined(_WIN32) && !defined(_WIN64) +TEST_F(PalTests, SystemTimeInTicksPreservesSubMillisecondPrecision) +{ + constexpr int64_t TicksPerMillisecond = 10000; + bool observedSubMillisecondTick = false; + + for (int i = 0; i < 1000 && !observedSubMillisecondTick; ++i) + { + observedSubMillisecondTick = + PAL::getUtcSystemTimeinTicks() % TicksPerMillisecond != 0; + } + + EXPECT_TRUE(observedSubMillisecondTick); +} +#endif + TEST_F(PalTests, FormatUtcTimestampMsAsISO8601) { EXPECT_THAT(PAL::formatUtcTimestampMsAsISO8601(0ll), Eq("1970-01-01T00:00:00.000Z")); @@ -180,6 +216,43 @@ TEST_F(PalTests, SdkVersion) EXPECT_THAT(PAL::getSdkVersion(), Eq(v)); } +namespace +{ + class ThrowingTaskHelper + { + public: + void ThrowStdException() { throw std::runtime_error("worker task boom"); } + void ThrowNonStdException() { throw 123; } + void Signal(std::atomic* ran) { ran->store(true); } + }; +} + +// A task throwing an exception must be contained by the worker thread loop; +// otherwise the exception unwinds out of the thread entry function and calls +// std::terminate, killing the host process. +TEST_F(PalTests, WorkerThreadContainsThrowingTask) +{ + auto dispatcher = PAL::WorkerThreadFactory::Create(); + ThrowingTaskHelper helper; + std::atomic ranAfterStdThrow(false); + std::atomic ranAfterNonStdThrow(false); + + PAL::dispatchTask(dispatcher.get(), &helper, &ThrowingTaskHelper::ThrowStdException); + PAL::dispatchTask(dispatcher.get(), &helper, &ThrowingTaskHelper::Signal, &ranAfterStdThrow); + + PAL::dispatchTask(dispatcher.get(), &helper, &ThrowingTaskHelper::ThrowNonStdException); + PAL::dispatchTask(dispatcher.get(), &helper, &ThrowingTaskHelper::Signal, &ranAfterNonStdThrow); + + // Wait for the follow-up tasks to run, proving the thread survived each throw. + for (int i = 0; i < 500 && !(ranAfterStdThrow.load() && ranAfterNonStdThrow.load()); ++i) + PAL::sleep(10); + + EXPECT_TRUE(ranAfterStdThrow.load()); + EXPECT_TRUE(ranAfterNonStdThrow.load()); + + dispatcher->Join(); +} + #ifdef HAVE_MAT_LOGGING class LogInitTest : public Test { diff --git a/tests/unittests/PayloadDecoderTests.cpp b/tests/unittests/PayloadDecoderTests.cpp new file mode 100644 index 000000000..8e8051cee --- /dev/null +++ b/tests/unittests/PayloadDecoderTests.cpp @@ -0,0 +1,86 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +#include "common/Common.hpp" +#include "PayloadDecoder.hpp" + +using namespace testing; +using namespace MAT; + +namespace +{ + // Builds a minimally-populated Common Schema record. to_json() in the + // PayloadDecoder unconditionally dereferences element [0] of every ext + // vector, so all seven must contain at least one element for serialization + // to succeed. + CsProtocol::Record MakeMinimalRecord() + { + CsProtocol::Record record; + record.ver = "3.0"; + record.name = "Test.Event"; + record.time = 0; + record.iKey = "o:0000"; + record.baseType = "custom"; + record.extProtocol.push_back(CsProtocol::Protocol{}); + record.extUser.push_back(CsProtocol::User{}); + record.extDevice.push_back(CsProtocol::Device{}); + record.extOs.push_back(CsProtocol::Os{}); + record.extApp.push_back(CsProtocol::App{}); + record.extNet.push_back(CsProtocol::Net{}); + record.extSdk.push_back(CsProtocol::Sdk{}); + return record; + } +} + +// A telemetry event field can legitimately contain bytes that are not valid +// UTF-8. nlohmann::json::dump() defaults to error_handler_t::strict, which +// throws type_error.316 on such input. Because DecodeRecord/DecodeRequest run +// on the decode path inside the hosting process, an unhandled throw terminates +// that process. These tests lock in the error_handler_t::replace behavior: no +// throw, and the malformed byte is emitted as the U+FFFD replacement character +// (EF BF BD). +TEST(PayloadDecoderTests, DecodeRecord_InvalidUtf8_DoesNotThrow) +{ + CsProtocol::Record record = MakeMinimalRecord(); + // Build the field with an explicit 0xFF byte (never valid UTF-8). A string + // literal escape ("...\xFF...") would rely on implementation-defined char + // conversion and can trip -Werror constant-conversion on some toolchains. + std::string name = "Bad"; + name.push_back(static_cast(0xFF)); + name += "Name"; + record.name = name; + + std::string out; + bool decoded = false; + EXPECT_NO_THROW({ decoded = exporters::DecodeRecord(record, out); }); + + // When the SDK is built with JSON + Zlib support the real decoder runs and + // must have replaced the bad byte. In a stubbed build DecodeRecord returns + // false with an empty string, in which case the no-throw guarantee above is + // what this test protects. + if (decoded) + { + EXPECT_NE(out.find("\xEF\xBF\xBD"), std::string::npos) + << "Malformed UTF-8 should be replaced with U+FFFD"; + EXPECT_EQ(out.find(static_cast(0xFF)), std::string::npos) + << "Raw invalid byte must not survive in the output"; + } +} + +TEST(PayloadDecoderTests, DecodeRecord_ValidUtf8_IsPreserved) +{ + CsProtocol::Record record = MakeMinimalRecord(); + record.name = "Valid.Event"; + + std::string out; + bool decoded = false; + EXPECT_NO_THROW({ decoded = exporters::DecodeRecord(record, out); }); + + if (decoded) + { + EXPECT_NE(out.find("Valid.Event"), std::string::npos); + EXPECT_EQ(out.find("\xEF\xBF\xBD"), std::string::npos) + << "Valid UTF-8 must not be altered"; + } +} diff --git a/tests/unittests/TaskDispatcherCAPITests.cpp b/tests/unittests/TaskDispatcherCAPITests.cpp index 0867ad046..b227deb13 100644 --- a/tests/unittests/TaskDispatcherCAPITests.cpp +++ b/tests/unittests/TaskDispatcherCAPITests.cpp @@ -9,6 +9,8 @@ #include "pal/typename.hpp" #include "mat.h" +#include + using namespace testing; using namespace MAT; using namespace PAL; @@ -227,3 +229,22 @@ TEST(TaskDispatcherCAPITests, Join) EXPECT_EQ(wasJoined, true); } +TEST(TaskDispatcherCAPITests, ExecuteCallbackThatThrowsIsContained) +{ + TaskDispatcher_CAPI taskDispatcher(&OnTaskDispatcherQueue, &OnTaskDispatcherCancel, &OnTaskDispatcherJoin); + + AutoTestHelper testHelper; + testHelper->SetShouldExecute(true); + + bool wasExecuted = false; + testHelper->SetCallbackValidation([&wasExecuted](int /*param1*/, int /*param2*/) { + wasExecuted = true; + throw std::runtime_error("task threw"); + }); + + // The dispatcher must contain the exception so it never escapes back into + // the host's dispatcher thread (which would terminate the process). + EXPECT_NO_THROW(dispatchTask(&taskDispatcher, testHelper.get(), &TestHelper::Callback, 10 /*param1*/, 20 /*param2*/)); + EXPECT_EQ(wasExecuted, true); +} + diff --git a/tests/vcpkg/test-vcpkg-android.sh b/tests/vcpkg/test-vcpkg-android.sh index c43c73967..f49a24195 100755 --- a/tests/vcpkg/test-vcpkg-android.sh +++ b/tests/vcpkg/test-vcpkg-android.sh @@ -10,6 +10,10 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" OVERLAY_PORTS="${REPO_ROOT}/tools/ports" +# Build the working tree under review (not a pinned release) so this test +# validates the actual SDK source together with the port manifest/portfile. +export MATSDK_VCPKG_SOURCE_DIR="${REPO_ROOT}" + # Android ABI/API (defaults match the repo's Android minSdk) ANDROID_ABI="${1:-arm64-v8a}" ANDROID_API="${2:-23}" diff --git a/tests/vcpkg/test-vcpkg-ios.sh b/tests/vcpkg/test-vcpkg-ios.sh index f564e4615..c1097c3bd 100755 --- a/tests/vcpkg/test-vcpkg-ios.sh +++ b/tests/vcpkg/test-vcpkg-ios.sh @@ -10,6 +10,10 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" OVERLAY_PORTS="${REPO_ROOT}/tools/ports" + +# Build the working tree under review (not a pinned release) so this test +# validates the actual SDK source together with the port manifest/portfile. +export MATSDK_VCPKG_SOURCE_DIR="${REPO_ROOT}" USE_SIMULATOR=false for arg in "$@"; do diff --git a/tests/vcpkg/test-vcpkg-linux.sh b/tests/vcpkg/test-vcpkg-linux.sh index 3482abe53..d98757db8 100755 --- a/tests/vcpkg/test-vcpkg-linux.sh +++ b/tests/vcpkg/test-vcpkg-linux.sh @@ -9,6 +9,10 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" BUILD_DIR="${SCRIPT_DIR}/build-linux" OVERLAY_PORTS="${REPO_ROOT}/tools/ports" +# Build the working tree under review (not a pinned release) so this test +# validates the actual SDK source together with the port manifest/portfile. +export MATSDK_VCPKG_SOURCE_DIR="${REPO_ROOT}" + echo "=== MSTelemetry vcpkg port test (Linux) ===" echo "Repository root: ${REPO_ROOT}" diff --git a/tests/vcpkg/test-vcpkg-macos.sh b/tests/vcpkg/test-vcpkg-macos.sh index d864928c8..9a7d1bfd3 100755 --- a/tests/vcpkg/test-vcpkg-macos.sh +++ b/tests/vcpkg/test-vcpkg-macos.sh @@ -9,6 +9,10 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" BUILD_DIR="${SCRIPT_DIR}/build-macos" OVERLAY_PORTS="${REPO_ROOT}/tools/ports" +# Build the working tree under review (not a pinned release) so this test +# validates the actual SDK source together with the port manifest/portfile. +export MATSDK_VCPKG_SOURCE_DIR="${REPO_ROOT}" + echo "=== MSTelemetry vcpkg port test (macOS) ===" echo "Repository root: ${REPO_ROOT}" diff --git a/tests/vcpkg/test-vcpkg-windows.ps1 b/tests/vcpkg/test-vcpkg-windows.ps1 index 0536f2b8f..b1390425a 100644 --- a/tests/vcpkg/test-vcpkg-windows.ps1 +++ b/tests/vcpkg/test-vcpkg-windows.ps1 @@ -14,6 +14,20 @@ $RepoRoot = (Resolve-Path (Join-Path $ScriptDir "..\..")).Path $BuildDir = Join-Path $ScriptDir "build-windows" $OverlayPorts = Join-Path $RepoRoot "tools\ports" +# Build the working tree under review (not a pinned release) so this test +# validates the actual SDK source together with the port manifest/portfile. +$env:MATSDK_VCPKG_SOURCE_DIR = $RepoRoot +# On Windows, vcpkg runs portfiles in a sanitized environment and strips custom +# variables unless they are allow-listed here. Without this, the portfile does +# not see MATSDK_VCPKG_SOURCE_DIR and silently builds the pinned release instead +# of the working tree (POSIX vcpkg passes the variable through, so the Linux/ +# macOS scripts do not need this). +if ($env:VCPKG_KEEP_ENV_VARS) { + $env:VCPKG_KEEP_ENV_VARS = "$($env:VCPKG_KEEP_ENV_VARS);MATSDK_VCPKG_SOURCE_DIR" +} else { + $env:VCPKG_KEEP_ENV_VARS = "MATSDK_VCPKG_SOURCE_DIR" +} + Write-Host "=== MSTelemetry vcpkg port test (Windows) ===" -ForegroundColor Cyan # Resolve vcpkg root: parameter > VCPKG_ROOT env var > error diff --git a/tools/.vsconfig.vs2022 b/tools/.vsconfig.vs2022 new file mode 100644 index 000000000..48e609352 --- /dev/null +++ b/tools/.vsconfig.vs2022 @@ -0,0 +1,23 @@ +{ + "version": "1.0", + "components": [ + "Microsoft.VisualStudio.Component.Windows10SDK", + "Microsoft.VisualStudio.Component.Windows11SDK.22621", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "Microsoft.VisualStudio.Component.VC.Tools.ARM", + "Microsoft.VisualStudio.Component.VC.Tools.ARM64", + "Microsoft.VisualStudio.Component.VC.Redist.14.Latest", + "Microsoft.VisualStudio.Component.VC.CMake.Project", + "Microsoft.VisualStudio.Component.VC.ATL", + "Microsoft.VisualStudio.Component.VC.ATL.ARM", + "Microsoft.VisualStudio.Component.VC.ATL.ARM64", + "Microsoft.VisualStudio.Component.VC.ATLMFC", + "Microsoft.VisualStudio.Component.VC.CLI.Support", + "Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset", + "Microsoft.VisualStudio.Component.VC.Llvm.Clang", + "Microsoft.Component.VC.Runtime.UCRTSDK", + "Microsoft.VisualStudio.Workload.NativeDesktop", + "Microsoft.VisualStudio.ComponentGroup.UWP.VC", + "Microsoft.VisualStudio.Workload.Universal" + ] +} diff --git a/tools/.vsconfig.vs2026 b/tools/.vsconfig.vs2026 new file mode 100644 index 000000000..48e609352 --- /dev/null +++ b/tools/.vsconfig.vs2026 @@ -0,0 +1,23 @@ +{ + "version": "1.0", + "components": [ + "Microsoft.VisualStudio.Component.Windows10SDK", + "Microsoft.VisualStudio.Component.Windows11SDK.22621", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "Microsoft.VisualStudio.Component.VC.Tools.ARM", + "Microsoft.VisualStudio.Component.VC.Tools.ARM64", + "Microsoft.VisualStudio.Component.VC.Redist.14.Latest", + "Microsoft.VisualStudio.Component.VC.CMake.Project", + "Microsoft.VisualStudio.Component.VC.ATL", + "Microsoft.VisualStudio.Component.VC.ATL.ARM", + "Microsoft.VisualStudio.Component.VC.ATL.ARM64", + "Microsoft.VisualStudio.Component.VC.ATLMFC", + "Microsoft.VisualStudio.Component.VC.CLI.Support", + "Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset", + "Microsoft.VisualStudio.Component.VC.Llvm.Clang", + "Microsoft.Component.VC.Runtime.UCRTSDK", + "Microsoft.VisualStudio.Workload.NativeDesktop", + "Microsoft.VisualStudio.ComponentGroup.UWP.VC", + "Microsoft.VisualStudio.Workload.Universal" + ] +} diff --git a/tools/RunMsBuild.bat b/tools/RunMsBuild.bat index 38bab739c..2e6b9bf7a 100644 --- a/tools/RunMsBuild.bat +++ b/tools/RunMsBuild.bat @@ -4,18 +4,46 @@ set TARGETPLATFORM=%1 set CONFIGURATION=%2 set TARGETS=%~3 +call :NormalizeTargets "%TARGETS%" set CUSTOM_PROPS= -if ("%~4" == "") goto endCustomProps +if "%~4" == "" goto endCustomProps set CUSTOM_PROPS=%4 echo Using custom properties file for the build: echo %CUSTOM_PROPS% :endCustomProps call tools\vcvars.cmd +if "%VSTOOLS_NOTFOUND%"=="1" ( + echo. + echo ERROR: Visual Studio was not detected, so the build cannot continue. + echo Install Visual Studio 2019, 2022, or 2026 with the Desktop development with C++ workload, + echo or run tools\setup-buildtools.cmd to install the required build tools and components. + echo To target a specific installed version, set VSTOOLS_VERSION first, for example: + echo set VSTOOLS_VERSION=vs2022 + echo. + exit /b 1 +) set MAXCPUCOUNT=%NUMBER_OF_PROCESSORS% set platform= set SOLUTION=Solutions\MSTelemetrySDK.sln -msbuild %SOLUTION% /target:%TARGETS% /p:BuildProjectReferences=true /maxcpucount:%MAXCPUCOUNT% /detailedsummary /p:Configuration=%CONFIGURATION% /p:Platform=%TARGETPLATFORM% %CUSTOM_PROPS% \ No newline at end of file +msbuild %SOLUTION% /target:%TARGETS% /p:BuildProjectReferences=true /maxcpucount:%MAXCPUCOUNT% /detailedsummary /p:Configuration=%CONFIGURATION% /p:Platform=%TARGETPLATFORM% %CUSTOM_PROPS% +exit /b %ERRORLEVEL% + +:NormalizeTargets +setlocal ENABLEDELAYEDEXPANSION +set "TARGETS_IN=%~1" +set "NORMALIZED_TARGETS=" +for %%T in ("!TARGETS_IN:,=" "!") do ( + set "TARGET=%%~T" + if /I "!TARGET:~-6!"==":Build" set "TARGET=!TARGET:~0,-6!" + if defined NORMALIZED_TARGETS ( + set "NORMALIZED_TARGETS=!NORMALIZED_TARGETS!,!TARGET!" + ) else ( + set "NORMALIZED_TARGETS=!TARGET!" + ) +) +endlocal & set "TARGETS=%NORMALIZED_TARGETS%" +exit /b 0 \ No newline at end of file diff --git a/tools/build-Win10-compact-exp.cmd b/tools/build-Win10-compact-exp.cmd index 96b7f41ac..9b2509aad 100644 --- a/tools/build-Win10-compact-exp.cmd +++ b/tools/build-Win10-compact-exp.cmd @@ -1,3 +1,2 @@ @echo off -cd .. -build-all.bat %CD%\Solutions\build.compact-exp.props +call "%~dp0..\build-all-windows.bat" "%~dp0..\Solutions\build.compact-exp.props" diff --git a/tools/ports/cpp-client-telemetry/portfile.cmake b/tools/ports/cpp-client-telemetry/portfile.cmake index b0ce77107..b2fdab830 100644 --- a/tools/ports/cpp-client-telemetry/portfile.cmake +++ b/tools/ports/cpp-client-telemetry/portfile.cmake @@ -1,10 +1,36 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO microsoft/cpp_client_telemetry - REF 4485b82005abf1d24336ace99b11df88dd578eb0 - SHA512 1f3ee1c26f1ae9e7323262c9b4c8796efba2c6addcde432d6c6c77b8c1c2f254cb8ff334b1dd0a72dc8ecfbfbae04ab374ec5ac7e5d286d6042953d53e50fd5b - HEAD_REF main -) +# In-repo overlay-port use should build the working tree under review instead of +# a pinned release -- this is what lets local port installs and tests exercise +# the SDK source + manifest together. The registry copy of this port is not under +# the SDK checkout, so it falls back to the pinned release below. +if(DEFINED ENV{MATSDK_VCPKG_SOURCE_DIR}) + set(SOURCE_PATH "$ENV{MATSDK_VCPKG_SOURCE_DIR}") + if(NOT EXISTS "${SOURCE_PATH}/CMakeLists.txt") + message(FATAL_ERROR + "MATSDK_VCPKG_SOURCE_DIR is set to '${SOURCE_PATH}', but no CMakeLists.txt " + "was found there. It must point to a cpp_client_telemetry source checkout.") + endif() + message(STATUS "cpp-client-telemetry: building local source $ENV{MATSDK_VCPKG_SOURCE_DIR} (MATSDK_VCPKG_SOURCE_DIR is set)") +else() + get_filename_component(_matsdk_overlay_source "${CURRENT_PORT_DIR}/../../.." ABSOLUTE) +endif() + +if(NOT DEFINED SOURCE_PATH + AND EXISTS "${_matsdk_overlay_source}/CMakeLists.txt" + AND EXISTS "${_matsdk_overlay_source}/lib/CMakeLists.txt" + AND EXISTS "${_matsdk_overlay_source}/tools/ports/cpp-client-telemetry/portfile.cmake") + set(SOURCE_PATH "${_matsdk_overlay_source}") + message(STATUS "cpp-client-telemetry: building in-repo overlay source ${SOURCE_PATH}") +endif() + +if(NOT DEFINED SOURCE_PATH) + vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO microsoft/cpp_client_telemetry + REF v3.10.173.1 + SHA512 e55bc35274236f57757660073c4dccccab3462342c8566212f1df4bf8824295a2bb3d3d79a11f3950e7c9252641827e9dd3d7c28c421dea3bdaee277e4f2ce32 + HEAD_REF main + ) +endif() # Determine if Apple HTTP should be used (no curl needed). # Note: BUILD_APPLE_HTTP must remain ON for macOS/iOS because the vcpkg.json @@ -20,10 +46,79 @@ if(VCPKG_TARGET_IS_IOS) set(MATSDK_BUILD_IOS ON) endif() +set(MATSDK_ANDROID_HTTP_CLIENT AUTO) +if(VCPKG_TARGET_IS_ANDROID) + file(READ "${SOURCE_PATH}/CMakeLists.txt" _matsdk_root_cmake) + if(NOT _matsdk_root_cmake MATCHES "MATSDK_ANDROID_HTTP_CLIENT") + message(FATAL_ERROR + "Android vcpkg builds require a cpp-client-telemetry source revision that " + "supports MATSDK_ANDROID_HTTP_CLIENT. Update this port's REF/SHA512 to a " + "newer SDK release, or set MATSDK_VCPKG_SOURCE_DIR to a local checkout " + "that contains the Android Java transport selector.") + endif() + if("android-curl-openssl" IN_LIST FEATURES OR "android-curl-mbedtls" IN_LIST FEATURES) + set(MATSDK_ANDROID_HTTP_CLIENT CURL) + endif() +endif() + +# curl-openssl/curl-mbedtls choose the Linux TLS backend. Android defaults to +# Java/JNI HTTP and uses separate explicit android-curl-* features for its curl +# escape hatch. vcpkg cannot express mutual exclusivity or "exactly one of", so +# validate it here -- but only where curl is actually used, to avoid failing +# legitimate cross-platform manifests on Windows/Apple. +set(_matsdk_http_features "") +if(VCPKG_TARGET_IS_ANDROID) + set(_matsdk_http_feature_candidates android-curl-openssl android-curl-mbedtls) +else() + set(_matsdk_http_feature_candidates curl-openssl curl-mbedtls) +endif() +foreach(_matsdk_http_feature ${_matsdk_http_feature_candidates}) + if(_matsdk_http_feature IN_LIST FEATURES) + list(APPEND _matsdk_http_features ${_matsdk_http_feature}) + endif() +endforeach() +list(LENGTH _matsdk_http_features _matsdk_http_feature_count) +if(VCPKG_TARGET_IS_LINUX OR MATSDK_ANDROID_HTTP_CLIENT STREQUAL "CURL") + if(_matsdk_http_feature_count GREATER 1) + message(FATAL_ERROR + "The curl HTTP backend features are mutually exclusive but multiple were " + "selected. On Linux, use exactly one of curl-openssl/curl-mbedtls. On " + "Android, use exactly one of android-curl-openssl/android-curl-mbedtls. " + "If you added a non-default backend, use the [core,...] form " + "(default-features=false) so the default curl-openssl feature is dropped.") + elseif(_matsdk_http_feature_count EQUAL 0 AND VCPKG_TARGET_IS_LINUX) + # The built-in curl HTTP client requires exactly one TLS backend. The [core,...] + # form drops the default curl-openssl, so fail fast (with a complete example) + # rather than letting the SDK CMake fail later on a missing libcurl. + message(FATAL_ERROR + "On Linux the built-in curl HTTP client requires exactly one TLS backend " + "feature, but none was selected. The [core,...] form drops the default " + "curl-openssl feature, so re-add a curl backend together with a SQLite " + "backend, e.g. " + "cpp-client-telemetry[core,curl-mbedtls,system-sqlite] " + "(or minimal-sqlite in place of system-sqlite).") + elseif(_matsdk_http_feature_count EQUAL 0) + message(FATAL_ERROR + "On Android, MATSDK_ANDROID_HTTP_CLIENT=CURL requires exactly one explicit " + "Android curl backend feature. Use android-curl-openssl or " + "android-curl-mbedtls together with a SQLite backend, e.g. " + "cpp-client-telemetry[core,android-curl-openssl,system-sqlite].") + endif() +endif() + +# minimal-sqlite -> -DMATSDK_MINIMAL_SQLITE=ON (private feature-stripped SQLite). +vcpkg_check_features( + OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + minimal-sqlite MATSDK_MINIMAL_SQLITE +) + vcpkg_cmake_configure( SOURCE_PATH "${SOURCE_PATH}" OPTIONS + ${FEATURE_OPTIONS} -DMATSDK_USE_VCPKG_DEPS=ON + -DMATSDK_ANDROID_HTTP_CLIENT=${MATSDK_ANDROID_HTTP_CLIENT} -DBUILD_HEADERS=ON -DBUILD_LIBRARY=ON -DBUILD_TEST_TOOL=OFF @@ -46,8 +141,5 @@ vcpkg_cmake_config_fixup(PACKAGE_NAME MSTelemetry CONFIG_PATH lib/cmake/MSTeleme file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") -# Install usage instructions -file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") - # Install license vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/tools/ports/cpp-client-telemetry/usage b/tools/ports/cpp-client-telemetry/usage deleted file mode 100644 index 736d289f6..000000000 --- a/tools/ports/cpp-client-telemetry/usage +++ /dev/null @@ -1,4 +0,0 @@ -cpp-client-telemetry provides CMake targets: - - find_package(MSTelemetry CONFIG REQUIRED) - target_link_libraries(main PRIVATE MSTelemetry::mat) diff --git a/tools/ports/cpp-client-telemetry/vcpkg.json b/tools/ports/cpp-client-telemetry/vcpkg.json index d721df65a..d183bf6ca 100644 --- a/tools/ports/cpp-client-telemetry/vcpkg.json +++ b/tools/ports/cpp-client-telemetry/vcpkg.json @@ -1,22 +1,12 @@ { "name": "cpp-client-telemetry", - "version": "3.10.161.1", + "version": "3.10.173.1", "description": "Microsoft 1DS C/C++ Client Telemetry Library", "homepage": "https://github.com/microsoft/cpp_client_telemetry", "license": "Apache-2.0", "supports": "((windows & !mingw) | linux | osx | ios | android) & !uwp", "dependencies": [ "nlohmann-json", - "sqlite3", - "zlib", - { - "name": "curl", - "default-features": false, - "features": [ - "openssl" - ], - "platform": "linux | android" - }, { "name": "vcpkg-cmake", "host": true @@ -24,6 +14,83 @@ { "name": "vcpkg-cmake-config", "host": true + }, + { + "name": "zlib", + "platform": "!osx & !ios" + } + ], + "default-features": [ + "curl-openssl", + "system-sqlite" + ], + "features": { + "android-curl-mbedtls": { + "description": "On Android, explicitly build the native libcurl HTTP client with the mbedTLS backend instead of the default Java/JNI HttpClient_Android bridge.", + "supports": "android", + "dependencies": [ + { + "name": "curl", + "default-features": false, + "features": [ + "mbedtls" + ], + "platform": "android" + } + ] + }, + "android-curl-openssl": { + "description": "On Android, explicitly build the native libcurl HTTP client with the OpenSSL backend instead of the default Java/JNI HttpClient_Android bridge.", + "supports": "android", + "dependencies": [ + { + "name": "curl", + "default-features": false, + "features": [ + "openssl" + ], + "platform": "android" + } + ] + }, + "curl-mbedtls": { + "description": "Built-in libcurl HTTP client with the mbedTLS backend instead of OpenSSL (smaller footprint). Affects Linux only. Use [core,curl-mbedtls,system-sqlite] to drop the default OpenSSL curl; the [core,...] form drops all defaults (including system-sqlite), so also re-select system-sqlite or minimal-sqlite.", + "dependencies": [ + { + "name": "curl", + "default-features": false, + "features": [ + "mbedtls" + ], + "platform": "linux" + } + ] + }, + "curl-openssl": { + "description": "Built-in libcurl HTTP client with the OpenSSL TLS backend (default). Affects Linux only; Android uses the Java/JNI bridge unless an android-curl-* feature is selected, Windows uses WinInet, and Apple uses NSURLSession.", + "dependencies": [ + { + "name": "curl", + "default-features": false, + "features": [ + "openssl" + ], + "platform": "linux" + } + ] + }, + "minimal-sqlite": { + "description": "Build a private, feature-stripped SQLite compiled from the SDK's vendored amalgamation instead of linking the external sqlite3 package. Smaller footprint; combine with [core,minimal-sqlite] to also drop the sqlite3 dependency." + }, + "system-sqlite": { + "description": "Link the external vcpkg sqlite3 package for the offline storage cache (default). On macOS/iOS the SDK links the system libsqlite3 instead, so this dependency is not pulled there.", + "dependencies": [ + { + "name": "sqlite3", + "default-features": false, + "platform": "!osx & !ios" + } + ] } - ] + } } diff --git a/tools/setup-buildtools.cmd b/tools/setup-buildtools.cmd index 890467256..d8a143697 100644 --- a/tools/setup-buildtools.cmd +++ b/tools/setup-buildtools.cmd @@ -38,10 +38,10 @@ if NOT exist "%VSINSTALLER%" ( echo Visual Studio installer: echo %VSINSTALLER% -REM Install optional components required for ARM build - vs2017-BuildTools +REM Install optional components required for supported Windows build targets. if exist "%VSINSTALLDIR%" ( echo Running Visual Studio installer.. - "%VSINSTALLER%" modify --installPath "%VSINSTALLDIR%" --config "%~dp0\.vsconfig.%VSVERSION%" --force --quiet --norestart + "%VSINSTALLER%" modify --installPath "%VSINSTALLDIR%" --config "%~dp0\.vsconfig.vs%VSVERSION%" --force --quiet --norestart ) where /Q vcpkg.exe diff --git a/tools/vcvars.cmd b/tools/vcvars.cmd index 18ae09e0a..ea0ec1c38 100644 --- a/tools/vcvars.cmd +++ b/tools/vcvars.cmd @@ -8,9 +8,34 @@ REM 2. Visual Studio 2017 BuildTools REM 3. Visual Studio 2019 Enterprise REM 4. Visual Studio 2019 Community REM 5. Visual Studio 2019 BuildTools +REM 6. Visual Studio 2022 Enterprise +REM 7. Visual Studio 2022 Professional +REM 8. Visual Studio 2022 Community +REM 9. Visual Studio 2022 BuildTools +REM 10. Visual Studio 2026 Enterprise +REM 11. Visual Studio 2026 Professional +REM 12. Visual Studio 2026 Community +REM 13. Visual Studio 2026 BuildTools REM REM 1st parameter - Visual Studio version + +REM Start from a clean detection state so values left in the shell by a previous +REM run (or the caller environment) can't be mistaken for the result of this run. +REM Only the matching detection path below sets these again; in particular this +REM prevents a stale VSINSTALLDIR/VSVERSION from making callers such as +REM tools\setup-buildtools.cmd act on the wrong install after a failed detection. +set "VSTOOLS_NOTFOUND=" +set "VSINSTALLDIR=" +set "VSDEVCMD=" +set "VSVERSION=" + +REM Remember an explicit version request so we can warn later if detection falls +REM back to a different Visual Studio install than the one that was asked for. +set "VSTOOLS_REQUESTED=" +if "%1" neq "" set "VSTOOLS_REQUESTED=%1" +if not defined VSTOOLS_REQUESTED if "%VSTOOLS_VERSION%" neq "" set "VSTOOLS_REQUESTED=%VSTOOLS_VERSION%" + if "%1" neq "" ( goto %1 ) @@ -84,6 +109,16 @@ if exist "%VSDEVCMD%" ( goto tools_configured ) +:vs2022_professional +SET VSVERSION=2022 +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\2022\Professional\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\2022\Professional" + echo Building with vs2022 Professional... + call "%VSDEVCMD%" + goto tools_configured +) + :vs2022_community SET VSVERSION=2022 set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\2022\Community\Common7\Tools\VsDevCmd.bat" @@ -94,6 +129,89 @@ if exist "%VSDEVCMD%" ( goto tools_configured ) +:vs2022_buildtools +SET VSVERSION=2022 +set "VSDEVCMD=%ProgramFiles(x86)%\Microsoft Visual Studio\2022\BuildTools\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles(x86)%\Microsoft Visual Studio\2022\BuildTools" + echo Building with vs2022 BuildTools... + call "%VSDEVCMD%" + goto tools_configured +) + +:vs2026 +:vs2026_enterprise +SET VSVERSION=2026 +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\2026\Enterprise\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\2026\Enterprise" + echo Building with vs2026 Enterprise... + call "%VSDEVCMD%" + goto tools_configured +) + +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\18\Enterprise\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\18\Enterprise" + echo Building with vs2026 Enterprise... + call "%VSDEVCMD%" + goto tools_configured +) + +:vs2026_professional +SET VSVERSION=2026 +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\2026\Professional\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\2026\Professional" + echo Building with vs2026 Professional... + call "%VSDEVCMD%" + goto tools_configured +) + +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\18\Professional\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\18\Professional" + echo Building with vs2026 Professional... + call "%VSDEVCMD%" + goto tools_configured +) + +:vs2026_community +SET VSVERSION=2026 +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\2026\Community\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\2026\Community" + echo Building with vs2026 Community... + call "%VSDEVCMD%" + goto tools_configured +) + +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\18\Community\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\18\Community" + echo Building with vs2026 Community... + call "%VSDEVCMD%" + goto tools_configured +) + +:vs2026_buildtools +SET VSVERSION=2026 +set "VSDEVCMD=%ProgramFiles(x86)%\Microsoft Visual Studio\2026\BuildTools\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles(x86)%\Microsoft Visual Studio\2026\BuildTools" + echo Building with vs2026 BuildTools... + call "%VSDEVCMD%" + goto tools_configured +) + +set "VSDEVCMD=%ProgramFiles(x86)%\Microsoft Visual Studio\18\BuildTools\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles(x86)%\Microsoft Visual Studio\18\BuildTools" + echo Building with vs2026 BuildTools... + call "%VSDEVCMD%" + goto tools_configured +) + echo WARNING:********************************************* echo WARNING: cannot auto-detect Visual Studio version !!! echo WARNING:********************************************* @@ -102,3 +220,20 @@ set VSVERSION= exit /b 0 :tools_configured + +REM Warn if an explicit version was requested but detection fell back to a +REM different Visual Studio install (the label cascade silently moves forward, +REM e.g. a vs2022 request can end up on vs2026), which can lead to a confusing +REM toolset mismatch later when PlatformToolset is pinned to the requested version. +if not defined VSTOOLS_REQUESTED goto :tools_configured_done +if not defined VSVERSION goto :tools_configured_done +echo "%VSTOOLS_REQUESTED%" | findstr /I /C:"%VSVERSION%" >nul +if errorlevel 1 ( + echo WARNING: Requested Visual Studio "%VSTOOLS_REQUESTED%" was not found; using Visual Studio %VSVERSION% instead. + echo WARNING: If a specific toolset is required, install that Visual Studio version or set VSTOOLS_VERSION/PlatformToolset to match what is installed. +) + +:tools_configured_done +REM Visual Studio was configured; callers rely on VSTOOLS_NOTFOUND rather than +REM this script's exit code, so return success regardless of the version probe. +exit /b 0 diff --git a/tools/version.js b/tools/version.js index 4f090e1be..45c4c922e 100644 --- a/tools/version.js +++ b/tools/version.js @@ -44,8 +44,11 @@ function generateVersionHpp() { // Read version tag var ver1 = readAll("..\\Solutions\\version.txt"); - // Remove end-of-line - ver1 = ver1.trim(); + // Remove leading/trailing whitespace. Use a regex rather than String.trim() so this + // runs under the Windows Script Host JScript engine (cscript), which does not + // implement String.prototype.trim(); the global anchored pattern also fully strips + // trailing newlines (the CodeQL incomplete-sanitization concern). + ver1 = ver1.replace(/^\s+|\s+$/g, ""); ver1 = updateYearAndDay(ver1); // console.log("version.txt => " + ver1 + "\n"); var ver2 = ver1.split(".").join(","); diff --git a/wrappers/obj-c/ODWLogConfiguration.h b/wrappers/obj-c/ODWLogConfiguration.h index 3cbdaaa61..6e3f77946 100644 --- a/wrappers/obj-c/ODWLogConfiguration.h +++ b/wrappers/obj-c/ODWLogConfiguration.h @@ -44,6 +44,11 @@ extern NSString * _Nonnull const ODWCFG_BOOL_ENABLE_WAL_JOURNAL; */ extern NSString * _Nonnull const ODWCFG_BOOL_ENABLE_NET_DETECT; +/*! + Scrub (obfuscate) the client IP address at the collector. Applied unless explicitly set to false (on by default; not present in the default configuration). +*/ +extern NSString * _Nonnull const ODWCFG_BOOL_ENABLE_IP_SCRUBBING; + /*! The event collection URI. */ diff --git a/wrappers/obj-c/ODWLogConfiguration.mm b/wrappers/obj-c/ODWLogConfiguration.mm index d69ddf70c..611b92940 100644 --- a/wrappers/obj-c/ODWLogConfiguration.mm +++ b/wrappers/obj-c/ODWLogConfiguration.mm @@ -50,6 +50,11 @@ */ NSString *const ODWCFG_BOOL_ENABLE_NET_DETECT = @"enableNetworkDetector"; +/*! + Scrub (obfuscate) the client IP address at the collector. Applied unless explicitly set to false (on by default; not present in the default configuration). +*/ +NSString *const ODWCFG_BOOL_ENABLE_IP_SCRUBBING = @"enableIpScrubbing"; + /*! The event collection URI. */