From 8853890fe95eb042f4cc4b5f2a58f4b110fcc066 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 26 Aug 2026 15:11:18 +0800 Subject: [PATCH 1/2] [fix](thirdparty) order Paimon codec dependencies --- .github/workflows/build-thirdparty.yml | 5 +- thirdparty/arrow-paimon-vars.sh | 4 +- .../paimon-cpp-buildutils-static-deps.patch | 16 ++++ .../test/paimon-codec-dependency-test.sh | 76 +++++++++++++++++++ 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100755 thirdparty/test/paimon-codec-dependency-test.sh diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 7ae7cb2d43ed1b..af4d524d80b9f9 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -59,6 +59,7 @@ jobs: - 'thirdparty/patches/apache-arrow-*.patch' - 'thirdparty/patches/paimon-cpp-*.patch' - 'thirdparty/test/arrow-paimon-lifecycle-test.sh' + - 'thirdparty/test/paimon-codec-dependency-test.sh' - '.github/workflows/build-thirdparty.yml' arrow_paimon_lifecycle_test: @@ -71,7 +72,9 @@ jobs: uses: actions/checkout@v4 - name: Test focused thirdparty lifecycle - run: thirdparty/test/arrow-paimon-lifecycle-test.sh + run: | + thirdparty/test/arrow-paimon-lifecycle-test.sh + thirdparty/test/paimon-codec-dependency-test.sh script_test: name: Thirdparty Script Test diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh index e6f8cfb7c6f27c..77e6f8b9e1a236 100644 --- a/thirdparty/arrow-paimon-vars.sh +++ b/thirdparty/arrow-paimon-vars.sh @@ -61,7 +61,9 @@ PAIMON_BUILD_SCHEMA_VERSION="1" # mandatory before reuse. Delete the lists once master has published a prebuilt that # carries the semantic marker itself. ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="ab79ab0bbfbf93f9860050fb751b20fee9e40d96" -PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="cb82e41ba46f534e611cdd52e66b53c227d49bf8" +# Dependency ordering does not change the installed Paimon artifacts, so existing +# complete prebuilts remain compatible with the corrected clean-build graph. +PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="fe08a2f6c995dd469baa774245dc7edb1475b435" ARROW_LEGACY_BUILD_FINGERPRINTS=( # published 2026-08-19, master after apache/doris#66842 8356a3a464b93f3a59a6bdae6a51258947ab35d2 diff --git a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch index fc1fe1553126b9..7cca89e6df8c4c 100644 --- a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch +++ b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch @@ -343,3 +343,19 @@ diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/b + list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR) + list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR) + list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH) + +diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt +index 76dd03b..9317324 100644 +--- a/src/paimon/CMakeLists.txt ++++ b/src/paimon/CMakeLists.txt +@@ -281,6 +281,10 @@ add_paimon_lib(paimon + ${PAIMON_COMMON_SRCS} + ${PAIMON_CORE_SRCS} + DEPENDENCIES ++ # Codec headers come from ExternalProject install steps, so object ++ # compilation must not race those steps in clean parallel builds. ++ zstd ++ snappy + arrow + tbb + glog diff --git a/thirdparty/test/paimon-codec-dependency-test.sh b/thirdparty/test/paimon-codec-dependency-test.sh new file mode 100755 index 00000000000000..98e4c8f6d3051c --- /dev/null +++ b/thirdparty/test/paimon-codec-dependency-test.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +tmpdir="$(mktemp -d)" +trap 'rm -rf "${tmpdir}"' EXIT + +fixture="${tmpdir}/paimon-codec-dependency" +focused_patch="${fixture}/codec-dependency.patch" +mkdir -p "${fixture}/src/paimon" +{ + printf '%s\n' 'add_paimon_lib(paimon' + printf '%s\n' ' SOURCES' + printf '%s\n' ' ${PAIMON_COMMON_SRCS}' + printf '%s\n' ' ${PAIMON_CORE_SRCS}' + printf '%s\n' ' DEPENDENCIES' + printf '%s\n' ' arrow' + printf '%s\n' ' tbb' + printf '%s\n' ' glog' + printf '%s\n' ' fmt' + printf '%s\n' ' roaring_bitmap' + printf '%s\n' ' xxhash' + printf '%s\n' ' Threads::Threads' + printf '%s\n' ' RapidJSON' + printf '%s\n' ' STATIC_LINK_LIBS' + printf '%s\n' ' arrow)' +} >"${fixture}/src/paimon/CMakeLists.txt" + +awk ' + /^diff --git / { + if (found) { + exit + } + if ($0 == "diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt") { + found = 1 + } + } + found { print } +' "${ROOT}/patches/paimon-cpp-buildutils-static-deps.patch" >"${focused_patch}" +[[ -s "${focused_patch}" ]] || fail "the Paimon codec dependency patch is missing" +( + cd "${fixture}" + patch -s -p1 <"${focused_patch}" +) || fail "the Paimon codec dependency patch did not apply" + +dependencies="$(sed -n '/^[[:space:]]*DEPENDENCIES$/,/^[[:space:]]*STATIC_LINK_LIBS$/p' \ + "${fixture}/src/paimon/CMakeLists.txt")" +grep -Eq '^[[:space:]]+zstd$' <<<"${dependencies}" || + fail "paimon_objlib does not wait for the ZSTD headers" +grep -Eq '^[[:space:]]+snappy$' <<<"${dependencies}" || + fail "paimon_objlib does not wait for the Snappy headers" + +echo "PASS" From c8a0c50fa422a6d28d0cb049d7bda94c47680aa2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 26 Aug 2026 16:14:29 +0800 Subject: [PATCH 2/2] [fix](thirdparty) order Paimon LZ4 dependency --- thirdparty/arrow-paimon-vars.sh | 2 +- thirdparty/patches/paimon-cpp-buildutils-static-deps.patch | 3 ++- thirdparty/test/paimon-codec-dependency-test.sh | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh index 77e6f8b9e1a236..d532a740e63d68 100644 --- a/thirdparty/arrow-paimon-vars.sh +++ b/thirdparty/arrow-paimon-vars.sh @@ -63,7 +63,7 @@ PAIMON_BUILD_SCHEMA_VERSION="1" ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="ab79ab0bbfbf93f9860050fb751b20fee9e40d96" # Dependency ordering does not change the installed Paimon artifacts, so existing # complete prebuilts remain compatible with the corrected clean-build graph. -PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="fe08a2f6c995dd469baa774245dc7edb1475b435" +PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT="9887cf1ec13106da4250f8d7ea931a258879089c" ARROW_LEGACY_BUILD_FINGERPRINTS=( # published 2026-08-19, master after apache/doris#66842 8356a3a464b93f3a59a6bdae6a51258947ab35d2 diff --git a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch index 7cca89e6df8c4c..213cc279f52e5c 100644 --- a/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch +++ b/thirdparty/patches/paimon-cpp-buildutils-static-deps.patch @@ -348,7 +348,7 @@ diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 76dd03b..9317324 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt -@@ -281,6 +281,10 @@ add_paimon_lib(paimon +@@ -281,6 +281,11 @@ add_paimon_lib(paimon ${PAIMON_COMMON_SRCS} ${PAIMON_CORE_SRCS} DEPENDENCIES @@ -356,6 +356,7 @@ index 76dd03b..9317324 100644 + # compilation must not race those steps in clean parallel builds. + zstd + snappy ++ lz4 arrow tbb glog diff --git a/thirdparty/test/paimon-codec-dependency-test.sh b/thirdparty/test/paimon-codec-dependency-test.sh index 98e4c8f6d3051c..4d3b92458b8c26 100755 --- a/thirdparty/test/paimon-codec-dependency-test.sh +++ b/thirdparty/test/paimon-codec-dependency-test.sh @@ -72,5 +72,7 @@ grep -Eq '^[[:space:]]+zstd$' <<<"${dependencies}" || fail "paimon_objlib does not wait for the ZSTD headers" grep -Eq '^[[:space:]]+snappy$' <<<"${dependencies}" || fail "paimon_objlib does not wait for the Snappy headers" +grep -Eq '^[[:space:]]+lz4$' <<<"${dependencies}" || + fail "paimon_objlib does not wait for the LZ4 headers" echo "PASS"