diff --git a/.github/workflows/IO_Iceberg_Integration_Tests.yml b/.github/workflows/IO_Iceberg_Integration_Tests.yml index 6341c51124c5..ef6fe3485286 100644 --- a/.github/workflows/IO_Iceberg_Integration_Tests.yml +++ b/.github/workflows/IO_Iceberg_Integration_Tests.yml @@ -72,6 +72,8 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: 17 - name: Run IcebergIO Integration Test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/IO_Iceberg_Integration_Tests_Dataflow.yml b/.github/workflows/IO_Iceberg_Integration_Tests_Dataflow.yml index 38a3a2514855..ec0659394813 100644 --- a/.github/workflows/IO_Iceberg_Integration_Tests_Dataflow.yml +++ b/.github/workflows/IO_Iceberg_Integration_Tests_Dataflow.yml @@ -72,6 +72,8 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: 17 - name: Run IcebergIO Integration Tests on Dataflow uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/IO_Iceberg_Managed_Integration_Tests_Dataflow.yml b/.github/workflows/IO_Iceberg_Managed_Integration_Tests_Dataflow.yml index ba72a85204a1..fb4d1a1e4a74 100644 --- a/.github/workflows/IO_Iceberg_Managed_Integration_Tests_Dataflow.yml +++ b/.github/workflows/IO_Iceberg_Managed_Integration_Tests_Dataflow.yml @@ -72,6 +72,8 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: 17 - name: Run IcebergIO Managed Integration Tests on Dataflow uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/IO_Iceberg_Performance_Tests.yml b/.github/workflows/IO_Iceberg_Performance_Tests.yml index e3ec8b569012..1b6642b06dda 100644 --- a/.github/workflows/IO_Iceberg_Performance_Tests.yml +++ b/.github/workflows/IO_Iceberg_Performance_Tests.yml @@ -72,6 +72,8 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: 17 - name: Run IcebergIO Performance Test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/IO_Iceberg_Unit_Tests.yml b/.github/workflows/IO_Iceberg_Unit_Tests.yml index dd8069c74af0..dbf0342ea779 100644 --- a/.github/workflows/IO_Iceberg_Unit_Tests.yml +++ b/.github/workflows/IO_Iceberg_Unit_Tests.yml @@ -91,6 +91,8 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: 17 - name: run IcebergIO build script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml index 2f9741c01a01..aa8f5b3bb3fd 100644 --- a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml +++ b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml @@ -117,4 +117,5 @@ jobs: with: gradle-command: :sdks:python:test-suites:direct:crossLanguageWrapperValidationPreCommit --info arguments: | + -PtestJavaVersion=17 -Pjava17Home=$JAVA_HOME_17_X64 diff --git a/CHANGES.md b/CHANGES.md index bd9c6ea246a5..4c4bf3bd12fa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -62,7 +62,6 @@ * Python SDK now supports memory profiling with Memray ([#38853](https://github.com/apache/beam/issues/38853)). * (Python) Added [Qdrant](https://qdrant.tech/) VectorDatabaseWriteConfig implementation ([#38141](https://github.com/apache/beam/issues/38141)). - ## I/Os * Support for reading from Delta Lake added (Java) ([#38551](https://github.com/apache/beam/issues/38551)). @@ -82,6 +81,8 @@ * (Python) Typehints of dataclass fields are honored during type inferences. To restore the behavior of fallback-to-any, use pipeline option `--exclude_infer_dataclass_field_type` ([#38797](https://github.com/apache/beam/issues/38797)). However fixing forward is recommended. +* (Java) IcebergIO now requires Java 17 at runtime. This raises the floor in preparation for the Iceberg 1.11.0 upgrade, which is published as Java 17 bytecode. Pipelines using `Managed.read(ICEBERG)`/`Managed.write(ICEBERG)` must run on a Java 17+ JVM ([#38925](https://github.com/apache/beam/issues/38925)). +* (Java) Beam SQL's IcebergIO extension now requires Java 17, as it depends on the IcebergIO module ([#38925](https://github.com/apache/beam/issues/38925)). ## Deprecations diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index b5aef5d23ad4..95f8ee232695 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1224,20 +1224,26 @@ class BeamModulePlugin implements Plugin { maxHeapSize = '2g' } + // NOTE: Use the character class "[.]" instead of an escaped "\\." to match a literal dot in + // these Checker Framework -AskipDefs/-AskipUses regexes. When a module is compiled on an older + // host JDK and forked to a newer JDK via javaXXHome (e.g. iceberg's requireJavaVersion 17 on a + // Java 11 CI host), Gradle passes the javac arguments through an @argfile. Backslash escapes do + // not survive that round-trip intact, so "\\." becomes a literal-backslash regex that matches + // nothing and the suppression is silently dropped. "[.]" is backslash-free and survives. List skipDefRegexes = [] skipDefRegexes << "AutoValue_.*" skipDefRegexes << "AutoBuilder_.*" skipDefRegexes << "AutoOneOf_.*" - skipDefRegexes << ".*\\.jmh_generated\\..*" + skipDefRegexes << ".*[.]jmh_generated[.].*" skipDefRegexes += configuration.generatedClassPatterns skipDefRegexes += configuration.classesTriggerCheckerBugs.keySet() String skipDefCombinedRegex = skipDefRegexes.collect({ regex -> "(${regex})"}).join("|") List skipUsesRegexes = [] // zstd-jni is not annotated, handles Zstd(De)CompressCtx.loadDict(null) just fine - skipUsesRegexes << "^com\\.github\\.luben\\.zstd\\..*" + skipUsesRegexes << "^com[.]github[.]luben[.]zstd[.].*" // SLF4J logger handles null log message parameters - skipUsesRegexes << "^org\\.slf4j\\.Logger.*" + skipUsesRegexes << "^org[.]slf4j[.]Logger.*" String skipUsesCombinedRegex = skipUsesRegexes.collect({ regex -> "(${regex})"}).join("|") project.apply plugin: 'org.checkerframework' @@ -2715,6 +2721,11 @@ class BeamModulePlugin implements Plugin { def usesDataflowRunner = config.pythonPipelineOptions.contains("--runner=TestDataflowRunner") || config.pythonPipelineOptions.contains("--runner=DataflowRunner") String ver = project.findProperty('testJavaVersion') def javaContainerSuffix = ver ? getSupportedJavaVersion(ver) : getSupportedJavaVersion() + // When a specific test JDK is requested (-PtestJavaVersion), launch the Python-started + // expansion service on it as well (see the exec block below). Some bundled IOs (e.g. + // IcebergIO) are compiled for Java 17, so the expansion service must run on a JDK that can + // load them. Mirrors the JAVA_HOME handling in the other cross-language task factories. + String testJavaHome = ver ? project.findProperty("java${ver}Home") : null // Sets up, collects, and runs Python pipeline tests project.tasks.register(config.name+"PythonUsingJava") { @@ -2744,6 +2755,9 @@ class BeamModulePlugin implements Plugin { project.exec { // environment variable to indicate that jars have been built environment "EXPANSION_JARS", config.expansionProjectPaths + if (testJavaHome) { + environment "JAVA_HOME", testJavaHome + } String additionalDependencyCmd = "" if (config.additionalDeps != null && !config.additionalDeps.isEmpty()){ additionalDependencyCmd = "&& pip install ${config.additionalDeps.join(' ')} " diff --git a/examples/java/iceberg/build.gradle b/examples/java/iceberg/build.gradle index 4d4a1fb44413..2cff15c5fb0d 100644 --- a/examples/java/iceberg/build.gradle +++ b/examples/java/iceberg/build.gradle @@ -26,8 +26,8 @@ plugins { applyJavaNature( exportJavadoc: false, automaticModuleName: 'org.apache.beam.examples.iceberg', - // iceberg requires Java11+ - requireJavaVersion: JavaVersion.VERSION_11 + // iceberg requires Java17+ + requireJavaVersion: JavaVersion.VERSION_17 ) description = "Apache Beam :: Examples :: Java :: Iceberg" diff --git a/sdks/java/extensions/sql/iceberg/build.gradle b/sdks/java/extensions/sql/iceberg/build.gradle index 893a485e7d86..97a128d69e9e 100644 --- a/sdks/java/extensions/sql/iceberg/build.gradle +++ b/sdks/java/extensions/sql/iceberg/build.gradle @@ -22,8 +22,8 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.extensions.sql.meta.provider.hcatalog', - // iceberg requires Java11+ - requireJavaVersion: JavaVersion.VERSION_11, + // iceberg requires Java17+ + requireJavaVersion: JavaVersion.VERSION_17, ) dependencies { diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index 8142c5f5b90b..7cdd32ed90e4 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -23,8 +23,8 @@ import java.util.stream.Collectors plugins { id 'org.apache.beam.module' } applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.io.iceberg', - // iceberg ended support for Java 8 in 1.7.0 - requireJavaVersion: JavaVersion.VERSION_11, + // iceberg ended support for Java 11 in 1.11.0 + requireJavaVersion: JavaVersion.VERSION_17, ) description = "Apache Beam :: SDKs :: Java :: IO :: Iceberg"