From 7cc385b9f50af92dbd3888c202842ea592b8672a Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 8 Oct 2026 12:32:55 +0200 Subject: [PATCH 1/3] chore: Remove support for hadoop 3.4.2 --- .github/workflows/maven.yaml | 4 ++-- README.md | 2 +- pom.xml | 14 -------------- test/topology-provider/stack/03-hdfs.yaml | 5 ++--- test/topology-provider/stack/04-spark.yaml | 4 ++-- test/topology-provider/stack/05-access-hdfs.yaml | 2 +- test/topology-provider/teragen/Dockerfile | 7 +++++-- 7 files changed, 13 insertions(+), 25 deletions(-) diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml index cde8ff3..c439cd8 100644 --- a/.github/workflows/maven.yaml +++ b/.github/workflows/maven.yaml @@ -34,7 +34,7 @@ jobs: cache: maven - name: Build with Maven - run: mvn -P hadoop-3.4.2 -B verify + run: mvn -P hadoop-3.5.0 -B verify - name: Update dependency graph if: | @@ -42,4 +42,4 @@ jobs: && !github.event.pull_request.head.repo.fork uses: advanced-security/maven-dependency-submission-action@b275d12641ac2d2108b2cbb7598b154ad2f2cee8 # v5.0.0 with: - maven-args: -P hadoop-3.4.2 + maven-args: -P hadoop-3.5.0 diff --git a/README.md b/README.md index d332fd2..d4c68a9 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This project contains multiple plugins for Apache Hadoop, which are intended to ## Installation -Currently you can compile hdfs-utils against Hadoop 3.4.2, 3.4.3 or 3.5.0. You need to specify the version by activating either the `hadoop-3.4.2`, `hadoop-3.4.3` or the `hadoop-3.5.0` profile below. +Currently you can compile hdfs-utils against Hadoop 3.4.3 or 3.5.0. You need to specify the version by activating either the `hadoop-3.4.3` or the `hadoop-3.5.0` profile below. Run e.g. `mvn clean package -P hadoop-3.5.0` and put the resulting `target/hdfs-utils-*.jar` file on your HDFS classpath. The easiest way to achieve this is to put it in the directory `/stackable/hadoop/share/hadoop/tools/lib/`. diff --git a/pom.xml b/pom.xml index ef7402c..901be84 100644 --- a/pom.xml +++ b/pom.xml @@ -314,20 +314,6 @@ - - hadoop-3.4.2 - - - 3.4.2 - - 1.7.36 - - hadoop-3.4.3 diff --git a/test/topology-provider/stack/03-hdfs.yaml b/test/topology-provider/stack/03-hdfs.yaml index 0d98522..d873f1c 100644 --- a/test/topology-provider/stack/03-hdfs.yaml +++ b/test/topology-provider/stack/03-hdfs.yaml @@ -5,7 +5,7 @@ metadata: name: simple-zk spec: image: - productVersion: 3.9.4 + productVersion: 3.9.6 pullPolicy: IfNotPresent servers: roleGroups: @@ -18,8 +18,7 @@ metadata: name: simple-hdfs spec: image: - productVersion: 3.4.2 - custom: oci.stackable.tech/sandbox/hadoop:3.4.2-stackable0.0.0-dev + productVersion: 3.5.0 pullPolicy: IfNotPresent clusterConfig: dfsReplication: 1 diff --git a/test/topology-provider/stack/04-spark.yaml b/test/topology-provider/stack/04-spark.yaml index bfbf724..5601901 100644 --- a/test/topology-provider/stack/04-spark.yaml +++ b/test/topology-provider/stack/04-spark.yaml @@ -5,8 +5,8 @@ metadata: name: spark-teragen spec: sparkImage: - custom: oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-terasort - productVersion: 3.5.6 + custom: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-terasort + productVersion: 3.5.8 pullPolicy: IfNotPresent mode: cluster mainApplicationFile: local:///tmp/spark-terasort-1.2-SNAPSHOT.jar diff --git a/test/topology-provider/stack/05-access-hdfs.yaml b/test/topology-provider/stack/05-access-hdfs.yaml index 2c0c69f..72415cf 100644 --- a/test/topology-provider/stack/05-access-hdfs.yaml +++ b/test/topology-provider/stack/05-access-hdfs.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: access-hdfs - image: oci.stackable.tech/sandbox/hadoop:3.4.2-stackable0.0.0-dev + image: oci.stackable.tech/sdp/hadoop:3.5.0-stackable0.0.0-dev imagePullPolicy: IfNotPresent env: - name: HADOOP_CONF_DIR diff --git a/test/topology-provider/teragen/Dockerfile b/test/topology-provider/teragen/Dockerfile index e054981..70dd8aa 100644 --- a/test/topology-provider/teragen/Dockerfile +++ b/test/topology-provider/teragen/Dockerfile @@ -1,9 +1,12 @@ # # cd test/stack/teragen -# docker build . -t oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-terasort +# docker build . -t oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-terasort # -FROM oci.stackable.tech/sdp/spark-k8s:3.5.6-stackable0.0.0-dev +FROM oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev # this .jar is compiled from the code here: https://github.com/ehiggs/spark-terasort +# N.B. 3.5.8 is used as that compiles against scala 2.12, as is the case for the terasort jar. +# For newer spark versions (3.5.8 is marked for removal in 26.11), the tereasort code will need +# to be changed to compile against scala 2.13. COPY --chown=stackable:stackable ./spark-terasort-1.2-SNAPSHOT.jar /tmp/ From 5a6fc00b154967cb0187beafa376e6d4c12bb1f1 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 8 Oct 2026 12:33:57 +0200 Subject: [PATCH 2/3] changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f940189..7a4bc79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Removed + +- Removed support for Hadoop `3.4.2` ([#66]). + +[#66]: https://github.com/stackabletech/hdfs-utils/pull/66 + ## [0.6.0] - 2026-06-09 ### Added From 6a7227410e9f8ce7db8673ca779524434e832df0 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 8 Oct 2026 12:48:56 +0200 Subject: [PATCH 3/3] bumped reviewdog action version --- .github/workflows/reviewdog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reviewdog.yaml b/.github/workflows/reviewdog.yaml index 15312b6..16b1592 100644 --- a/.github/workflows/reviewdog.yaml +++ b/.github/workflows/reviewdog.yaml @@ -57,7 +57,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: reviewdog/action-markdownlint@6f0e0e8703a9e840ffefa37a2775832c51c1c640 # v0.28.0 + - uses: reviewdog/action-markdownlint@34762a7b2dddebf3eb82f7e3a63b9d0242d0a42a # v0.31.4 with: github_token: ${{ secrets.GITHUB_TOKEN }}