fix(hadoop): Backport HADOOP-18583 & fix OpenSSL native library

CHANGELOG.md

@@ -59,6 +59,7 @@ All notable changes to this project will be documented in this file.
 - zookeeper: bump jetty version for CVE-2024-13009 in 3.9.3 ([#1179])
 - zookeeper: bump netty version for CVE-2025-24970 in 3.9.3 ([#1180])
 - hadoop: backport HADOOP-19352, HADOOP-19335, HADOOP-19465, HADOOP-19456 and HADOOP-19225 to fix vulnerabilities in Hadoop `3.4.1` ([#1184])
+- hadoop: Backport HADOOP-18583 to make OpenSSL 3.x work with the native hadoop libraries ([#1209]).
 
 ### Changed
 
@@ -221,6 +222,7 @@ All notable changes to this project will be documented in this file.
 [#1188]: https://github.com/stackabletech/docker-images/pull/1188
 [#1189]: https://github.com/stackabletech/docker-images/pull/1189
 [#1197]: https://github.com/stackabletech/docker-images/pull/1197
+[#1209]: https://github.com/stackabletech/docker-images/pull/1209
 
 ## [25.3.0] - 2025-03-21
 

hadoop/Dockerfile

@@ -136,6 +136,15 @@ ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/j
 # Set correct permissions and ownerships
 chown --recursive ${STACKABLE_USER_UID}:0 /stackable/hadoop /stackable/jmx /stackable/async-profiler "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}"
 chmod --recursive g=u /stackable/jmx /stackable/async-profiler "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE}"
+
+# Workaround for https://issues.apache.org/jira/browse/HADOOP-12845
+# The problem is that our stackable-devel image does contain the openssl-devel package
+# That package creates a symlink from /usr/lib/libcrypto.so to the real libcrypto
+# The non -devel package, which is used in this image, does NOT create this symlink.
+# That's why the Hadoop build works even with the 'require.openssl' flag but in the production
+# image the 'hadoop checknative' tool still fails because it can't find the 'libcrypto.so' symlink.
+# Therefore we create this symlink here.
+ln -s /usr/lib64/libcrypto.so.3 /usr/lib64/libcrypto.so
 EOF
 
 RUN <<EOF

hadoop/hadoop/Dockerfile

@@ -69,6 +69,12 @@ sed -e '/<artifactId>hadoop-pipes<\/artifactId>/,/<\/dependency>/ { s/<version>.
 # Create snapshot of the source code including custom patches
 tar -czf /stackable/hadoop-${NEW_VERSION}-src.tar.gz .
 
+# We do not pass require.snappy because that is only built in to the MapReduce client and we don't need that
+#
+# Passing require.openssl SHOULD make the build fail if OpenSSL is not present.
+# This does not work properly however because this builder image contains the openssl-devel package which creates a symlink from /usr/lib64/libcrypto.so to the real version.
+# Therefore, this build does work but the final image does NOT contain the openssl-devel package which is why it fails there which is why we have to create the symlink over there manually.
+# We still leave this flag in to automatically fail should anything with the packages or symlinks ever fail.
 mvn \
     --batch-mode \
     --no-transfer-progress \
@@ -77,6 +83,7 @@ mvn \
     -pl '!hadoop-tools/hadoop-pipes' \
     -Dhadoop.version=${NEW_VERSION} \
     -Drequire.fuse=true \
+    -Drequire.openssl=true \
     -DskipTests \
     -Dmaven.javadoc.skip=true
 

hadoop/hadoop/stackable/patches/3.3.6/0012-HADOOP-18583.-Fix-loading-of-OpenSSL-3.x-symbols-525.patch

@@ -0,0 +1,115 @@
+From baa7ec826f3f6d044f5307efe4b5d3bdd111bf4e Mon Sep 17 00:00:00 2001
+From: Sebastian Klemke <[email protected]>
+Date: Thu, 7 Nov 2024 19:14:13 +0100
+Subject: HADOOP-18583. Fix loading of OpenSSL 3.x symbols (#5256) (#7149)
+
+Contributed by Sebastian Klemke
+---
+ .../org/apache/hadoop/crypto/OpensslCipher.c  | 68 +++++++++++++++++--
+ 1 file changed, 64 insertions(+), 4 deletions(-)
+
+diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
+index abff7ea5f1..f17169dec2 100644
+--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
+@@ -24,6 +24,57 @@
+
+ #include "org_apache_hadoop_crypto_OpensslCipher.h"
+
++/*
++   # OpenSSL ABI Symbols
++
++   Available on all OpenSSL versions:
++
++   | Function                       | 1.0 | 1.1 | 3.0 |
++   |--------------------------------|-----|-----|-----|
++   | EVP_CIPHER_CTX_new             | YES | YES | YES |
++   | EVP_CIPHER_CTX_free            | YES | YES | YES |
++   | EVP_CIPHER_CTX_set_padding     | YES | YES | YES |
++   | EVP_CIPHER_CTX_test_flags      | YES | YES | YES |
++   | EVP_CipherInit_ex              | YES | YES | YES |
++   | EVP_CipherUpdate               | YES | YES | YES |
++   | EVP_CipherFinal_ex             | YES | YES | YES |
++   | ENGINE_by_id                   | YES | YES | YES |
++   | ENGINE_free                    | YES | YES | YES |
++   | EVP_aes_256_ctr                | YES | YES | YES |
++   | EVP_aes_128_ctr                | YES | YES | YES |
++
++   Available on old versions:
++
++   | Function                       | 1.0 | 1.1 | 3.0 |
++   |--------------------------------|-----|-----|-----|
++   | EVP_CIPHER_CTX_cleanup         | YES | --- | --- |
++   | EVP_CIPHER_CTX_init            | YES | --- | --- |
++   | EVP_CIPHER_CTX_block_size      | YES | YES | --- |
++   | EVP_CIPHER_CTX_encrypting      | --- | YES | --- |
++
++   Available on new versions:
++
++   | Function                       | 1.0 | 1.1 | 3.0 |
++   |--------------------------------|-----|-----|-----|
++   | OPENSSL_init_crypto            | --- | YES | YES |
++   | EVP_CIPHER_CTX_reset           | --- | YES | YES |
++   | EVP_CIPHER_CTX_get_block_size  | --- | --- | YES |
++   | EVP_CIPHER_CTX_is_encrypting   | --- | --- | YES |
++
++   Optionally available on new versions:
++
++   | Function                       | 1.0 | 1.1 | 3.0 |
++   |--------------------------------|-----|-----|-----|
++   | EVP_sm4_ctr                    | --- | opt | opt |
++
++   Name changes:
++
++   | < 3.0 name                 | >= 3.0 name                    |
++   |----------------------------|--------------------------------|
++   | EVP_CIPHER_CTX_block_size  | EVP_CIPHER_CTX_get_block_size  |
++   | EVP_CIPHER_CTX_encrypting  | EVP_CIPHER_CTX_is_encrypting   |
++ */
++
+ #ifdef UNIX
+ static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void);
+ static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
+@@ -87,6 +138,15 @@ static __dlsym_EVP_aes_128_ctr dlsym_EVP_aes_128_ctr;
+ static HMODULE openssl;
+ #endif
+
++// names changed in OpenSSL 3 ABI - see History section in EVP_EncryptInit(3)
++#if OPENSSL_VERSION_NUMBER >= 0x30000000L
++#define CIPHER_CTX_BLOCK_SIZE "EVP_CIPHER_CTX_get_block_size"
++#define CIPHER_CTX_ENCRYPTING "EVP_CIPHER_CTX_is_encrypting"
++#else
++#define CIPHER_CTX_BLOCK_SIZE "EVP_CIPHER_CTX_block_size"
++#define CIPHER_CTX_ENCRYPTING "EVP_CIPHER_CTX_encrypting"
++#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */
++
+ static void loadAesCtr(JNIEnv *env)
+ {
+ #ifdef UNIX
+@@ -142,10 +202,10 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
+   LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl,  \
+                       "EVP_CIPHER_CTX_test_flags");
+   LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_block_size, env, openssl,  \
+-                      "EVP_CIPHER_CTX_block_size");
++                      CIPHER_CTX_BLOCK_SIZE);
+ #if OPENSSL_VERSION_NUMBER >= 0x10100000L
+   LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_encrypting, env, openssl,  \
+-                      "EVP_CIPHER_CTX_encrypting");
++                      CIPHER_CTX_ENCRYPTING);
+ #endif
+   LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherInit_ex, env, openssl,  \
+                       "EVP_CipherInit_ex");
+@@ -173,11 +233,11 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
+                       openssl, "EVP_CIPHER_CTX_test_flags");
+   LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_block_size,  \
+                       dlsym_EVP_CIPHER_CTX_block_size, env,  \
+-                      openssl, "EVP_CIPHER_CTX_block_size");
++                      openssl, CIPHER_CTX_BLOCK_SIZE);
+ #if OPENSSL_VERSION_NUMBER >= 0x10100000L
+   LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_encrypting,  \
+                       dlsym_EVP_CIPHER_CTX_encrypting, env,  \
+-                      openssl, "EVP_CIPHER_CTX_encrypting");
++                      openssl, CIPHER_CTX_ENCRYPTING);
+ #endif
+   LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherInit_ex, dlsym_EVP_CipherInit_ex,  \
+                       env, openssl, "EVP_CipherInit_ex");

hadoop/hadoop/stackable/patches/3.4.1/0011-HADOOP-18583.-Fix-loading-of-OpenSSL-3.x-symbols-525.patch

@@ -0,0 +1,115 @@
+From cd1c23ea5bddd2796caf2590fef467e488c3bcbf Mon Sep 17 00:00:00 2001
+From: Sebastian Klemke <[email protected]>
+Date: Thu, 7 Nov 2024 19:14:13 +0100
+Subject: HADOOP-18583. Fix loading of OpenSSL 3.x symbols  (#5256) (#7149)
+
+Contributed by Sebastian Klemke
+---
+ .../org/apache/hadoop/crypto/OpensslCipher.c  | 68 +++++++++++++++++--
+ 1 file changed, 64 insertions(+), 4 deletions(-)
+
+diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
+index 976bf135ce..33be4a394f 100644
+--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
+@@ -24,6 +24,57 @@
+
+ #include "org_apache_hadoop_crypto_OpensslCipher.h"
+
++/*
++   # OpenSSL ABI Symbols
++
++   Available on all OpenSSL versions:
++
++   | Function                       | 1.0 | 1.1 | 3.0 |
++   |--------------------------------|-----|-----|-----|
++   | EVP_CIPHER_CTX_new             | YES | YES | YES |
++   | EVP_CIPHER_CTX_free            | YES | YES | YES |
++   | EVP_CIPHER_CTX_set_padding     | YES | YES | YES |
++   | EVP_CIPHER_CTX_test_flags      | YES | YES | YES |
++   | EVP_CipherInit_ex              | YES | YES | YES |
++   | EVP_CipherUpdate               | YES | YES | YES |
++   | EVP_CipherFinal_ex             | YES | YES | YES |
++   | ENGINE_by_id                   | YES | YES | YES |
++   | ENGINE_free                    | YES | YES | YES |
++   | EVP_aes_256_ctr                | YES | YES | YES |
++   | EVP_aes_128_ctr                | YES | YES | YES |
++
++   Available on old versions:
++
++   | Function                       | 1.0 | 1.1 | 3.0 |
++   |--------------------------------|-----|-----|-----|
++   | EVP_CIPHER_CTX_cleanup         | YES | --- | --- |
++   | EVP_CIPHER_CTX_init            | YES | --- | --- |
++   | EVP_CIPHER_CTX_block_size      | YES | YES | --- |
++   | EVP_CIPHER_CTX_encrypting      | --- | YES | --- |
++
++   Available on new versions:
++
++   | Function                       | 1.0 | 1.1 | 3.0 |
++   |--------------------------------|-----|-----|-----|
++   | OPENSSL_init_crypto            | --- | YES | YES |
++   | EVP_CIPHER_CTX_reset           | --- | YES | YES |
++   | EVP_CIPHER_CTX_get_block_size  | --- | --- | YES |
++   | EVP_CIPHER_CTX_is_encrypting   | --- | --- | YES |
++
++   Optionally available on new versions:
++
++   | Function                       | 1.0 | 1.1 | 3.0 |
++   |--------------------------------|-----|-----|-----|
++   | EVP_sm4_ctr                    | --- | opt | opt |
++
++   Name changes:
++
++   | < 3.0 name                 | >= 3.0 name                    |
++   |----------------------------|--------------------------------|
++   | EVP_CIPHER_CTX_block_size  | EVP_CIPHER_CTX_get_block_size  |
++   | EVP_CIPHER_CTX_encrypting  | EVP_CIPHER_CTX_is_encrypting   |
++ */
++
+ #ifdef UNIX
+ static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void);
+ static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
+@@ -106,6 +157,15 @@ static __dlsym_ENGINE_free dlsym_ENGINE_free;
+ static HMODULE openssl;
+ #endif
+
++// names changed in OpenSSL 3 ABI - see History section in EVP_EncryptInit(3)
++#if OPENSSL_VERSION_NUMBER >= 0x30000000L
++#define CIPHER_CTX_BLOCK_SIZE "EVP_CIPHER_CTX_get_block_size"
++#define CIPHER_CTX_ENCRYPTING "EVP_CIPHER_CTX_is_encrypting"
++#else
++#define CIPHER_CTX_BLOCK_SIZE "EVP_CIPHER_CTX_block_size"
++#define CIPHER_CTX_ENCRYPTING "EVP_CIPHER_CTX_encrypting"
++#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */
++
+ static void loadAesCtr(JNIEnv *env)
+ {
+ #ifdef UNIX
+@@ -170,10 +230,10 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
+   LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl,  \
+                       "EVP_CIPHER_CTX_test_flags");
+   LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_block_size, env, openssl,  \
+-                      "EVP_CIPHER_CTX_block_size");
++                      CIPHER_CTX_BLOCK_SIZE);
+ #if OPENSSL_VERSION_NUMBER >= 0x10100000L
+   LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_encrypting, env, openssl,  \
+-                      "EVP_CIPHER_CTX_encrypting");
++                      CIPHER_CTX_ENCRYPTING);
+ #endif
+   LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherInit_ex, env, openssl,  \
+                       "EVP_CipherInit_ex");
+@@ -209,11 +269,11 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
+                       openssl, "EVP_CIPHER_CTX_test_flags");
+   LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_block_size,  \
+                       dlsym_EVP_CIPHER_CTX_block_size, env,  \
+-                      openssl, "EVP_CIPHER_CTX_block_size");
++                      openssl, CIPHER_CTX_BLOCK_SIZE);
+ #if OPENSSL_VERSION_NUMBER >= 0x10100000L
+   LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_encrypting,  \
+                       dlsym_EVP_CIPHER_CTX_encrypting, env,  \
+-                      openssl, "EVP_CIPHER_CTX_encrypting");
++                      openssl, CIPHER_CTX_ENCRYPTING);
+ #endif
+   LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherInit_ex, dlsym_EVP_CipherInit_ex,  \
+                       env, openssl, "EVP_CipherInit_ex");