Tolerate incompatible versions with different build hash (#128589)

Tolerate incompatible versions with different build hash.
I'm keeping the serverless feature flag to not create warnings there.

Relates to ES-11869

Author: mosche

server/src/main/java/org/elasticsearch/transport/TransportService.java

@@ -685,21 +685,21 @@ public HandshakeResponse(StreamInput in) throws IOException {
                 // message, but recognise that this may fail
                 discoveryNode = new DiscoveryNode(in);
             } catch (Exception e) {
-                maybeThrowOnIncompatibleBuild(null, e);
+                maybeWarnOnIncompatibleBuild(null, e);
                 throw e;
             }
-            maybeThrowOnIncompatibleBuild(discoveryNode, null);
+            maybeWarnOnIncompatibleBuild(discoveryNode, null);
             clusterName = new ClusterName(in);
         }
 
-        private void maybeThrowOnIncompatibleBuild(@Nullable DiscoveryNode node, @Nullable Exception e) {
+        private void maybeWarnOnIncompatibleBuild(@Nullable DiscoveryNode node, @Nullable Exception e) {
             if (SERVERLESS_TRANSPORT_FEATURE_FLAG == false && isIncompatibleBuild(version, buildHash)) {
-                throwOnIncompatibleBuild(node, e);
+                warnOnIncompatibleBuild(node, e);
             }
         }
 
-        private void throwOnIncompatibleBuild(@Nullable DiscoveryNode node, @Nullable Exception e) {
-            throw new IllegalArgumentException(
+        private void warnOnIncompatibleBuild(@Nullable DiscoveryNode node, @Nullable Exception e) {
+            logger.warn(
                 "remote node ["
                     + (node == null ? "unidentifiable" : node)
                     + "] is build ["

server/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java

@@ -9,8 +9,8 @@
 
 package org.elasticsearch.transport;
 
+import org.apache.logging.log4j.Level;
 import org.elasticsearch.Build;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.Version;
@@ -27,6 +27,7 @@
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.MockLog;
 import org.elasticsearch.test.TransportVersionUtils;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -350,23 +351,26 @@ public void testRejectsMismatchedBuildHash() {
             .version(Version.CURRENT.minimumCompatibilityVersion(), IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current())
             .build();
         try (
+            MockLog mockLog = MockLog.capture(TransportService.class);
             Transport.Connection connection = AbstractSimpleTransportTestCase.openConnection(
                 transportServiceA,
                 discoveryNode,
                 TestProfiles.LIGHT_PROFILE
             )
         ) {
-            assertThat(
-                ExceptionsHelper.unwrap(
-                    safeAwaitFailure(
-                        TransportSerializationException.class,
-                        DiscoveryNode.class,
-                        listener -> transportServiceA.handshake(connection, timeout, listener)
-                    ),
-                    IllegalArgumentException.class
-                ).getMessage(),
-                containsString("which has an incompatible wire format")
+            mockLog.addExpectation(
+                new MockLog.SeenEventExpectation(
+                    "message",
+                    TransportService.class.getCanonicalName(),
+                    Level.WARN,
+                    "which has an incompatible wire format"
+                )
             );
+
+            DiscoveryNode connectedNode = safeAwait(listener -> transportServiceA.handshake(connection, timeout, listener));
+            assertNotNull(connectedNode);
+
+            mockLog.awaitAllExpectationsMatched();
         }
         assertFalse(transportServiceA.nodeConnected(discoveryNode));
     }