Skip to content

Commit 486c361

Browse files
nishithakbhaskaranbibith4imjalpreet
committed
Upgrade to Hadoop 3.4.1
Co-authored-by: Shijin K <bibith4@gmail.com> Co-authored-by: Jalpreet Singh Nanda (:imjalpreet) <jalpreetnanda@gmail.com>
1 parent 847acc4 commit 486c361

File tree

17 files changed

+5626
-25
lines changed

17 files changed

+5626
-25
lines changed

‎pom.xml

Lines changed: 72 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
<groupId>com.facebook.presto.hadoop</groupId>
1212
<artifactId>hadoop-apache2</artifactId>
13-
<version>3.2.0-1-SNAPSHOT</version>
13+
<version>3.4.1-1-SNAPSHOT</version>
1414

1515
<name>hadoop-apache2</name>
1616
<description>Shaded version of Apache Hadoop for Presto</description>
@@ -34,21 +34,17 @@
3434
<connection>scm:git:git://github.com/prestodb/presto-hadoop-apache2.git</connection>
3535
<developerConnection>scm:git:git://github.com/prestodb/presto-hadoop-apache2.git</developerConnection>
3636
<url>https://github.com/prestodb/presto-hadoop-apache2</url>
37-
<tag>3.2.0-1</tag>
37+
<tag>HEAD</tag>
3838
</scm>
3939

4040
<properties>
4141
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
4242
<project.build.targetJdk>1.8</project.build.targetJdk>
4343
<shadeBase>com.facebook.presto.hadoop.\$internal</shadeBase>
44-
<dep.slf4j.version>1.7.25</dep.slf4j.version>
45-
<dep.hadoop.version>3.2.0</dep.hadoop.version>
44+
<dep.slf4j.version>2.0.16</dep.slf4j.version>
45+
<dep.hadoop.version>3.4.1</dep.hadoop.version>
4646

4747
<dep.nexus-staging-plugin.version>1.6.8</dep.nexus-staging-plugin.version>
48-
49-
<air.check.skip-dependency>true</air.check.skip-dependency>
50-
<air.check.skip-duplicate-finder>true</air.check.skip-duplicate-finder>
51-
<air.check.skip-modernizer>true</air.check.skip-modernizer>
5248
<air.check.skip-enforcer>true</air.check.skip-enforcer>
5349
<air.check.skip-modernizer>true</air.check.skip-modernizer>
5450
<air.check.skip-duplicate-finder>true</air.check.skip-duplicate-finder>
@@ -71,6 +67,18 @@
7167
<groupId>com.google.code.findbugs</groupId>
7268
<artifactId>jsr305</artifactId>
7369
</exclusion>
70+
<exclusion>
71+
<groupId>com.google.errorprone</groupId>
72+
<artifactId>error_prone_annotations</artifactId>
73+
</exclusion>
74+
<exclusion>
75+
<groupId>com.google.j2objc</groupId>
76+
<artifactId>j2objc-annotations</artifactId>
77+
</exclusion>
78+
<exclusion>
79+
<groupId>org.checkerframework</groupId>
80+
<artifactId>checker-qual</artifactId>
81+
</exclusion>
7482
</exclusions>
7583
</dependency>
7684

@@ -131,10 +139,6 @@
131139
<groupId>org.apache.directory.server</groupId>
132140
<artifactId>apacheds-kerberos-codec</artifactId>
133141
</exclusion>
134-
<exclusion>
135-
<groupId>org.apache.commons</groupId>
136-
<artifactId>commons-compress</artifactId>
137-
</exclusion>
138142
<exclusion>
139143
<groupId>org.apache.commons</groupId>
140144
<artifactId>commons-math</artifactId>
@@ -179,6 +183,10 @@
179183
<groupId>com.sun.jersey</groupId>
180184
<artifactId>jersey-servlet</artifactId>
181185
</exclusion>
186+
<exclusion>
187+
<groupId>com.github.pjfanning</groupId>
188+
<artifactId>jersey-json</artifactId>
189+
</exclusion>
182190
<exclusion>
183191
<groupId>javax.servlet</groupId>
184192
<artifactId>javax.servlet-api</artifactId>
@@ -243,6 +251,30 @@
243251
<groupId>org.slf4j</groupId>
244252
<artifactId>slf4j-log4j12</artifactId>
245253
</exclusion>
254+
<exclusion>
255+
<groupId>org.slf4j</groupId>
256+
<artifactId>slf4j-reload4j</artifactId>
257+
</exclusion>
258+
<exclusion>
259+
<groupId>ch.qos.reload4j</groupId>
260+
<artifactId>reload4j</artifactId>
261+
</exclusion>
262+
<exclusion>
263+
<groupId>org.bouncycastle</groupId>
264+
<artifactId>bcprov-jdk18on</artifactId>
265+
</exclusion>
266+
<exclusion>
267+
<groupId>io.dropwizard.metrics</groupId>
268+
<artifactId>metrics-core</artifactId>
269+
</exclusion>
270+
<exclusion>
271+
<groupId>javax.xml.bind</groupId>
272+
<artifactId>jaxb-api</artifactId>
273+
</exclusion>
274+
<exclusion>
275+
<groupId>jakarta.activation</groupId>
276+
<artifactId>jakarta.activation-api</artifactId>
277+
</exclusion>
246278
</exclusions>
247279
</dependency>
248280

@@ -369,6 +401,12 @@
369401
</exclusions>
370402
</dependency>
371403

404+
<dependency>
405+
<groupId>jakarta.annotation</groupId>
406+
<artifactId>jakarta.annotation-api</artifactId>
407+
<version>2.1.1</version>
408+
</dependency>
409+
372410
<!-- send logging to java.util.logging -->
373411
<dependency>
374412
<groupId>org.slf4j</groupId>
@@ -403,6 +441,20 @@
403441
<version>6.2.1</version>
404442
<scope>test</scope>
405443
</dependency>
444+
445+
<dependency>
446+
<groupId>org.lz4</groupId>
447+
<artifactId>lz4-java</artifactId>
448+
<version>1.8.0</version>
449+
<scope>test</scope>
450+
</dependency>
451+
452+
<dependency>
453+
<groupId>org.xerial.snappy</groupId>
454+
<artifactId>snappy-java</artifactId>
455+
<version>1.1.10.6</version>
456+
<scope>test</scope>
457+
</dependency>
406458
</dependencies>
407459

408460
<build>
@@ -456,6 +508,10 @@
456508
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
457509
</transformers>
458510
<relocations>
511+
<relocation>
512+
<pattern>jakarta.annotation</pattern>
513+
<shadedPattern>${shadeBase}.jakarta.annotation</shadedPattern>
514+
</relocation>
459515
<relocation>
460516
<pattern>org.iq80.leveldb</pattern>
461517
<shadedPattern>${shadeBase}.org.iq80.leveldb</shadedPattern>
@@ -532,6 +588,10 @@
532588
<pattern>org.codehaus.stax2</pattern>
533589
<shadedPattern>${shadeBase}.org.codehaus.stax2</shadedPattern>
534590
</relocation>
591+
<relocation>
592+
<pattern>org.codehaus.jettison</pattern>
593+
<shadedPattern>${shadeBase}.org.codehaus.jettison</shadedPattern>
594+
</relocation>
535595
<relocation>
536596
<pattern>org.apache.log4j</pattern>
537597
<shadedPattern>${shadeBase}.org.apache.log4j</shadedPattern>

‎src/main/java/com/facebook/presto/hadoop/HadoopNative.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@ public static synchronized void requireHadoopNative()
4545
}
4646
try {
4747
loadLibrary("hadoop");
48-
loadLibrary("snappy");
4948
loadLibrary("zstd");
5049
setStatic(NativeCodeLoader.class.getDeclaredField("nativeCodeLoaded"), true);
5150

‎src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
package org.apache.hadoop.crypto.key.kms;
1515

1616
import org.apache.commons.codec.binary.Base64;
17+
import org.apache.commons.lang3.StringUtils;
1718
import org.apache.hadoop.classification.InterfaceAudience;
1819
import org.apache.hadoop.conf.Configuration;
1920
import org.apache.hadoop.crypto.key.KeyProvider;
@@ -75,9 +76,9 @@
7576
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
7677

7778
import com.fasterxml.jackson.databind.ObjectMapper;
78-
import com.google.common.annotations.VisibleForTesting;
79-
import com.google.common.base.Preconditions;
80-
import com.google.common.base.Strings;
79+
import org.apache.hadoop.classification.VisibleForTesting;
80+
import org.apache.hadoop.util.Preconditions;
81+
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
8182

8283
import static org.apache.hadoop.util.KMSUtil.checkNotEmpty;
8384
import static org.apache.hadoop.util.KMSUtil.checkNotNull;
@@ -558,16 +559,17 @@ private <T> T call(HttpURLConnection conn, Object jsonOutput,
558559
throw ex;
559560
}
560561
if ((conn.getResponseCode() == HttpURLConnection.HTTP_FORBIDDEN
562+
&& (!StringUtils.isEmpty(conn.getResponseMessage())
561563
&& (conn.getResponseMessage().equals(ANONYMOUS_REQUESTS_DISALLOWED) ||
562-
conn.getResponseMessage().contains(INVALID_SIGNATURE)))
564+
conn.getResponseMessage().contains(INVALID_SIGNATURE))))
563565
|| conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
564566
// Ideally, this should happen only when there is an Authentication
565567
// failure. Unfortunately, the AuthenticationFilter returns 403 when it
566568
// cannot authenticate (Since a 401 requires Server to send
567569
// WWW-Authenticate header as well)..
568570
if (LOG.isDebugEnabled()) {
569-
LOG.debug("Response={}({}), resetting authToken",
570-
conn.getResponseCode(), conn.getResponseMessage());
571+
LOG.debug("Response={}, resetting authToken",
572+
conn.getResponseCode());
571573
}
572574
KMSClientProvider.this.authToken =
573575
new DelegationTokenAuthenticatedURL.Token();

0 commit comments

Comments
 (0)