From bbd95dd11f2ccb7a09a56cd4b2344891b8bd1787 Mon Sep 17 00:00:00 2001 From: amaruthappan Date: Mon, 3 Jul 2023 14:28:51 -0700 Subject: [PATCH] PIG-5442: Add only credentials from setStoreLocation to the Job Conf --- .../executionengine/tez/TezDagBuilder.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java b/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java index eb47b71b57..b22fd96982 100644 --- a/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java +++ b/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java @@ -1077,9 +1077,7 @@ private LinkedList processStores(TezOperator tezOp, ArrayList storeLocations = new ArrayList(); for (POStore st : stores) { storeLocations.add(st); - StoreFuncInterface sFunc = st.getStoreFunc(); - sFunc.setStoreLocation(st.getSFile().getFileName(), job); - sFunc.addCredentials(job.getCredentials(), job.getConfiguration()); + addCredentials(job, st); } Path tmpLocation = null; @@ -1135,6 +1133,19 @@ private LinkedList processStores(TezOperator tezOp, return stores; } + private static void addCredentials(Job job, POStore st) throws IOException { + StoreFuncInterface sFunc = st.getStoreFunc(); + sFunc.addCredentials(job.getCredentials(), job.getConfiguration()); + + // Backward compatibility - before addCredentials API was introduced, credentials were set in setStoreLocation method + Job storeJob = new Job(job.getConfiguration()); + sFunc.setStoreLocation(st.getSFile().getFileName(), storeJob); + // Should always be the same object - Just future proofing + if (job.getCredentials() != storeJob.getCredentials()) { + job.getCredentials().mergeAll(storeJob.getCredentials()); + } + } + @SuppressWarnings("rawtypes") private void setIntermediateOutputKeyValue(byte keyType, Configuration conf, TezOperator tezOp, boolean isConnectedToPackage, boolean isMergedInput) throws JobCreationException, ExecException {