Skip to content

Commit 7f30feb

Browse files
lbergelsonlbergelson
committed
Add new argument to allow setting the google requester pays project
* New argument REQUESTER_PAYS_PROJECT added to command line program. This allows configuring the project to use with requester pays buckets in GCS * This can also be configured by setting a new system property: picard.googleProjectForRequesterPays * Updated the initialization of GCS and HTTP filesystem providers
1 parent 63138fc commit 7f30feb

File tree

6 files changed

+182
-58
lines changed

6 files changed

+182
-58
lines changed

‎src/main/java/picard/cmdline/CommandLineProgram.java

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -53,20 +53,20 @@
5353
import org.broadinstitute.barclay.argparser.SpecialArgumentsCollection;
5454
import picard.cmdline.argumentcollections.OptionalReferenceArgumentCollection;
5555
import picard.cmdline.argumentcollections.ReferenceArgumentCollection;
56+
import picard.cmdline.argumentcollections.RequesterPaysArgumentCollection;
5657
import picard.cmdline.argumentcollections.RequiredReferenceArgumentCollection;
57-
import picard.nio.PathProvider;
58+
import picard.nio.GoogleStorageUtils;
59+
import picard.nio.HttpNioUtils;
5860
import picard.nio.PicardHtsPath;
5961
import picard.util.RExecutor;
6062

6163
import java.io.File;
6264
import java.net.InetAddress;
6365
import java.text.DecimalFormat;
6466
import java.util.ArrayList;
65-
import java.util.Arrays;
6667
import java.util.Collections;
6768
import java.util.Date;
6869
import java.util.List;
69-
import java.util.stream.Collectors;
7070

7171
/**
7272
* Abstract class to facilitate writing command-line programs.
@@ -85,8 +85,8 @@
8585
*
8686
*/
8787
public abstract class CommandLineProgram {
88-
private static String PROPERTY_USE_LEGACY_PARSER = "picard.useLegacyParser";
89-
private static String PROPERTY_CONVERT_LEGACY_COMMAND_LINE = "picard.convertCommandLine";
88+
private static final String PROPERTY_USE_LEGACY_PARSER = "picard.useLegacyParser";
89+
private static final String PROPERTY_CONVERT_LEGACY_COMMAND_LINE = "picard.convertCommandLine";
9090
private static Boolean useLegacyParser;
9191
public static String SYNTAX_TRANSITION_URL =
9292
"https://github.com/broadinstitute/picard/wiki/Command-Line-Syntax-Transition-For-Users-(Pre-Transition)";
@@ -132,6 +132,9 @@ public abstract class CommandLineProgram {
132132
// after argument parsing using the value established by the user in the referenceSequence argument collection.
133133
protected File REFERENCE_SEQUENCE = Defaults.REFERENCE_FASTA;
134134

135+
@ArgumentCollection
136+
public RequesterPaysArgumentCollection requesterPays = new RequesterPaysArgumentCollection();
137+
135138
@ArgumentCollection(doc="Special Arguments that have meaning to the argument parsing system. " +
136139
"It is unlikely these will ever need to be accessed by the command line program")
137140
public Object specialArgumentsCollection = useLegacyParser() ?
@@ -182,7 +185,7 @@ public void instanceMainWithExit(final String[] argv) {
182185
}
183186

184187
public int instanceMain(final String[] argv) {
185-
String actualArgs[] = argv;
188+
String[] actualArgs = argv;
186189

187190
if (System.getProperty(PROPERTY_CONVERT_LEGACY_COMMAND_LINE, "false").equals("true")) {
188191
actualArgs = CommandLineSyntaxTranslater.convertPicardStyleToPosixStyle(argv);
@@ -249,16 +252,15 @@ public int instanceMain(final String[] argv) {
249252
// default reader factory. At least until https://github.com/samtools/htsjdk/issues/1666 is resolved
250253
SamReaderFactory.setDefaultValidationStringency(VALIDATION_STRINGENCY);
251254

255+
// Configure the various filesystem providers
256+
GoogleStorageUtils.initialize(requesterPays.getProjectForRequesterPays());
257+
HttpNioUtils.initialize();
258+
252259
if (!QUIET) {
253260
System.err.println("[" + new Date() + "] " + commandLine);
254261

255-
// Output a one liner about who/where and what software/os we're running on
262+
// Output a one-liner about who/where and what software/os we're running on
256263
try {
257-
final String pathProvidersMessage =
258-
Arrays.stream(PathProvider.values())
259-
.map(provider -> String.format("Provider %s is%s available;", provider.name(), provider.isAvailable ? "" : " not"))
260-
.collect(Collectors.joining(" "));
261-
262264
final boolean usingIntelDeflater = (BlockCompressedOutputStream.getDefaultDeflaterFactory() instanceof IntelDeflaterFactory &&
263265
((IntelDeflaterFactory)BlockCompressedOutputStream.getDefaultDeflaterFactory()).usingIntelDeflater());
264266
final boolean usingIntelInflater = (BlockGunzipper.getDefaultInflaterFactory() instanceof IntelInflaterFactory &&
@@ -269,7 +271,7 @@ public int instanceMain(final String[] argv) {
269271
System.getProperty("os.name"), System.getProperty("os.version"), System.getProperty("os.arch"),
270272
System.getProperty("java.vm.name"), System.getProperty("java.runtime.version"),
271273
usingIntelDeflater ? "Intel" : "Jdk", usingIntelInflater ? "Intel" : "Jdk",
272-
pathProvidersMessage,
274+
requesterPays.getDescription(),
273275
getCommandLineParser().getVersion());
274276
System.err.println(msg);
275277
}
@@ -409,7 +411,7 @@ public CommandLineParser getCommandLineParser() {
409411
commandLineParser = useLegacyParser() ?
410412
new LegacyCommandLineArgumentParser(this) :
411413
new CommandLineArgumentParser(this,
412-
Collections.EMPTY_LIST,
414+
Collections.emptyList(),
413415
Collections.singleton(CommandLineParserOptions.APPEND_TO_COLLECTIONS));
414416
}
415417
return commandLineParser;
@@ -427,9 +429,7 @@ public CommandLineParser getCommandLineParser() {
427429
public static boolean useLegacyParser() {
428430
if (useLegacyParser == null) {
429431
final String legacyPropertyValue = System.getProperty(PROPERTY_USE_LEGACY_PARSER);
430-
useLegacyParser = legacyPropertyValue == null ?
431-
false :
432-
Boolean.parseBoolean(legacyPropertyValue);
432+
useLegacyParser = Boolean.parseBoolean(legacyPropertyValue);
433433
}
434434
return useLegacyParser;
435435
}
@@ -475,7 +475,7 @@ public List<Header> getDefaultHeaders() {
475475
public static String getStandardUsagePreamble(final Class<?> mainClass) {
476476
return "USAGE: " + mainClass.getSimpleName() +" [options]\n\n" +
477477
(hasWebDocumentation(mainClass) ?
478-
"Documentation: http://broadinstitute.github.io/picard/command-line-overview.html" +
478+
"Documentation: https://broadinstitute.github.io/picard/command-line-overview.html" +
479479
mainClass.getSimpleName() + "\n\n" :
480480
"");
481481
}
@@ -499,7 +499,7 @@ public static boolean hasWebDocumentation(final Class<?> clazz){
499499
* @return the link to a FAQ
500500
*/
501501
public static String getFaqLink() {
502-
return "To get help, see http://broadinstitute.github.io/picard/index.html#GettingHelp";
502+
return "To get help, see https://broadinstitute.github.io/picard/index.html#GettingHelp";
503503
}
504504

505505
/**

‎src/main/java/picard/cmdline/argumentcollections/RequesterPaysArgumentCollection.java

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package picard.cmdline.argumentcollections;
2+
3+
import com.google.common.base.Strings;
4+
import org.broadinstitute.barclay.argparser.Argument;
5+
6+
/**
7+
* Argument collection to encapsulate special behavior of the REQUESTER_PAYS_PROJECT argument
8+
*/
9+
public class RequesterPaysArgumentCollection {
10+
/** The System property which acts as a default for REQUESTER_PAYS_PROJECT */
11+
public static final String PROPERTY_NAME = "picard.googleProjectForRequesterPays";
12+
public static final String REQUESTER_PAYS_PROJECT_FULL_NAME = "REQUESTER_PAYS_PROJECT";
13+
14+
@Argument(doc="Google project for access to 'requester pays' buckets and objects. " +
15+
"If this is not specified then value of the system property " + PROPERTY_NAME + " acts as the default.",
16+
common = true, optional = true,
17+
fullName = REQUESTER_PAYS_PROJECT_FULL_NAME)
18+
public String requesterPaysProject = null;
19+
20+
public String getProjectForRequesterPays() {
21+
final String value = ! Strings.isNullOrEmpty(requesterPaysProject)
22+
? requesterPaysProject
23+
: getSystemProperty();
24+
return Strings.isNullOrEmpty(value) ? null : value; // "" -> null
25+
}
26+
27+
private String getSystemProperty() {
28+
return System.getProperty(PROPERTY_NAME);
29+
}
30+
31+
public String getDescription(){
32+
final String value = getProjectForRequesterPays();
33+
if(!Strings.isNullOrEmpty(requesterPaysProject)){
34+
return "Requester Pays Project set by argument: " + value;
35+
} else if( !Strings.isNullOrEmpty(getSystemProperty())){
36+
return "Requester Pays Project set by system property: " + value;
37+
} else {
38+
return "Requester Pays Project not set.";
39+
}
40+
}
41+
42+
}

‎src/main/java/picard/nio/GoogleStorageUtils.java

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,22 +36,26 @@
3636
/**
3737
* This class serves as a connection to google's implementation of nio support for GCS housed files.
3838
*
39-
* While the actual code required to connect isn't packaged with Picard (only compiled), the Readme.md file in the
40-
* github repository describes how it can be used. Additionally, Picard is bundled in GATK4, and its tools exposed via
41-
* the GATK engine, since the nio library _is_ included the GATK4 jar. NIO enabled tools in picard can connect to
42-
* GCS when called through GATK4.
43-
*
4439
* This class contains hard-coded setting that have been found to work for the access patterns that characterize genomics
4540
* work. In the future it would make sense to expose these parameters so that they can be controlled via the commandline.
4641
* However, as the list of Path-enabled tools in picard is small, there seems to be little impetus to do so right now.
4742
*
4843
*
4944
*/
50-
class GoogleStorageUtils {
45+
public final class GoogleStorageUtils {
46+
47+
public static final int MAX_REOPENS = 20;
48+
49+
private GoogleStorageUtils(){}
5150

52-
public static void initialize() {
51+
/**
52+
* Set appropriate configuration options for the GCS file system provider.
53+
*
54+
* @param requesterProject the project to pay with when accessing requester pays buckets
55+
*/
56+
public static void initialize(final String requesterProject) {
5357
// requester pays support is currently not configured
54-
CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(GoogleStorageUtils.getCloudStorageConfiguration(20, null));
58+
CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(GoogleStorageUtils.getCloudStorageConfiguration(MAX_REOPENS, requesterProject));
5559
CloudStorageFileSystemProvider.setStorageOptions(GoogleStorageUtils.setGenerousTimeouts(StorageOptions.newBuilder()).build());
5660
}
5761

@@ -62,7 +66,8 @@ private static CloudStorageConfiguration getCloudStorageConfiguration(int maxReo
6266
.maxChannelReopens(maxReopens);
6367
if (!Strings.isNullOrEmpty(requesterProject)) {
6468
// enable requester pays and indicate who pays
65-
builder = builder.autoDetectRequesterPays(true).userProject(requesterProject);
69+
builder.autoDetectRequesterPays(true)
70+
.userProject(requesterProject);
6671
}
6772

6873
// this causes the gcs filesystem to treat files that end in a / as a directory

‎src/main/java/picard/nio/HttpNioUtils.java

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package picard.nio;
2+
3+
import org.broadinstitute.http.nio.HttpFileSystemProvider;
4+
import org.broadinstitute.http.nio.HttpFileSystemProviderSettings;
5+
import org.broadinstitute.http.nio.RetryHandler;
6+
7+
import java.time.Duration;
8+
9+
/**
10+
* This class provides a way to easily configure the HttpNioProvider
11+
*
12+
* This class contains hard-coded setting that have been found to work for the access patterns that characterize genomics
13+
* work.
14+
*
15+
*/
16+
public final class HttpNioUtils {
17+
18+
private HttpNioUtils() {}
19+
public static final Duration MAX_TIMEOUT = Duration.ofMillis(120_000);
20+
public static final int MAX_RETRIES = 20;
21+
22+
public static void initialize() {
23+
final HttpFileSystemProviderSettings.RetrySettings retrySettings = new HttpFileSystemProviderSettings.RetrySettings(
24+
MAX_RETRIES,
25+
RetryHandler.DEFAULT_RETRYABLE_HTTP_CODES,
26+
RetryHandler.DEFAULT_RETRYABLE_EXCEPTIONS,
27+
RetryHandler.DEFALT_RETRYABLE_MESSAGES,
28+
e -> false);
29+
30+
final HttpFileSystemProviderSettings settings = new HttpFileSystemProviderSettings(
31+
MAX_TIMEOUT,
32+
HttpFileSystemProviderSettings.DEFAULT_SETTINGS.redirect(),
33+
retrySettings);
34+
35+
HttpFileSystemProvider.setSettings(settings);
36+
}
37+
}

‎src/main/java/picard/nio/PathProvider.java

Lines changed: 0 additions & 30 deletions
This file was deleted.

‎src/test/java/picard/cmdline/argumentcollections/RequesterPaysArgumentCollectionTest.java

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package picard.cmdline.argumentcollections;
2+
3+
import org.testng.Assert;
4+
import org.testng.annotations.DataProvider;
5+
import org.testng.annotations.Test;
6+
7+
public class RequesterPaysArgumentCollectionTest {
8+
9+
public static final String P1 = "project1";
10+
public static final String P2 = "project2";
11+
12+
@DataProvider
13+
public static Object[][] settings() {
14+
15+
return new Object[][]{
16+
{null, null, null},
17+
{"", null, null},
18+
{null, "", null},
19+
{"", "", null},
20+
{P1, null, P1},
21+
{null, P2, P2},
22+
{P1, P2, P1},
23+
{"", P2, P2},
24+
{P1, "", P1}
25+
};
26+
}
27+
28+
@Test(dataProvider = "settings")
29+
public void testCorrectValues(String arg, String sys, String expected){
30+
runWithSystemProperty(
31+
() -> {
32+
final RequesterPaysArgumentCollection rpc = new RequesterPaysArgumentCollection();
33+
rpc.requesterPaysProject = arg;
34+
Assert.assertEquals(rpc.getProjectForRequesterPays(), expected);
35+
}, RequesterPaysArgumentCollection.PROPERTY_NAME, sys
36+
);
37+
}
38+
39+
@Test(dataProvider = "settings")
40+
public void testCorrectDescription(String arg, String sys, String expected){
41+
runWithSystemProperty(
42+
() -> {
43+
final RequesterPaysArgumentCollection rpc = new RequesterPaysArgumentCollection();
44+
rpc.requesterPaysProject = arg;
45+
final String description = rpc.getDescription();
46+
final String value = rpc.getProjectForRequesterPays();
47+
if(expected == null) {Assert.assertEquals(description, "Requester Pays Project not set."); }
48+
else if(expected.equals(P1)) { Assert.assertEquals(description, "Requester Pays Project set by argument: " + value); }
49+
else if(expected.equals(P2)) { Assert.assertEquals(description, "Requester Pays Project set by system property: " + value); }
50+
else { Assert.fail("should have been one of the of the previous"); }
51+
}, RequesterPaysArgumentCollection.PROPERTY_NAME, sys);
52+
}
53+
private static void runWithSystemProperty(Runnable toRun, String name, String value){
54+
String previousValue = null;
55+
try {
56+
if(value != null) {
57+
previousValue = System.setProperty(name, value);
58+
}
59+
60+
toRun.run();
61+
62+
} finally {
63+
if(previousValue == null){
64+
System.clearProperty(name);
65+
} else {
66+
System.setProperty(name, previousValue);
67+
}
68+
}
69+
}
70+
}

0 commit comments

Comments
 (0)