Skip to content

Commit fa0c569

Browse files
Integrate logs with stackdriver
1 parent b123027 commit fa0c569

File tree

20 files changed

+1019
-3
lines changed

20 files changed

+1019
-3
lines changed
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Copyright © 2024 Cask Data, Inc.
4+
5+
Licensed under the Apache License, Version 2.0 (the "License"); you may not
6+
use this file except in compliance with the License. You may obtain a copy of
7+
the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
License for the specific language governing permissions and limitations under
15+
the License.
16+
-->
17+
18+
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xmlns="http://maven.apache.org/POM/4.0.0"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<artifactId>cdap-cloud-log-appender-ext-stackdriver</artifactId>
22+
<dependencies>
23+
<dependency>
24+
<artifactId>google-cloud-logging</artifactId>
25+
<groupId>com.google.cloud</groupId>
26+
</dependency>
27+
<dependency>
28+
<groupId>io.cdap.cdap</groupId>
29+
<artifactId>cdap-log-appender-spi</artifactId>
30+
<version>${project.version}</version>
31+
</dependency>
32+
<dependency>
33+
<groupId>com.google.inject</groupId>
34+
<artifactId>guice</artifactId>
35+
</dependency>
36+
</dependencies>
37+
<dependencyManagement>
38+
<dependencies>
39+
<dependency>
40+
<artifactId>libraries-bom</artifactId>
41+
<groupId>com.google.cloud</groupId>
42+
<scope>import</scope>
43+
<type>pom</type>
44+
<version>26.42.0</version>
45+
</dependency>
46+
<dependency>
47+
<artifactId>guava</artifactId>
48+
<groupId>com.google.guava</groupId>
49+
<version>31.1-jre</version>
50+
</dependency>
51+
</dependencies>
52+
</dependencyManagement>
53+
54+
55+
<modelVersion>4.0.0</modelVersion>
56+
<name>CDAP Cloud Log Appender Extension Stackdriver</name>
57+
<packaging>jar</packaging>
58+
<parent>
59+
<artifactId>cdap</artifactId>
60+
<groupId>io.cdap.cdap</groupId>
61+
<version>6.11.0-SNAPSHOT</version>
62+
</parent>
63+
<profiles>
64+
<profile>
65+
<build>
66+
<plugins>
67+
<plugin>
68+
<artifactId>maven-dependency-plugin</artifactId>
69+
<executions>
70+
<execution>
71+
<configuration combine.self="override">
72+
<includeScope>runtime</includeScope>
73+
<outputDirectory>${project.build.directory}/libexec</outputDirectory>
74+
<overWriteIfNewer>true</overWriteIfNewer>
75+
<overWriteReleases>false</overWriteReleases>
76+
<overWriteSnapshots>false</overWriteSnapshots>
77+
<prependGroupId>true</prependGroupId>
78+
<silent>true</silent>
79+
</configuration>
80+
<goals>
81+
<goal>copy-dependencies</goal>
82+
</goals>
83+
<id>copy-dependencies</id>
84+
<phase>prepare-package</phase>
85+
</execution>
86+
</executions>
87+
<groupId>org.apache.maven.plugins</groupId>
88+
<version>2.8</version>
89+
</plugin>
90+
<plugin>
91+
<artifactId>maven-jar-plugin</artifactId>
92+
<executions>
93+
<execution>
94+
<configuration combine.self="override">
95+
<finalName>${project.groupId}.${project.build.finalName}</finalName>
96+
<outputDirectory>${project.build.directory}/libexec</outputDirectory>
97+
</configuration>
98+
<goals>
99+
<goal>jar</goal>
100+
</goals>
101+
<id>jar</id>
102+
<phase>prepare-package</phase>
103+
</execution>
104+
</executions>
105+
<groupId>org.apache.maven.plugins</groupId>
106+
<version>2.4</version>
107+
</plugin>
108+
</plugins>
109+
</build>
110+
<id>dist</id>
111+
</profile>
112+
</profiles>
113+
114+
<properties>
115+
<guava.version>31.1-jre</guava.version>
116+
</properties>
117+
</project>
Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
/*
2+
* Copyright © 2024 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.cdap.stackdriver.logs;
18+
19+
import com.google.api.client.util.ExponentialBackOff;
20+
import com.google.api.client.util.GenericData;
21+
import com.google.auth.oauth2.AccessToken;
22+
import com.google.auth.oauth2.GoogleCredentials;
23+
import com.google.common.io.CharStreams;
24+
import com.google.gson.Gson;
25+
import java.io.IOException;
26+
import java.io.InputStreamReader;
27+
import java.io.Reader;
28+
import java.net.HttpURLConnection;
29+
import java.net.URL;
30+
import java.nio.charset.StandardCharsets;
31+
import java.security.SecureRandom;
32+
import java.security.cert.X509Certificate;
33+
import java.util.Date;
34+
import java.util.concurrent.ConcurrentHashMap;
35+
import javax.annotation.Nullable;
36+
import javax.net.ssl.HttpsURLConnection;
37+
import javax.net.ssl.SSLContext;
38+
import javax.net.ssl.TrustManager;
39+
import javax.net.ssl.X509TrustManager;
40+
41+
/**
42+
* Provides ComputeEngineCredentials either locally if no endpoint is provided, or remotely if
43+
* endpoint is provided.
44+
* <p>
45+
* This class is copied from <a href="https://github.com/cdapio/cdap"> CDAP repo </a> . Copying the
46+
* class keeps the dependencies for the extension simpler.
47+
*/
48+
public final class ComputeEngineCredentials extends GoogleCredentials {
49+
50+
// private static final Logger LOG = LoggerFactory.getLogger(ComputeEngineCredentials.class);
51+
private static final Gson GSON = new Gson();
52+
private static final String ACCESS_TOKEN_KEY = "access_token";
53+
private static final String EXPIRES_IN_KEY = "expires_in";
54+
private static final String LOCAL_COMPUTE_ENGINE_CREDENTIALS = "local";
55+
private static final ConcurrentHashMap<String, ComputeEngineCredentials> cachedComputeEngineCredentials =
56+
new ConcurrentHashMap<>();
57+
/**
58+
* Time (in millisecond) to refresh the credentials before it expires.
59+
*/
60+
private static final int NUMBER_OF_RETRIES = 20;
61+
private static final int MIN_WAIT_TIME_MILLISECOND = 500;
62+
private static final int MAX_WAIT_TIME_MILLISECOND = 10000;
63+
private static final SecureRandom SECURE_RANDOM = new SecureRandom();
64+
private final String endPoint;
65+
66+
private ComputeEngineCredentials(@Nullable String endPoint) {
67+
this.endPoint = endPoint;
68+
}
69+
70+
/**
71+
* Return a ComputeEngineCredentials with the provided endpoint if it has already been created.
72+
* Otherwise, it instantiates one, and returns it.
73+
*
74+
* @param endpoint endpoint for fetching the token from. A null endpoint results in fetching the
75+
* token locally.
76+
* @return ComputeEngineCredentials
77+
*/
78+
public static ComputeEngineCredentials getOrCreate(@Nullable String endpoint) throws IOException {
79+
String key = endpoint != null ? endpoint : LOCAL_COMPUTE_ENGINE_CREDENTIALS;
80+
// LOG.debug("Using token endpoint {}.", key);
81+
if (!cachedComputeEngineCredentials.containsKey(key)) {
82+
synchronized (cachedComputeEngineCredentials) {
83+
if (!cachedComputeEngineCredentials.containsKey(key)) {
84+
ComputeEngineCredentials credentials = new ComputeEngineCredentials(endpoint);
85+
credentials.refresh();
86+
cachedComputeEngineCredentials.put(key, credentials);
87+
}
88+
}
89+
}
90+
return cachedComputeEngineCredentials.get(key);
91+
}
92+
93+
private AccessToken getAccessTokenLocally() throws IOException {
94+
try {
95+
GoogleCredentials googleCredentials = com.google.auth.oauth2.ComputeEngineCredentials.create();
96+
return googleCredentials.refreshAccessToken();
97+
} catch (IOException e) {
98+
throw new IOException("Unable to get credentials from the environment. "
99+
+ "Please explicitly set the account key.", e);
100+
}
101+
}
102+
103+
private void disableVerifySSL(HttpsURLConnection connection) throws IOException {
104+
try {
105+
SSLContext sslContextWithNoVerify = SSLContext.getInstance("SSL");
106+
TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() {
107+
public X509Certificate[] getAcceptedIssuers() {
108+
return null;
109+
}
110+
111+
@Override
112+
public void checkClientTrusted(X509Certificate[] arg0, String arg1) {
113+
// No-op
114+
}
115+
116+
@Override
117+
public void checkServerTrusted(X509Certificate[] arg0, String arg1) {
118+
// No-op
119+
}
120+
}};
121+
sslContextWithNoVerify.init(null, trustAllCerts, SECURE_RANDOM);
122+
connection.setSSLSocketFactory(sslContextWithNoVerify.getSocketFactory());
123+
connection.setHostnameVerifier((s, sslSession) -> true);
124+
} catch (Exception e) {
125+
// LOG.error("Unable to initialize SSL context", e);
126+
throw new IOException(e.getMessage());
127+
}
128+
}
129+
130+
private AccessToken getAccessTokenRemotely(String endPoint) throws IOException {
131+
URL url = new URL(endPoint);
132+
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
133+
if (connection instanceof HttpsURLConnection) {
134+
// TODO (CDAP-18047) enable ssl verification
135+
disableVerifySSL(((HttpsURLConnection) connection));
136+
}
137+
connection.connect();
138+
try (Reader reader = new InputStreamReader(connection.getInputStream(),
139+
StandardCharsets.UTF_8)) {
140+
if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
141+
throw new IOException(CharStreams.toString(reader));
142+
}
143+
GenericData token = GSON.fromJson(reader, GenericData.class);
144+
if (!token.containsKey(ACCESS_TOKEN_KEY) || !token.containsKey(EXPIRES_IN_KEY)) {
145+
throw new IOException("Received invalid token");
146+
}
147+
String key = token.get(ACCESS_TOKEN_KEY).toString();
148+
Double expiration = Double.parseDouble(token.get(EXPIRES_IN_KEY).toString());
149+
long expiresAtMilliseconds = System.currentTimeMillis()
150+
+ expiration.longValue() * 1000;
151+
return new AccessToken(key, new Date(expiresAtMilliseconds));
152+
} finally {
153+
connection.disconnect();
154+
}
155+
}
156+
157+
@Override
158+
public AccessToken refreshAccessToken() throws IOException {
159+
ExponentialBackOff backOff = new ExponentialBackOff.Builder()
160+
.setInitialIntervalMillis(MIN_WAIT_TIME_MILLISECOND)
161+
.setMaxIntervalMillis(MAX_WAIT_TIME_MILLISECOND).build();
162+
Exception exception = null;
163+
int counter = 0;
164+
while (counter < NUMBER_OF_RETRIES) {
165+
counter++;
166+
try {
167+
if (endPoint != null) {
168+
return getAccessTokenRemotely(endPoint);
169+
}
170+
return getAccessTokenLocally();
171+
} catch (Exception ex) {
172+
// exception does not get logged since it might get too chatty.
173+
exception = ex;
174+
}
175+
try {
176+
Thread.sleep(backOff.nextBackOffMillis());
177+
} catch (InterruptedException ex) {
178+
exception = ex;
179+
break;
180+
}
181+
}
182+
throw new IOException(exception.getMessage(), exception);
183+
}
184+
}
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
*Copyright © 2020 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
18+
package io.cdap.cdap.stackdriver.logs;
19+
20+
/**
21+
* Definition for Stackdriver label.
22+
*/
23+
public class LabelMapping {
24+
25+
private final String label;
26+
private final String value;
27+
28+
public LabelMapping(String label, String value) {
29+
this.label = label;
30+
this.value = value;
31+
}
32+
33+
public String getLabel() {
34+
return label;
35+
}
36+
37+
public String getValue() {
38+
return value;
39+
}
40+
41+
@Override
42+
public String toString() {
43+
return "LabelMapping{label=" + label
44+
+ ", value=" + value
45+
+ '}';
46+
}
47+
}

0 commit comments

Comments
 (0)