From e2c27a7b113f02bf22ed5e4acc5cf9cea04debaa Mon Sep 17 00:00:00 2001 From: "Wang, Fei" Date: Sun, 26 Oct 2025 14:14:01 -0700 Subject: [PATCH 1/4] Support to get kyuubi server event with RESTful api --- .../apache/kyuubi/client/AdminRestApi.java | 7 + .../client/api/v1/dto/KyuubiServerEvent.java | 146 ++++++++++++++++++ .../apache/kyuubi/server/KyuubiServer.scala | 4 + .../apache/kyuubi/server/api/ApiUtils.scala | 15 +- .../kyuubi/server/api/v1/AdminResource.scala | 15 +- .../server/api/v1/AdminResourceSuite.scala | 11 ++ .../rest/client/AdminRestApiSuite.scala | 10 ++ 7 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiServerEvent.java diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java index c616352142f..b06dbbc7f29 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java @@ -20,6 +20,7 @@ import java.util.*; import org.apache.commons.lang3.StringUtils; import org.apache.kyuubi.client.api.v1.dto.Engine; +import org.apache.kyuubi.client.api.v1.dto.KyuubiServerEvent; import org.apache.kyuubi.client.api.v1.dto.OperationData; import org.apache.kyuubi.client.api.v1.dto.ServerData; import org.apache.kyuubi.client.api.v1.dto.SessionData; @@ -165,6 +166,12 @@ public List listServers() { return Arrays.asList(result); } + public KyuubiServerEvent getServerEvent() { + return this.getClient() + .get( + API_BASE_PATH + "/server/event", null, KyuubiServerEvent.class, client.getAuthHeader()); + } + private IRestClient getClient() { return this.client.getHttpClient(); } diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiServerEvent.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiServerEvent.java new file mode 100644 index 00000000000..e542642ceb1 --- /dev/null +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiServerEvent.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.client.api.v1.dto; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +public class KyuubiServerEvent { + private String serverName; + private Long startTime; + private Long eventTime; + private String state; + private String serverIp; + private Map serverConf; + private Map serverEnv; + + public KyuubiServerEvent() {} + + public KyuubiServerEvent( + String serverName, + Long startTime, + Long eventTime, + String state, + String serverIp, + Map serverConf, + Map serverEnv) { + this.serverName = serverName; + this.startTime = startTime; + this.eventTime = eventTime; + this.state = state; + this.serverIp = serverIp; + this.serverConf = serverConf; + this.serverEnv = serverEnv; + } + + public String getServerName() { + return serverName; + } + + public void setServerName(String serverName) { + this.serverName = serverName; + } + + public Long getStartTime() { + return startTime; + } + + public void setStartTime(Long startTime) { + this.startTime = startTime; + } + + public Long getEventTime() { + return eventTime; + } + + public void setEventTime(Long eventTime) { + this.eventTime = eventTime; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getServerIp() { + return serverIp; + } + + public void setServerIp(String serverIp) { + this.serverIp = serverIp; + } + + public Map getServerConf() { + if (null == serverConf) { + return Collections.emptyMap(); + } + return serverConf; + } + + public void setServerConf(Map serverConf) { + this.serverConf = serverConf; + } + + public Map getServerEnv() { + if (null == serverEnv) { + return Collections.emptyMap(); + } + return serverEnv; + } + + public void setServerEnv(Map serverEnv) { + this.serverEnv = serverEnv; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KyuubiServerEvent that = (KyuubiServerEvent) o; + return Objects.equals(getServerName(), that.getServerName()) + && Objects.equals(getStartTime(), that.getStartTime()) + && Objects.equals(getEventTime(), that.getEventTime()) + && Objects.equals(getState(), that.getState()) + && Objects.equals(getServerIp(), that.getServerIp()) + && Objects.equals(getServerConf(), that.getServerConf()) + && Objects.equals(getServerEnv(), that.getServerEnv()); + } + + @Override + public int hashCode() { + return Objects.hash( + getServerName(), + getStartTime(), + getEventTime(), + getState(), + getServerIp(), + getServerConf(), + getServerEnv()); + } + + @Override + public String toString() { + return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE); + } +} diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala index 0996472653b..ee4c16b6ca4 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala @@ -231,6 +231,10 @@ class KyuubiServer(name: String) extends Serverable(name) { super.stop() } + def getServerEvent(): Option[KyuubiServerInfoEvent] = { + KyuubiServerInfoEvent(this, state) + } + private def initLoggerEventHandler(conf: KyuubiConf): Unit = { ServerEventHandlerRegister.registerEventLoggers(conf) } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala index 1676ba15759..a1154498309 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala @@ -21,7 +21,8 @@ import scala.collection.JavaConverters._ import org.apache.kyuubi.{Logging, Utils} import org.apache.kyuubi.client.api.v1.dto -import org.apache.kyuubi.client.api.v1.dto.{OperationData, OperationProgress, ServerData, SessionData} +import org.apache.kyuubi.client.api.v1.dto.{KyuubiServerEvent, OperationData, OperationProgress, ServerData, SessionData} +import org.apache.kyuubi.events.KyuubiServerInfoEvent import org.apache.kyuubi.ha.client.ServiceNodeInfo import org.apache.kyuubi.operation.KyuubiOperation import org.apache.kyuubi.session.KyuubiSession @@ -136,6 +137,18 @@ object ApiUtils extends Logging { "Running") } + def serverEvent(serverEvent: KyuubiServerInfoEvent): KyuubiServerEvent = { + if (serverEvent == null) return new KyuubiServerEvent() + new KyuubiServerEvent( + serverEvent.serverName, + serverEvent.startTime, + serverEvent.eventTime, + serverEvent.state, + serverEvent.serverIP, + serverEvent.serverConf.asJava, + serverEvent.serverEnv.asJava) + } + def logAndRefineErrorMsg(errorMsg: String, throwable: Throwable): String = { error(errorMsg, throwable) s"$errorMsg: ${Utils.prettyPrint(throwable)}" diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala index 6b354820ac9..ad6283d61cd 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/AdminResource.scala @@ -378,7 +378,7 @@ private[v1] class AdminResource extends ApiRequestContext with Logging { new Content( mediaType = MediaType.APPLICATION_JSON, array = new ArraySchema(schema = new Schema(implementation = - classOf[OperationData])))), + classOf[ServerData])))), description = "list all live kyuubi servers") @GET @Path("server") @@ -401,6 +401,19 @@ private[v1] class AdminResource extends ApiRequestContext with Logging { servers.toSeq } + @ApiResponse( + responseCode = "200", + content = Array( + new Content( + mediaType = MediaType.APPLICATION_JSON, + schema = new Schema(implementation = classOf[KyuubiServerEvent]))), + description = "Get the server event") + @GET + @Path("server/event") + def getServerEvent(): KyuubiServerEvent = { + ApiUtils.serverEvent(KyuubiServer.kyuubiServer.getServerEvent().orNull) + } + private def normalizeEngineInfo( userName: String, engineType: String, diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/AdminResourceSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/AdminResourceSuite.scala index 669a0af0f80..5fb20d73ba2 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/AdminResourceSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/api/v1/AdminResourceSuite.scala @@ -841,4 +841,15 @@ class AdminResourceSuite extends KyuubiFunSuite with RestFrontendTestHelper { assert("Running".equals(testServer.getStatus)) } } + + test("get server event") { + val response = webTarget.path("api/v1/admin/server/event") + .request() + .header(AUTHORIZATION_HEADER, HttpAuthUtils.basicAuthorizationHeader(Utils.currentUser)) + .get + + assert(response.getStatus === 200) + val serverEvent = response.readEntity(classOf[KyuubiServerEvent]) + assert(serverEvent.getStartTime > 0) + } } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala index 9aaefd2735c..0c5a707625e 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/AdminRestApiSuite.scala @@ -173,4 +173,14 @@ class AdminRestApiSuite extends RestClientTestHelper { assert(servers.map(s => s.getInstance()).contains(server.frontendServices.last.connectionUrl)) } } + + test("get server event") { + val spnegoKyuubiRestClient: KyuubiRestClient = + KyuubiRestClient.builder(baseUri.toString) + .authHeaderMethod(KyuubiRestClient.AuthHeaderMethod.SPNEGO) + .spnegoHost("localhost") + .build() + val adminRestApi = new AdminRestApi(spnegoKyuubiRestClient) + assert(adminRestApi.getServerEvent.getStartTime > 0) + } } From 329aa0b4b47a5e69603e831de3238eaeede55980 Mon Sep 17 00:00:00 2001 From: "Wang, Fei" Date: Sun, 26 Oct 2025 15:40:27 -0700 Subject: [PATCH 2/4] return build info --- .../client/api/v1/dto/KyuubiServerEvent.java | 22 ++++++++++++++++--- .../apache/kyuubi/server/api/ApiUtils.scala | 6 ++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiServerEvent.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiServerEvent.java index e542642ceb1..b164baf55cf 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiServerEvent.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiServerEvent.java @@ -31,6 +31,7 @@ public class KyuubiServerEvent { private String serverIp; private Map serverConf; private Map serverEnv; + private Map buildInfo; public KyuubiServerEvent() {} @@ -41,7 +42,8 @@ public KyuubiServerEvent( String state, String serverIp, Map serverConf, - Map serverEnv) { + Map serverEnv, + Map buildInfo) { this.serverName = serverName; this.startTime = startTime; this.eventTime = eventTime; @@ -49,6 +51,7 @@ public KyuubiServerEvent( this.serverIp = serverIp; this.serverConf = serverConf; this.serverEnv = serverEnv; + this.buildInfo = buildInfo; } public String getServerName() { @@ -113,6 +116,17 @@ public void setServerEnv(Map serverEnv) { this.serverEnv = serverEnv; } + public Map getBuildInfo() { + if (null == buildInfo) { + return Collections.emptyMap(); + } + return buildInfo; + } + + public void setBuildInfo(Map buildInfo) { + this.buildInfo = buildInfo; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -124,7 +138,8 @@ public boolean equals(Object o) { && Objects.equals(getState(), that.getState()) && Objects.equals(getServerIp(), that.getServerIp()) && Objects.equals(getServerConf(), that.getServerConf()) - && Objects.equals(getServerEnv(), that.getServerEnv()); + && Objects.equals(getServerEnv(), that.getServerEnv()) + && Objects.equals(getBuildInfo(), that.getBuildInfo()); } @Override @@ -136,7 +151,8 @@ public int hashCode() { getState(), getServerIp(), getServerConf(), - getServerEnv()); + getServerEnv(), + getBuildInfo()); } @Override diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala index a1154498309..25566b28dff 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala @@ -146,7 +146,11 @@ object ApiUtils extends Logging { serverEvent.state, serverEvent.serverIP, serverEvent.serverConf.asJava, - serverEvent.serverEnv.asJava) + serverEvent.serverEnv.asJava, + (Map( + "BUILD_USER" -> serverEvent.BUILD_USER, + "BUILD_DATE" -> serverEvent.BUILD_DATE, + "REPO_URL" -> serverEvent.REPO_URL) ++ serverEvent.VERSION_INFO).asJava) } def logAndRefineErrorMsg(errorMsg: String, throwable: Throwable): String = { From aabf80e95813445b78d6764ec781c3c077201fad Mon Sep 17 00:00:00 2001 From: "Wang, Fei" Date: Sun, 26 Oct 2025 20:52:29 -0700 Subject: [PATCH 3/4] serverIP --- .../client/api/v1/dto/KyuubiServerEvent.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiServerEvent.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiServerEvent.java index b164baf55cf..7b508970caf 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiServerEvent.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiServerEvent.java @@ -28,7 +28,7 @@ public class KyuubiServerEvent { private Long startTime; private Long eventTime; private String state; - private String serverIp; + private String serverIP; private Map serverConf; private Map serverEnv; private Map buildInfo; @@ -40,7 +40,7 @@ public KyuubiServerEvent( Long startTime, Long eventTime, String state, - String serverIp, + String serverIP, Map serverConf, Map serverEnv, Map buildInfo) { @@ -48,7 +48,7 @@ public KyuubiServerEvent( this.startTime = startTime; this.eventTime = eventTime; this.state = state; - this.serverIp = serverIp; + this.serverIP = serverIP; this.serverConf = serverConf; this.serverEnv = serverEnv; this.buildInfo = buildInfo; @@ -86,12 +86,12 @@ public void setState(String state) { this.state = state; } - public String getServerIp() { - return serverIp; + public String getServerIP() { + return serverIP; } - public void setServerIp(String serverIp) { - this.serverIp = serverIp; + public void setServerIP(String serverIP) { + this.serverIP = serverIP; } public Map getServerConf() { @@ -136,7 +136,7 @@ public boolean equals(Object o) { && Objects.equals(getStartTime(), that.getStartTime()) && Objects.equals(getEventTime(), that.getEventTime()) && Objects.equals(getState(), that.getState()) - && Objects.equals(getServerIp(), that.getServerIp()) + && Objects.equals(getServerIP(), that.getServerIP()) && Objects.equals(getServerConf(), that.getServerConf()) && Objects.equals(getServerEnv(), that.getServerEnv()) && Objects.equals(getBuildInfo(), that.getBuildInfo()); @@ -149,7 +149,7 @@ public int hashCode() { getStartTime(), getEventTime(), getState(), - getServerIp(), + getServerIP(), getServerConf(), getServerEnv(), getBuildInfo()); From 2ee76ced301ec24a19722931b25d5b7da4af4618 Mon Sep 17 00:00:00 2001 From: "Wang, Fei" Date: Sat, 6 Dec 2025 15:16:00 -0800 Subject: [PATCH 4/4] address comments --- docs/configuration/settings.md | 41 ++++++++++--------- .../org/apache/kyuubi/config/KyuubiConf.scala | 20 +++++++++ .../apache/kyuubi/server/api/ApiUtils.scala | 16 +++++++- 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index ab1966cde62..26751877c3f 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -444,26 +444,27 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co ### Server -| Key | Default | Meaning | Type | Since | -|----------------------------------------------------------|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------| -| kyuubi.server.administrators || Comma-separated list of Kyuubi service administrators. We use this config to grant admin permission to any service accounts when security mechanism is enabled. Note, when kyuubi.authentication is configured to NOSASL or NONE, everyone is treated as administrator. | set | 1.8.0 | -| kyuubi.server.info.provider | ENGINE | The server information provider name, some clients may rely on this information to check the server compatibilities and functionalities.
  • SERVER: Return Kyuubi server information.
  • ENGINE: Return Kyuubi engine information.
    • | string | 1.6.1 | -| kyuubi.server.limit.batch.connections.per.ipaddress | <undefined> | Maximum kyuubi server batch connections per ipaddress. Any user exceeding this limit will not be allowed to connect. | int | 1.7.0 | -| kyuubi.server.limit.batch.connections.per.user | <undefined> | Maximum kyuubi server batch connections per user. Any user exceeding this limit will not be allowed to connect. | int | 1.7.0 | -| kyuubi.server.limit.batch.connections.per.user.ipaddress | <undefined> | Maximum kyuubi server batch connections per user:ipaddress combination. Any user-ipaddress exceeding this limit will not be allowed to connect. | int | 1.7.0 | -| kyuubi.server.limit.client.fetch.max.rows | <undefined> | Max rows limit for getting result row set operation. If the max rows specified by client-side is larger than the limit, request will fail directly. | int | 1.8.0 | -| kyuubi.server.limit.connections.ip.deny.list || The client ip in the deny list will be denied to connect to kyuubi server. | set | 1.9.1 | -| kyuubi.server.limit.connections.per.ipaddress | <undefined> | Maximum kyuubi server connections per ipaddress. Any user exceeding this limit will not be allowed to connect. | int | 1.6.0 | -| kyuubi.server.limit.connections.per.user | <undefined> | Maximum kyuubi server connections per user. Any user exceeding this limit will not be allowed to connect. | int | 1.6.0 | -| kyuubi.server.limit.connections.per.user.ipaddress | <undefined> | Maximum kyuubi server connections per user:ipaddress combination. Any user-ipaddress exceeding this limit will not be allowed to connect. | int | 1.6.0 | -| kyuubi.server.limit.connections.user.deny.list || The user in the deny list will be denied to connect to kyuubi server, if the user has configured both user.unlimited.list and user.deny.list, the priority of the latter is higher. | set | 1.8.0 | -| kyuubi.server.limit.connections.user.unlimited.list || The maximum connections of the user in the white list will not be limited. | set | 1.7.0 | -| kyuubi.server.name | <undefined> | The name of Kyuubi Server. | string | 1.5.0 | -| kyuubi.server.periodicGC.interval | PT30M | How often to trigger the periodic garbage collection. 0 will disable it. | duration | 1.7.0 | -| kyuubi.server.redaction.regex | <undefined> | Regex to decide which Kyuubi contain sensitive information. When this regex matches a property key or value, the value is redacted from the various logs. || 1.6.0 | -| kyuubi.server.tempFile.expireTime | P30D | Expiration timout for cleanup server-side temporary files, e.g. operation logs. | duration | 1.10.0 | -| kyuubi.server.tempFile.maxCount | <undefined> | The upper threshold size of server-side temporary file paths to cleanup | int | 1.10.0 | -| kyuubi.server.thrift.resultset.default.fetch.size | 1000 | The number of rows sent in one Fetch RPC call by the server to the client, if not specified by the client. Respect `hive.server2.thrift.resultset.default.fetch.size` hive conf. | int | 1.9.1 | +| Key | Default | Meaning | Type | Since | +|----------------------------------------------------------|-------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------| +| kyuubi.server.administrators || Comma-separated list of Kyuubi service administrators. We use this config to grant admin permission to any service accounts when security mechanism is enabled. Note, when kyuubi.authentication is configured to NOSASL or NONE, everyone is treated as administrator. | set | 1.8.0 | +| kyuubi.server.event.conf.mode | REDACTED | Specifies how server configuration is exposed in server events. Available options are:
    • ORIGINAL: Return the original server configuration.
    • REDACTED: Redact sensitive information based on kyuubi.server.redaction.regex.
    • NONE: Do not return any server configuration.
    | string | 1.11.0 | +| kyuubi.server.info.provider | ENGINE | The server information provider name, some clients may rely on this information to check the server compatibilities and functionalities.
  • SERVER: Return Kyuubi server information.
  • ENGINE: Return Kyuubi engine information.
    • | string | 1.6.1 | +| kyuubi.server.limit.batch.connections.per.ipaddress | <undefined> | Maximum kyuubi server batch connections per ipaddress. Any user exceeding this limit will not be allowed to connect. | int | 1.7.0 | +| kyuubi.server.limit.batch.connections.per.user | <undefined> | Maximum kyuubi server batch connections per user. Any user exceeding this limit will not be allowed to connect. | int | 1.7.0 | +| kyuubi.server.limit.batch.connections.per.user.ipaddress | <undefined> | Maximum kyuubi server batch connections per user:ipaddress combination. Any user-ipaddress exceeding this limit will not be allowed to connect. | int | 1.7.0 | +| kyuubi.server.limit.client.fetch.max.rows | <undefined> | Max rows limit for getting result row set operation. If the max rows specified by client-side is larger than the limit, request will fail directly. | int | 1.8.0 | +| kyuubi.server.limit.connections.ip.deny.list || The client ip in the deny list will be denied to connect to kyuubi server. | set | 1.9.1 | +| kyuubi.server.limit.connections.per.ipaddress | <undefined> | Maximum kyuubi server connections per ipaddress. Any user exceeding this limit will not be allowed to connect. | int | 1.6.0 | +| kyuubi.server.limit.connections.per.user | <undefined> | Maximum kyuubi server connections per user. Any user exceeding this limit will not be allowed to connect. | int | 1.6.0 | +| kyuubi.server.limit.connections.per.user.ipaddress | <undefined> | Maximum kyuubi server connections per user:ipaddress combination. Any user-ipaddress exceeding this limit will not be allowed to connect. | int | 1.6.0 | +| kyuubi.server.limit.connections.user.deny.list || The user in the deny list will be denied to connect to kyuubi server, if the user has configured both user.unlimited.list and user.deny.list, the priority of the latter is higher. | set | 1.8.0 | +| kyuubi.server.limit.connections.user.unlimited.list || The maximum connections of the user in the white list will not be limited. | set | 1.7.0 | +| kyuubi.server.name | <undefined> | The name of Kyuubi Server. | string | 1.5.0 | +| kyuubi.server.periodicGC.interval | PT30M | How often to trigger the periodic garbage collection. 0 will disable it. | duration | 1.7.0 | +| kyuubi.server.redaction.regex | <undefined> | Regex to decide which Kyuubi contain sensitive information. When this regex matches a property key or value, the value is redacted from the various logs. || 1.6.0 | +| kyuubi.server.tempFile.expireTime | P30D | Expiration timout for cleanup server-side temporary files, e.g. operation logs. | duration | 1.10.0 | +| kyuubi.server.tempFile.maxCount | <undefined> | The upper threshold size of server-side temporary file paths to cleanup | int | 1.10.0 | +| kyuubi.server.thrift.resultset.default.fetch.size | 1000 | The number of rows sent in one Fetch RPC call by the server to the client, if not specified by the client. Respect `hive.server2.thrift.resultset.default.fetch.size` hive conf. | int | 1.9.1 | ### Session diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 65864986552..9a8d3ab18f0 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -3940,4 +3940,24 @@ object KyuubiConf { .version("1.9.1") .serverOnly .fallbackConf(HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE) + + object ServerEventConfMode extends Enumeration { + type ServerEventConfMode = Value + val ORIGINAL, REDACTED, NONE = Value + } + + val SERVER_EVENT_CONF_MODE: ConfigEntry[String] = + buildConf("kyuubi.server.event.conf.mode") + .doc("Specifies how server configuration is exposed in server events. " + + "Available options are:
      " + + "
    • ORIGINAL: Return the original server configuration.
      • " + + "
    • REDACTED: Redact sensitive information based on" + + s" ${SERVER_SECRET_REDACTION_PATTERN.key}.
      • " + + "
    • NONE: Do not return any server configuration.
    ") + .version("1.11.0") + .serverOnly + .stringConf + .transformToUpperCase + .checkValues(ServerEventConfMode) + .createWithDefault(ServerEventConfMode.REDACTED.toString) } diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala index 25566b28dff..281243c9e39 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/ApiUtils.scala @@ -22,9 +22,11 @@ import scala.collection.JavaConverters._ import org.apache.kyuubi.{Logging, Utils} import org.apache.kyuubi.client.api.v1.dto import org.apache.kyuubi.client.api.v1.dto.{KyuubiServerEvent, OperationData, OperationProgress, ServerData, SessionData} +import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.events.KyuubiServerInfoEvent import org.apache.kyuubi.ha.client.ServiceNodeInfo import org.apache.kyuubi.operation.KyuubiOperation +import org.apache.kyuubi.server.KyuubiServer import org.apache.kyuubi.session.KyuubiSession object ApiUtils extends Logging { @@ -145,7 +147,7 @@ object ApiUtils extends Logging { serverEvent.eventTime, serverEvent.state, serverEvent.serverIP, - serverEvent.serverConf.asJava, + processServerConf(serverEvent.serverConf).asJava, serverEvent.serverEnv.asJava, (Map( "BUILD_USER" -> serverEvent.BUILD_USER, @@ -153,6 +155,18 @@ object ApiUtils extends Logging { "REPO_URL" -> serverEvent.REPO_URL) ++ serverEvent.VERSION_INFO).asJava) } + private def processServerConf(serverConf: Map[String, String]): Map[String, String] = { + val kyuubiConf = Option(KyuubiServer.kyuubiServer).map(_.getConf).getOrElse(new KyuubiConf()) + val confMode = kyuubiConf.get(KyuubiConf.SERVER_EVENT_CONF_MODE) + KyuubiConf.ServerEventConfMode.withName(confMode) match { + case KyuubiConf.ServerEventConfMode.ORIGINAL => serverConf + case KyuubiConf.ServerEventConfMode.REDACTED => + val redactionPattern = kyuubiConf.get(KyuubiConf.SERVER_SECRET_REDACTION_PATTERN) + Utils.redact(redactionPattern, serverConf.toSeq).toMap + case KyuubiConf.ServerEventConfMode.NONE => Map.empty[String, String] + } + } + def logAndRefineErrorMsg(errorMsg: String, throwable: Throwable): String = { error(errorMsg, throwable) s"$errorMsg: ${Utils.prettyPrint(throwable)}"