Skip to content

Commit a13e20b

Browse files
leixmleixm
committed
fix
1 parent 21cc1b7 commit a13e20b

File tree

16 files changed

+361
-146
lines changed

16 files changed

+361
-146
lines changed

‎common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala

+40-26
Original file line numberDiff line numberDiff line change
@@ -665,8 +665,6 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
665665
def estimatedPartitionSizeForEstimationUpdateInterval: Long =
666666
get(ESTIMATED_PARTITION_SIZE_UPDATE_INTERVAL)
667667
def masterResourceConsumptionInterval: Long = get(MASTER_RESOURCE_CONSUMPTION_INTERVAL)
668-
def masterUserDiskUsageThreshold: Long = get(MASTER_USER_DISK_USAGE_THRESHOLD)
669-
def masterClusterDiskUsageThreshold: Long = get(MASTER_CLUSTER_DISK_USAGE_THRESHOLD)
670668
def clusterName: String = get(CLUSTER_NAME)
671669

672670
// //////////////////////////////////////////////////////
@@ -2930,26 +2928,6 @@ object CelebornConf extends Logging {
29302928
.timeConf(TimeUnit.MILLISECONDS)
29312929
.createWithDefaultString("30s")
29322930

2933-
val MASTER_USER_DISK_USAGE_THRESHOLD: ConfigEntry[Long] =
2934-
buildConf("celeborn.master.userResourceConsumption.user.threshold")
2935-
.categories("master")
2936-
.doc("When user resource consumption exceeds quota, Master will " +
2937-
"interrupt some apps until user resource consumption is less " +
2938-
"than this value. Default value is Long.MaxValue which means disable check.")
2939-
.version("0.6.0")
2940-
.bytesConf(ByteUnit.BYTE)
2941-
.createWithDefault(Long.MaxValue)
2942-
2943-
val MASTER_CLUSTER_DISK_USAGE_THRESHOLD: ConfigEntry[Long] =
2944-
buildConf("celeborn.master.userResourceConsumption.cluster.threshold")
2945-
.categories("master")
2946-
.doc("When cluster resource consumption exceeds quota, Master will " +
2947-
"interrupt some apps until cluster resource consumption is less " +
2948-
"than this value. Default value is Long.MaxValue which means disable check.")
2949-
.version("0.6.0")
2950-
.bytesConf(ByteUnit.BYTE)
2951-
.createWithDefault(Long.MaxValue)
2952-
29532931
val CLUSTER_NAME: ConfigEntry[String] =
29542932
buildConf("celeborn.cluster.name")
29552933
.categories("master", "worker")
@@ -5396,7 +5374,7 @@ object CelebornConf extends Logging {
53965374
.stringConf
53975375
.createWithDefault(IdentityProvider.DEFAULT_USERNAME)
53985376

5399-
val QUOTA_DISK_BYTES_WRITTEN: ConfigEntry[Long] =
5377+
val QUOTA_TENANT_DISK_BYTES_WRITTEN: ConfigEntry[Long] =
54005378
buildConf("celeborn.quota.tenant.diskBytesWritten")
54015379
.categories("quota")
54025380
.dynamic
@@ -5405,7 +5383,7 @@ object CelebornConf extends Logging {
54055383
.bytesConf(ByteUnit.BYTE)
54065384
.createWithDefault(Long.MaxValue)
54075385

5408-
val QUOTA_DISK_FILE_COUNT: ConfigEntry[Long] =
5386+
val QUOTA_TENANT_DISK_FILE_COUNT: ConfigEntry[Long] =
54095387
buildConf("celeborn.quota.tenant.diskFileCount")
54105388
.categories("quota")
54115389
.dynamic
@@ -5414,7 +5392,7 @@ object CelebornConf extends Logging {
54145392
.bytesConf(ByteUnit.BYTE)
54155393
.createWithDefault(Long.MaxValue)
54165394

5417-
val QUOTA_HDFS_BYTES_WRITTEN: ConfigEntry[Long] =
5395+
val QUOTA_TENANT_HDFS_BYTES_WRITTEN: ConfigEntry[Long] =
54185396
buildConf("celeborn.quota.tenant.hdfsBytesWritten")
54195397
.categories("quota")
54205398
.dynamic
@@ -5423,7 +5401,7 @@ object CelebornConf extends Logging {
54235401
.bytesConf(ByteUnit.BYTE)
54245402
.createWithDefault(Long.MaxValue)
54255403

5426-
val QUOTA_HDFS_FILE_COUNT: ConfigEntry[Long] =
5404+
val QUOTA_TENANT_HDFS_FILE_COUNT: ConfigEntry[Long] =
54275405
buildConf("celeborn.quota.tenant.hdfsFileCount")
54285406
.categories("quota")
54295407
.dynamic
@@ -6072,4 +6050,40 @@ object CelebornConf extends Logging {
60726050
.version("0.6.0")
60736051
.longConf
60746052
.createWithDefault(Long.MaxValue)
6053+
6054+
val QUOTA_USER_DISK_BYTES_WRITTEN: ConfigEntry[Long] =
6055+
buildConf("celeborn.quota.user.diskBytesWritten")
6056+
.categories("quota")
6057+
.dynamic
6058+
.doc("Quota dynamic configuration for user written disk bytes.")
6059+
.version("0.6.0")
6060+
.bytesConf(ByteUnit.BYTE)
6061+
.createWithDefault(Long.MaxValue)
6062+
6063+
val QUOTA_USER_DISK_FILE_COUNT: ConfigEntry[Long] =
6064+
buildConf("celeborn.quota.user.diskFileCount")
6065+
.categories("quota")
6066+
.dynamic
6067+
.doc("Quota dynamic configuration for user written disk file count.")
6068+
.version("0.6.0")
6069+
.longConf
6070+
.createWithDefault(Long.MaxValue)
6071+
6072+
val QUOTA_USER_HDFS_BYTES_WRITTEN: ConfigEntry[Long] =
6073+
buildConf("celeborn.quota.user.hdfsBytesWritten")
6074+
.categories("quota")
6075+
.dynamic
6076+
.doc("Quota dynamic configuration for user written hdfs bytes.")
6077+
.version("0.6.0")
6078+
.bytesConf(ByteUnit.BYTE)
6079+
.createWithDefault(Long.MaxValue)
6080+
6081+
val QUOTA_USER_HDFS_FILE_COUNT: ConfigEntry[Long] =
6082+
buildConf("celeborn.quota.user.hdfsFileCount")
6083+
.categories("quota")
6084+
.dynamic
6085+
.doc("Quota dynamic configuration for user written hdfs file count.")
6086+
.version("0.6.0")
6087+
.longConf
6088+
.createWithDefault(Long.MaxValue)
60756089
}

‎docs/configuration/master.md

-2
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,7 @@ license: |
7979
| celeborn.master.slot.assign.loadAware.numDiskGroups | 5 | false | This configuration is a guidance for load-aware slot allocation algorithm. This value is control how many disk groups will be created. | 0.3.0 | celeborn.slots.assign.loadAware.numDiskGroups |
8080
| celeborn.master.slot.assign.maxWorkers | 10000 | false | Max workers that slots of one shuffle can be allocated on. Will choose the smaller positive one from Master side and Client side, see `celeborn.client.slot.assign.maxWorkers`. | 0.3.1 | |
8181
| celeborn.master.slot.assign.policy | ROUNDROBIN | false | Policy for master to assign slots, Celeborn supports two types of policy: roundrobin and loadaware. Loadaware policy will be ignored when `HDFS` is enabled in `celeborn.storage.availableTypes` | 0.3.0 | celeborn.slots.assign.policy |
82-
| celeborn.master.userResourceConsumption.cluster.threshold | 9223372036854775807b | false | When cluster resource consumption exceeds quota, Master will interrupt some apps until cluster resource consumption is less than this value. Default value is Long.MaxValue which means disable check. | 0.6.0 | |
8382
| celeborn.master.userResourceConsumption.update.interval | 30s | false | Time length for a window about compute user resource consumption. | 0.3.0 | |
84-
| celeborn.master.userResourceConsumption.user.threshold | 9223372036854775807b | false | When user resource consumption exceeds quota, Master will interrupt some apps until user resource consumption is less than this value. Default value is Long.MaxValue which means disable check. | 0.6.0 | |
8583
| celeborn.master.workerUnavailableInfo.expireTimeout | 1800s | false | Worker unavailable info would be cleared when the retention period is expired. Set -1 to disable the expiration. | 0.3.1 | |
8684
| celeborn.quota.enabled | true | false | When Master side sets to true, the master will enable to check the quota via QuotaManager. When Client side sets to true, LifecycleManager will request Master side to check whether the current user has enough quota before registration of shuffle. Fallback to the default shuffle service when Master side checks that there is no enough quota for current user. | 0.2.0 | |
8785
| celeborn.redaction.regex | (?i)secret|password|token|access[.]key | false | Regex to decide which Celeborn configuration properties and environment variables in master and worker environments contain sensitive information. When this regex matches a property key or value, the value is redacted from the logging. | 0.5.0 | |

‎docs/configuration/quota.md

+4
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,8 @@ license: |
3232
| celeborn.quota.tenant.diskFileCount | 9223372036854775807b | true | Quota dynamic configuration for written disk file count. | 0.5.0 | |
3333
| celeborn.quota.tenant.hdfsBytesWritten | 9223372036854775807b | true | Quota dynamic configuration for written hdfs bytes. | 0.5.0 | |
3434
| celeborn.quota.tenant.hdfsFileCount | 9223372036854775807 | true | Quota dynamic configuration for written hdfs file count. | 0.5.0 | |
35+
| celeborn.quota.user.diskBytesWritten | 9223372036854775807b | true | Quota dynamic configuration for user written disk bytes. | 0.6.0 | |
36+
| celeborn.quota.user.diskFileCount | 9223372036854775807 | true | Quota dynamic configuration for user written disk file count. | 0.6.0 | |
37+
| celeborn.quota.user.hdfsBytesWritten | 9223372036854775807b | true | Quota dynamic configuration for user written hdfs bytes. | 0.6.0 | |
38+
| celeborn.quota.user.hdfsFileCount | 9223372036854775807 | true | Quota dynamic configuration for user written hdfs file count. | 0.6.0 | |
3539
<!--end-include-->

‎docs/migration.md

+8
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,14 @@ license: |
2323

2424
# Upgrading from 0.5 to 0.6
2525

26+
- Since 0.6.0, Celeborn modified `celeborn.quota.tenant.diskBytesWritten` to `celeborn.quota.user.diskBytesWritten`. Please use `celeborn.quota.user.diskBytesWritten` if you want to set user level quota.
27+
28+
- Since 0.6.0, Celeborn modified `celeborn.quota.tenant.diskFileCount` to `celeborn.quota.user.diskFileCount`. Please use `celeborn.quota.user.diskFileCount` if you want to set user level quota.
29+
30+
- Since 0.6.0, Celeborn modified `celeborn.quota.tenant.hdfsBytesWritten` to `celeborn.quota.user.hdfsBytesWritten`. Please use `celeborn.quota.user.hdfsBytesWritten` if you want to set user level quota.
31+
32+
- Since 0.6.0, Celeborn modified `celeborn.quota.tenant.hdfsFileCount` to `celeborn.quota.user.hdfsFileCount`. Please use `celeborn.quota.user.hdfsFileCount` if you want to set user level quota.
33+
2634
- Since 0.6.0, Celeborn deprecate `celeborn.worker.congestionControl.low.watermark`. Please use `celeborn.worker.congestionControl.diskBuffer.low.watermark` instead.
2735

2836
- Since 0.6.0, Celeborn deprecate `celeborn.worker.congestionControl.high.watermark`. Please use `celeborn.worker.congestionControl.diskBuffer.high.watermark` instead.

‎master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java

+1-4
Original file line numberDiff line numberDiff line change
@@ -250,21 +250,18 @@ public void updateWorkerHeartbeatMeta(
250250
int fetchPort,
251251
int replicatePort,
252252
Map<String, DiskInfo> disks,
253-
Map<UserIdentifier, ResourceConsumption> userResourceConsumption,
254253
long time,
255254
WorkerStatus workerStatus,
256255
boolean highWorkload) {
257256
WorkerInfo worker =
258-
new WorkerInfo(
259-
host, rpcPort, pushPort, fetchPort, replicatePort, -1, disks, userResourceConsumption);
257+
new WorkerInfo(host, rpcPort, pushPort, fetchPort, replicatePort, -1, disks, null);
260258
AtomicLong availableSlots = new AtomicLong();
261259
LOG.debug("update worker {}:{} heartbeat {}", host, rpcPort, disks);
262260
synchronized (workersMap) {
263261
Optional<WorkerInfo> workerInfo = Optional.ofNullable(workersMap.get(worker.toUniqueId()));
264262
workerInfo.ifPresent(
265263
info -> {
266264
info.updateThenGetDiskInfos(disks, Option.apply(estimatedPartitionSize));
267-
info.updateThenGetUserResourceConsumption(userResourceConsumption);
268265
availableSlots.set(info.totalAvailableSlots());
269266
info.lastHeartbeat_$eq(time);
270267
info.setWorkerStatus(workerStatus);

‎master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/IMetadataHandler.java

-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@ void handleWorkerHeartbeat(
6666
int fetchPort,
6767
int replicatePort,
6868
Map<String, DiskInfo> disks,
69-
Map<UserIdentifier, ResourceConsumption> userResourceConsumption,
7069
long time,
7170
boolean highWorkload,
7271
WorkerStatus workerStatus,

‎master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/SingleMasterMetaManager.java

+1-11
Original file line numberDiff line numberDiff line change
@@ -118,22 +118,12 @@ public void handleWorkerHeartbeat(
118118
int fetchPort,
119119
int replicatePort,
120120
Map<String, DiskInfo> disks,
121-
Map<UserIdentifier, ResourceConsumption> userResourceConsumption,
122121
long time,
123122
boolean highWorkload,
124123
WorkerStatus workerStatus,
125124
String requestId) {
126125
updateWorkerHeartbeatMeta(
127-
host,
128-
rpcPort,
129-
pushPort,
130-
fetchPort,
131-
replicatePort,
132-
disks,
133-
userResourceConsumption,
134-
time,
135-
workerStatus,
136-
highWorkload);
126+
host, rpcPort, pushPort, fetchPort, replicatePort, disks, time, workerStatus, highWorkload);
137127
}
138128

139129
@Override

‎master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java

-3
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,6 @@ public void handleWorkerHeartbeat(
272272
int fetchPort,
273273
int replicatePort,
274274
Map<String, DiskInfo> disks,
275-
Map<UserIdentifier, ResourceConsumption> userResourceConsumption,
276275
long time,
277276
boolean highWorkload,
278277
WorkerStatus workerStatus,
@@ -290,8 +289,6 @@ public void handleWorkerHeartbeat(
290289
.setFetchPort(fetchPort)
291290
.setReplicatePort(replicatePort)
292291
.putAllDisks(MetaUtil.toPbDiskInfos(disks))
293-
.putAllUserResourceConsumption(
294-
MetaUtil.toPbUserResourceConsumption(userResourceConsumption))
295292
.setWorkerStatus(MetaUtil.toPbWorkerStatus(workerStatus))
296293
.setTime(time)
297294
.setHighWorkload(highWorkload)

‎master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala

+5-5
Original file line numberDiff line numberDiff line change
@@ -186,16 +186,16 @@ private[celeborn] class Master(
186186
private val hasHDFSStorage = conf.hasHDFSStorage
187187
private val hasS3Storage = conf.hasS3Storage
188188

189+
// workerUniqueId -> ResourceConsumption
190+
private val workerToResourceConsumptions =
191+
JavaUtils.newConcurrentHashMap[String, util.Map[UserIdentifier, ResourceConsumption]]()
189192
private val quotaManager = new QuotaManager(
190-
statusSystem,
193+
workerToResourceConsumptions,
191194
masterSource,
192195
resourceConsumptionSource,
193196
conf,
194197
configService)
195198
private val tagsManager = new TagsManager(Option(configService))
196-
private val masterResourceConsumptionInterval = conf.masterResourceConsumptionInterval
197-
private val userResourceConsumptions =
198-
JavaUtils.newConcurrentHashMap[UserIdentifier, (ResourceConsumption, Long)]()
199199

200200
private val slotsAssignMaxWorkers = conf.masterSlotAssignMaxWorkers
201201
private val slotsAssignLoadAwareDiskGroupNum = conf.masterSlotAssignLoadAwareDiskGroupNum
@@ -670,13 +670,13 @@ private[celeborn] class Master(
670670
fetchPort,
671671
replicatePort,
672672
disks.map { disk => disk.mountPoint -> disk }.toMap.asJava,
673-
userResourceConsumption,
674673
System.currentTimeMillis(),
675674
highWorkload,
676675
workerStatus,
677676
requestId)
678677
}
679678

679+
workerToResourceConsumptions.put(targetWorker.toUniqueId(), userResourceConsumption)
680680
val expiredShuffleKeys = new util.HashSet[String]
681681
activeShuffleKeys.asScala.foreach { shuffleKey =>
682682
val (appId, shuffleId) = Utils.splitShuffleKey(shuffleKey)

0 commit comments

Comments
 (0)