From b67cac0e9c91598e1f6bca9aacb23055eb1b2d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=BA=90=E4=B8=89?= Date: Fri, 27 Nov 2020 15:23:27 +0800 Subject: [PATCH] PullRequest: 2 add test for slot store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 向旭 * What type of PR is this? add test for slot What this PR does / why we need it: Which issue(s) this PR fixes: Special notes for your reviewer: Does this PR introduce a user-facing change?: What type of PR is this? add test for slotstore What this PR does / why we need it: Which issue(s) this PR fixes: Special notes for your reviewer: Does this PR introduce a user-facing change?: --- .aci.yml | 51 ++++ .master.aci.yml | 51 ++++ client/all/pom.xml | 2 +- client/api/pom.xml | 2 +- client/impl/pom.xml | 2 +- client/log/pom.xml | 2 +- client/pom.xml | 2 +- core/pom.xml | 2 +- pom.xml | 2 +- server/common/model/pom.xml | 8 +- .../common/model/dataserver/DatumSummary.java | 2 +- .../common/model/slot/DataSlotDiffUtils.java | 62 ++-- .../slot/DataSlotDiffSyncResultTest.java | 276 ++++++++++++++++++ server/common/pom.xml | 2 +- server/common/util/pom.xml | 2 +- .../sofa/registry/util/CollectionUtils.java | 9 + server/consistency/pom.xml | 2 +- server/distribution/data/pom.xml | 2 +- server/distribution/integration/pom.xml | 2 +- server/distribution/meta/pom.xml | 2 +- server/distribution/pom.xml | 2 +- server/distribution/session/pom.xml | 2 +- server/pom.xml | 2 +- server/remoting/api/pom.xml | 2 +- server/remoting/bolt/pom.xml | 2 +- server/remoting/http/pom.xml | 2 +- server/remoting/pom.xml | 2 +- server/server/data/pom.xml | 2 +- .../data/bootstrap/DataServerConfig.java | 4 +- .../registry/server/data/MigrateTest.java | 6 +- server/server/integration/pom.xml | 4 +- server/server/meta/pom.xml | 2 +- server/server/pom.xml | 2 +- server/server/session/pom.xml | 2 +- server/store/api/pom.xml | 2 +- server/store/jraft/pom.xml | 2 +- server/store/pom.xml | 2 +- test/pom.xml | 4 +- tools/change_version.sh | 0 tools/check_format.sh | 0 40 files changed, 468 insertions(+), 63 deletions(-) create mode 100644 .aci.yml create mode 100644 .master.aci.yml create mode 100644 server/common/model/src/test/java/com/alipay/sofa/registry/common/model/slot/DataSlotDiffSyncResultTest.java mode change 100644 => 100755 tools/change_version.sh mode change 100644 => 100755 tools/check_format.sh diff --git a/.aci.yml b/.aci.yml new file mode 100644 index 000000000..8ab3ce16a --- /dev/null +++ b/.aci.yml @@ -0,0 +1,51 @@ +stages: # 定义了一条pipeline + - 代码检测 # 定义了stage + - 单元测试 + - 编译 + #- 发布JAR包 + +PMD扫描: # 定义了一个job + stage: 代码检测 # 绑定到stage + plugin: PMD # 指定使用的插件 + checkRule: # 校验结果 + - pmd1 = 0 && pmd2 = 0 + +STC扫描: + stage: 代码检测 + plugin: STC + checkRule: + - stc = 0 + +单元测试: + stage: 单元测试 + plugin: UT + checkRule: + - passRate = 100 + pluginConfig: + markdown: admin/result.md # 向页面输出一些自定义内容 + +编译: + stage: 编译 + plugin: ANT-BUILD # 使用插件ANT-BUILD + pluginConfig: + image: reg.docker.alibaba-inc.com/antb/sofa-build:0.0.2 + script: + - export PATH=$PATH:/opt/taobao/java/bin:/opt/taobao/maven/bin + - java -version + - mvn -version + - mvn clean package -Dmaven.test.skip=true -Pdev + outputs: + - name: registry.tgz + paths: ["./distribution-enterprise/alipay/target/registry.tgz"] + +发布JAR包: + stage: 发布JAR包 + plugin: ANT-BUILD + pluginConfig: + image: reg.docker.alibaba-inc.com/antb/jarbuild:0.0.1 + script: + - java -version + - mvn -version + - sed -i '//,/<\/mirrors>/d' /opt/taobao/maven_settings/settings-release.xml + - sed -i '//,/<\/profiles>/d' /opt/taobao/maven_settings/settings-release.xml + - mvn clean deploy -Dmaven.test.skip=true -s /opt/taobao/maven_settings/settings-release.xml -Pdev -Pantcode-release \ No newline at end of file diff --git a/.master.aci.yml b/.master.aci.yml new file mode 100644 index 000000000..d4113f0f3 --- /dev/null +++ b/.master.aci.yml @@ -0,0 +1,51 @@ +stages: # 定义了一条pipeline + - 代码检测 # 定义了stage + - 单元测试 + - 编译 + #- 发布JAR包 + +PMD扫描: # 定义了一个job + stage: 代码检测 # 绑定到stage + plugin: PMD # 指定使用的插件 + checkRule: # 校验结果 + - pmd1 = 0 && pmd2 = 0 + +STC扫描: + stage: 代码检测 + plugin: STC + checkRule: + - stc = 0 + +单元测试: + stage: 单元测试 + plugin: UT + checkRule: + - passRate = 100 + pluginConfig: + markdown: admin/result.md # 向页面输出一些自定义内容 + +编译: + stage: 编译 + plugin: ANT-BUILD # 使用插件ANT-BUILD + pluginConfig: + image: reg.docker.alibaba-inc.com/antb/sofa-build:0.0.2 + script: + - export PATH=$PATH:/opt/taobao/java/bin:/opt/taobao/maven/bin + - java -version + - mvn -version + - mvn clean package -Dmaven.test.skip=true -Ptest + outputs: + - name: registry.tgz + paths: ["./distribution-enterprise/alipay/target/registry.tgz"] + +发布JAR包: + stage: 发布JAR包 + plugin: ANT-BUILD + pluginConfig: + image: reg.docker.alibaba-inc.com/antb/jarbuild:0.0.1 + script: + - java -version + - mvn -version + - sed -i '//,/<\/mirrors>/d' /opt/taobao/maven_settings/settings-release.xml + - sed -i '//,/<\/profiles>/d' /opt/taobao/maven_settings/settings-release.xml + - mvn clean deploy -Dmaven.test.skip=true -s /opt/taobao/maven_settings/settings-release.xml -Ptest -Pantcode-release \ No newline at end of file diff --git a/client/all/pom.xml b/client/all/pom.xml index 6c1d73794..0b48a8f37 100644 --- a/client/all/pom.xml +++ b/client/all/pom.xml @@ -6,7 +6,7 @@ com.alipay.sofa registry-client-all - 5.4.5 + 6.0.1 ${project.groupId}:${project.artifactId} http://github.com/alipay/sofa-registry diff --git a/client/api/pom.xml b/client/api/pom.xml index 996a538c0..3464d3eb8 100644 --- a/client/api/pom.xml +++ b/client/api/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-client-parent - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/client/impl/pom.xml b/client/impl/pom.xml index 769c95d4a..b1b92d4e9 100644 --- a/client/impl/pom.xml +++ b/client/impl/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-client-parent - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/client/log/pom.xml b/client/log/pom.xml index 546462dbb..2da3735a5 100644 --- a/client/log/pom.xml +++ b/client/log/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-client-parent - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/client/pom.xml b/client/pom.xml index c1ceecd37..aa30333fe 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -7,7 +7,7 @@ com.alipay.sofa registry-parent - 5.4.5 + 6.0.1 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 39b8e57ba..9d47f61e4 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-parent - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index da389c1af..410dc6657 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 com.alipay.sofa registry-parent - 5.4.5 + 6.0.1 pom ${project.groupId}:${project.artifactId} diff --git a/server/common/model/pom.xml b/server/common/model/pom.xml index 49734dd14..48d0b165b 100644 --- a/server/common/model/pom.xml +++ b/server/common/model/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-common - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 @@ -17,6 +17,12 @@ + + junit + junit + test + + com.alipay.sofa registry-core diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/DatumSummary.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/DatumSummary.java index b57e2f886..76fbfa9de 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/DatumSummary.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/DatumSummary.java @@ -27,7 +27,7 @@ */ public class DatumSummary implements Serializable { private String dataInfoId; - private Map publisherDigests; + private Map publisherDigests = new HashMap<>(); public DatumSummary(String dataInfoId) { this.dataInfoId = dataInfoId; diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/slot/DataSlotDiffUtils.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/slot/DataSlotDiffUtils.java index fd2df6c59..f1c102a52 100644 --- a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/slot/DataSlotDiffUtils.java +++ b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/slot/DataSlotDiffUtils.java @@ -20,6 +20,8 @@ import com.alipay.sofa.registry.common.model.dataserver.DatumSummary; import com.alipay.sofa.registry.common.model.store.Publisher; import com.alipay.sofa.registry.log.Logger; +import com.alipay.sofa.registry.util.CollectionUtils; +import com.google.common.collect.Collections2; import java.util.*; @@ -32,56 +34,66 @@ public final class DataSlotDiffUtils { private DataSlotDiffUtils() { } - public static DataSlotDiffSyncResult diffDataInfoIdsResult(Set dataInfoIds, - Map> existingPublishers, + public static DataSlotDiffSyncResult diffDataInfoIdsResult(Set targetDataInfoIds, + Map> sourcePublishers, int publisherMaxNum) { Map> adds = new HashMap<>(); - for (Map.Entry> e : existingPublishers.entrySet()) { + for (Map.Entry> e : sourcePublishers.entrySet()) { final String dataInfoId = e.getKey(); - if (dataInfoIds.contains(dataInfoId)) { + if (targetDataInfoIds.contains(dataInfoId)) { continue; } - adds.put(dataInfoId, new ArrayList<>(e.getValue().size())); + if (CollectionUtils.isEmpty(e.getValue())) { + continue; + } + adds.put(dataInfoId, new ArrayList<>(e.getValue().values())); } - Map> updateds = new HashMap<>(existingPublishers.size()); + Map> updateds = new HashMap<>(sourcePublishers.size()); int publisherCount = 0; for (Map.Entry> add : adds.entrySet()) { - updateds.put(add.getKey(), add.getValue()); publisherCount += add.getValue().size(); - if (publisherCount > publisherMaxNum) { + + if (updateds.isEmpty()) { + // add at lease one + updateds.put(add.getKey(), add.getValue()); + continue; + } + if (publisherCount >= publisherMaxNum) { // too many publishers, mark has remain break; } + updateds.put(add.getKey(), add.getValue()); } + final boolean hasRemain = updateds.size() != adds.size(); // find the removed dataInfoIds List removeds = new ArrayList<>(); - for (String dataInfoId : dataInfoIds) { - if (!existingPublishers.containsKey(dataInfoId)) { + for (String dataInfoId : targetDataInfoIds) { + if (!sourcePublishers.containsKey(dataInfoId)) { removeds.add(dataInfoId); } } DataSlotDiffSyncResult result = new DataSlotDiffSyncResult(updateds, removeds, - new HashMap<>()); + new HashMap<>()); result.setHasRemain(hasRemain); return result; } - public static DataSlotDiffSyncResult diffPublishersResult(Map datumSummarys, - Map> existingPublishers, - int publisherMaxNum) { - Map> updateds = new HashMap<>(datumSummarys.size()); + public static DataSlotDiffSyncResult diffPublishersResult(Map targetDatumSummarys, + Map> sourcePublishers, + int publisherMaxNum) { + Map> updateds = new HashMap<>(targetDatumSummarys.size()); Map> removedPublishers = new HashMap<>(); List removedDataInfoIds = new ArrayList<>(); int publisherCount = 0; int checkRound = 0; - for (Map.Entry summary : datumSummarys.entrySet()) { + for (Map.Entry summary : targetDatumSummarys.entrySet()) { checkRound++; final String dataInfoId = summary.getKey(); - Map publisherMap = existingPublishers.get(dataInfoId); + Map publisherMap = sourcePublishers.get(dataInfoId); if (publisherMap == null) { // the dataInfoId has removed removedDataInfoIds.add(dataInfoId); @@ -90,7 +102,7 @@ public static DataSlotDiffSyncResult diffPublishersResult(Map registerIds = summary.getValue().getPublisherDigests().keySet(); for (String registerId : registerIds) { if (!publisherMap.containsKey(registerId)) { - List list = removedPublishers.computeIfAbsent(registerId, k -> new ArrayList<>()); + List list = removedPublishers.computeIfAbsent(dataInfoId, k -> new ArrayList<>()); list.add(registerId); } } @@ -114,13 +126,13 @@ public static DataSlotDiffSyncResult diffPublishersResult(Map publisherMaxNum) { + if (publisherCount >= publisherMaxNum) { // too many publishers, mark has remain break; } } // the iter has break - final boolean hasRemian = checkRound != datumSummarys.size(); + final boolean hasRemian = checkRound != targetDatumSummarys.size(); DataSlotDiffSyncResult result = new DataSlotDiffSyncResult(updateds, removedDataInfoIds, removedPublishers); result.setHasRemain(hasRemian); return result; @@ -129,17 +141,17 @@ public static DataSlotDiffSyncResult diffPublishersResult(Map m = Maps.newHashMap(); + m.put("a", 100); + m.put("b", 200); + Map> publishers = randPublishers(m); + // 400 more than 100+200 + DataSlotDiffSyncResult result = DataSlotDiffUtils.diffDataInfoIdsResult(Sets.newHashSet(), publishers, 400); + Assert.assertFalse(result.isHasRemain()); + Assert.assertEquals(result.getUpdatedPublishers().keySet(), m.keySet()); + Assert.assertTrue(result.getRemovedPublishers().isEmpty()); + checkUpdatedSize(m, result); + checkAddedPublisher(publishers, result); + + // collect parts + result = DataSlotDiffUtils.diffDataInfoIdsResult(Sets.newHashSet(), publishers, 200); + Assert.assertTrue(result.isHasRemain()); + // at lease contains one + Assert.assertEquals(result.getUpdatedPublishers().size(), 1); + Assert.assertTrue(result.getRemovedPublishers().isEmpty()); + checkUpdatedSize(m, result); + checkAddedPublisher(publishers, result); + + // collect parts + result = DataSlotDiffUtils.diffDataInfoIdsResult(Sets.newHashSet(), publishers, 1); + Assert.assertTrue(result.isHasRemain()); + // at lease contains one + Assert.assertEquals(result.getUpdatedPublishers().size(), 1); + Assert.assertTrue(result.getRemovedPublishers().isEmpty()); + checkUpdatedSize(m, result); + checkAddedPublisher(publishers, result); + + result = DataSlotDiffUtils.diffDataInfoIdsResult(Sets.newHashSet(), Maps.newHashMap(), 100); + Assert.assertFalse(result.isHasRemain()); + Assert.assertEquals(result.getUpdatedPublishers().size(), 0); + } + + @Test + public void testDiffDataInfoIdsResult_target() { + Map m = Maps.newHashMap(); + m.put("a", 100); + m.put("b", 200); + Map> publishers = randPublishers(m); + + DataSlotDiffSyncResult result = DataSlotDiffUtils + .diffDataInfoIdsResult(Sets.newHashSet("a", "b"), publishers, 400); + Assert.assertFalse(result.isHasRemain()); + Assert.assertTrue(result.getUpdatedPublishers().isEmpty()); + Assert.assertTrue(result.getRemovedPublishers().isEmpty()); + + result = DataSlotDiffUtils + .diffDataInfoIdsResult(Sets.newHashSet("a", "b", "c"), publishers, 400); + Assert.assertFalse(result.isHasRemain()); + Assert.assertTrue(result.getUpdatedPublishers().isEmpty()); + Assert.assertTrue(result.getRemovedPublishers().isEmpty()); + + result = DataSlotDiffUtils + .diffDataInfoIdsResult(Sets.newHashSet("a", "c"), publishers, 100); + Assert.assertFalse(result.isHasRemain()); + Assert.assertTrue(result.getRemovedPublishers().isEmpty()); + // add "b" + Assert.assertEquals(result.getUpdatedPublishers().keySet(), Sets.newHashSet("b")); + checkUpdatedSize(m, result); + checkAddedPublisher(publishers, result); + + // delete "c" + Assert.assertTrue(result.getRemovedDataInfoIds().size() == 1); + Assert.assertTrue(result.getRemovedDataInfoIds().contains("c")); + DataSlotDiffUtils.logDiffResult(result, 10, LOG); + + } + + @Test + public void testDiffPublishersResult() { + Map m = Maps.newHashMap(); + m.put("a", 100); + m.put("b", 200); + + Map> publishers = randPublishers(m); + + // the same + Map summaryMap = createSummary(publishers); + DataSlotDiffSyncResult result = DataSlotDiffUtils.diffPublishersResult(summaryMap, publishers, 400); + Assert.assertFalse(result.isHasRemain()); + Assert.assertTrue(result.getUpdatedPublishers().isEmpty()); + Assert.assertTrue(result.getRemovedPublishers().isEmpty()); + Assert.assertTrue(result.getRemovedDataInfoIds().isEmpty()); + + // now add c, d + Map newM = Maps.newHashMap(); + newM.put("c", 300); + newM.put("d", 400); + m.putAll(newM); + Map> addPublishers = randPublishers(newM); + publishers.putAll(addPublishers); + + // not reach max.publishers + summaryMap.put("c", new DatumSummary("c")); + summaryMap.put("d", new DatumSummary("d")); + result = DataSlotDiffUtils.diffPublishersResult(summaryMap, publishers, 1000); + Assert.assertFalse(result.isHasRemain()); + Assert.assertTrue(result.getRemovedPublishers().isEmpty()); + Assert.assertTrue(result.getRemovedDataInfoIds().isEmpty()); + + Assert.assertEquals(result.getUpdatedPublishers().keySet(), Sets.newHashSet("c", "d")); + checkUpdatedSize(m, result); + checkAddedPublisher(publishers, result); + + // reach max.publishers + result = DataSlotDiffUtils.diffPublishersResult(summaryMap, publishers, 100); + Assert.assertTrue(result.isHasRemain()); + Assert.assertTrue(result.getRemovedPublishers().isEmpty()); + Assert.assertTrue(result.getRemovedDataInfoIds().isEmpty()); + + Assert.assertEquals(result.getUpdatedPublishers().size(), 1); + Assert.assertTrue(result.getUpdatedPublishers().keySet().contains("c") || + result.getUpdatedPublishers().keySet().contains("d")); + + checkUpdatedSize(m, result); + checkAddedPublisher(publishers, result); + + // delete a dataInfoId, delete b.publisher, modify c.publisher, add d.publisher + summaryMap = createSummary(publishers); + publishers.remove("a"); + + Iterator> iter = publishers.get("b").entrySet().iterator(); + Publisher removedB = iter.next().getValue(); + iter.remove(); + + Publisher modifyC = null; + for (Map.Entry e : publishers.get("c").entrySet()) { + modifyC = e.getValue(); + modifyC.setRegisterTimestamp(System.nanoTime()); + break; + } + Publisher addD = randPublisher(); + publishers.get("d").put(addD.getRegisterId(), addD); + + // not reach max + result = DataSlotDiffUtils.diffPublishersResult(summaryMap, publishers, 400); + Assert.assertFalse(result.isHasRemain()); + Assert.assertEquals(result.getRemovedDataInfoIds().size(), 1); + Assert.assertEquals(result.getRemovedDataInfoIds().get(0), "a"); + Assert.assertTrue(result.getRemovedPublishers().get("b").size() == 1); + Assert.assertEquals(result.getRemovedPublishers().get("b").get(0), removedB.getRegisterId()); + + Assert.assertEquals(result.getUpdatedPublishers().keySet(), Sets.newHashSet("c", "d")); + Assert.assertEquals(result.getUpdatedPublishers().get("c").size(), 1); + Assert.assertEquals(result.getUpdatedPublishers().get("d").size(), 1); + Assert.assertEquals(result.getUpdatedPublishers().get("c").get(0), modifyC); + Assert.assertEquals(result.getUpdatedPublishers().get("d").get(0), addD); + + DataSlotDiffUtils.logDiffResult(result, 10, LOG); + + // reach max + result = DataSlotDiffUtils.diffPublishersResult(summaryMap, publishers, 1); + Assert.assertTrue(result.isHasRemain()); + Assert.assertEquals(result.getRemovedDataInfoIds().size(), 1); + Assert.assertEquals(result.getRemovedDataInfoIds().get(0), "a"); + Assert.assertTrue(result.getRemovedPublishers().get("b").size() == 1); + Assert.assertEquals(result.getRemovedPublishers().get("b").get(0), removedB.getRegisterId()); + + Assert.assertTrue( + result.getUpdatedPublishers().containsKey("c") || result.getUpdatedPublishers().containsKey("d")); + + Assert.assertEquals(result.getUpdatedPublishers().size(), 1); + + if (result.getUpdatedPublishers().containsKey("c")) { + Assert.assertEquals(result.getUpdatedPublishers().get("c").size(), 1); + Assert.assertEquals(result.getUpdatedPublishers().get("c").get(0), modifyC); + } + + if (result.getUpdatedPublishers().containsKey("d")) { + Assert.assertEquals(result.getUpdatedPublishers().get("d").size(), 1); + Assert.assertEquals(result.getUpdatedPublishers().get("d").get(0), addD); + } + DataSlotDiffUtils.logDiffResult(result, 10, LOG); + + } + + private static void checkAddedPublisher(Map> publishers, + DataSlotDiffSyncResult result) { + for (Map.Entry> e : result.getUpdatedPublishers().entrySet()) { + Map publisherMap = publishers.get(e.getKey()); + Assert.assertEquals(publisherMap.size(), e.getValue().size()); + for (Publisher p : e.getValue()) { + Assert.assertTrue(p == publisherMap.get(p.getRegisterId())); + } + } + } + + private static void checkUpdatedSize(Map m, DataSlotDiffSyncResult result) { + for (Map.Entry e : m.entrySet()) { + List publisherList = result.getUpdatedPublishers().get(e.getKey()); + if (publisherList != null) { + Assert.assertEquals(e.getValue().intValue(), publisherList.size()); + } + } + } + + private static Map createSummary(Map> publishers) { + Map summaryMap = Maps.newHashMap(); + for (Map.Entry> e : publishers.entrySet()) { + final DatumSummary summary = new DatumSummary(e.getKey()); + summary.setPublisherDigests(Maps.newHashMap()); + e.getValue().forEach((k, p) -> { + summary.getPublisherDigests().put(p.getRegisterId(), PublisherDigestUtil.getDigestValue(p)); + }); + summaryMap.put(e.getKey(), summary); + } + return summaryMap; + } + + private static Publisher randPublisher() { + Publisher p = new Publisher(); + p.setRegisterTimestamp(System.nanoTime()); + p.setRegisterId(System.currentTimeMillis() + "_" + SEQ.incrementAndGet()); + p.setVersion(1L); + return p; + } + + private static Map> randPublishers(Map publishers) { + Map> m = Maps.newHashMap(); + publishers.forEach((k, i) -> { + Map publisherMap = Maps.newHashMap(); + for (int j = 0; j < i; j++) { + Publisher p = randPublisher(); + publisherMap.put(p.getRegisterId(), p); + } + m.put(k, publisherMap); + }); + return m; + } +} diff --git a/server/common/pom.xml b/server/common/pom.xml index 40b568ca7..1eddf4d50 100644 --- a/server/common/pom.xml +++ b/server/common/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server-parent - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/common/util/pom.xml b/server/common/util/pom.xml index fd262ae30..9a8c2d183 100644 --- a/server/common/util/pom.xml +++ b/server/common/util/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-common - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/common/util/src/main/java/com/alipay/sofa/registry/util/CollectionUtils.java b/server/common/util/src/main/java/com/alipay/sofa/registry/util/CollectionUtils.java index cc6311dd6..8aed3bd6b 100644 --- a/server/common/util/src/main/java/com/alipay/sofa/registry/util/CollectionUtils.java +++ b/server/common/util/src/main/java/com/alipay/sofa/registry/util/CollectionUtils.java @@ -17,6 +17,7 @@ package com.alipay.sofa.registry.util; import java.util.Collection; +import java.util.Map; import java.util.Optional; /** @@ -37,4 +38,12 @@ public static Optional getRandom(Collection e) { return e.stream().skip((int) (e.size() * Math.random())).findFirst(); } + public static boolean isEmpty(Collection c) { + return c == null || c.isEmpty(); + } + + public static boolean isEmpty(Map m) { + return m == null || m.isEmpty(); + } + } diff --git a/server/consistency/pom.xml b/server/consistency/pom.xml index af9efc807..a79ddb33d 100644 --- a/server/consistency/pom.xml +++ b/server/consistency/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server-parent - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/distribution/data/pom.xml b/server/distribution/data/pom.xml index 00219e935..e74484256 100644 --- a/server/distribution/data/pom.xml +++ b/server/distribution/data/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-distribution - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/distribution/integration/pom.xml b/server/distribution/integration/pom.xml index d316c6418..7266c979c 100644 --- a/server/distribution/integration/pom.xml +++ b/server/distribution/integration/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-distribution - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/distribution/meta/pom.xml b/server/distribution/meta/pom.xml index de54ed413..3e068a166 100644 --- a/server/distribution/meta/pom.xml +++ b/server/distribution/meta/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-distribution - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/distribution/pom.xml b/server/distribution/pom.xml index 03eb6676e..ca329c4bd 100644 --- a/server/distribution/pom.xml +++ b/server/distribution/pom.xml @@ -6,7 +6,7 @@ com.alipay.sofa registry-server-parent - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/distribution/session/pom.xml b/server/distribution/session/pom.xml index 750e37529..3e4eb7f60 100644 --- a/server/distribution/session/pom.xml +++ b/server/distribution/session/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-distribution - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/pom.xml b/server/pom.xml index cdc71eb9f..4a4346d60 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -7,7 +7,7 @@ com.alipay.sofa registry-parent - 5.4.5 + 6.0.1 ../pom.xml diff --git a/server/remoting/api/pom.xml b/server/remoting/api/pom.xml index fcb259197..ca5e19741 100644 --- a/server/remoting/api/pom.xml +++ b/server/remoting/api/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-remoting - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/remoting/bolt/pom.xml b/server/remoting/bolt/pom.xml index 3c256443a..cccfe62d0 100644 --- a/server/remoting/bolt/pom.xml +++ b/server/remoting/bolt/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-remoting - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/remoting/http/pom.xml b/server/remoting/http/pom.xml index c6f568351..5de220e1b 100644 --- a/server/remoting/http/pom.xml +++ b/server/remoting/http/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-remoting - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/remoting/pom.xml b/server/remoting/pom.xml index d34affeb3..2f79d8635 100644 --- a/server/remoting/pom.xml +++ b/server/remoting/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server-parent - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/server/data/pom.xml b/server/server/data/pom.xml index e2f44c074..10007c6fc 100644 --- a/server/server/data/pom.xml +++ b/server/server/data/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerConfig.java b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerConfig.java index e04db95ee..93b0269c3 100644 --- a/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerConfig.java +++ b/server/server/data/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerConfig.java @@ -127,8 +127,8 @@ public class DataServerConfig { private int slotFollowerSyncLeaderExecutorQueueSize = 10000; private int slotFollowerSyncLeaderIntervalMs = 60000; - // the publisher.digest if len(registerId+long), 100bytes - private int slotSyncPublisherDigestMaxNum = 1024 * 4; + // the publisher.digest if len(registerId/uuid+long), 50bytes + private int slotSyncPublisherDigestMaxNum = 10000; private int slotSyncPublisherMaxNum = 512; diff --git a/server/server/data/src/test/java/com/alipay/sofa/registry/server/data/MigrateTest.java b/server/server/data/src/test/java/com/alipay/sofa/registry/server/data/MigrateTest.java index 796605f6d..be695232b 100644 --- a/server/server/data/src/test/java/com/alipay/sofa/registry/server/data/MigrateTest.java +++ b/server/server/data/src/test/java/com/alipay/sofa/registry/server/data/MigrateTest.java @@ -18,7 +18,7 @@ import com.alipay.remoting.serialization.HessianSerializer; import com.alipay.sofa.registry.common.model.dataserver.DatumSummary; -import com.alipay.sofa.registry.common.model.sessionserver.DataSlotMigrateRequest; +import com.alipay.sofa.registry.common.model.slot.DataSlotDiffPublisherRequest; import org.junit.Test; import java.io.BufferedOutputStream; @@ -37,8 +37,8 @@ public class MigrateTest { @Test public void testBody() throws Exception { HessianSerializer s = new HessianSerializer(); - DataSlotMigrateRequest request = new DataSlotMigrateRequest(100, 200, new HashMap<>()); - for (int i = 0; i < 5; i++) { + DataSlotDiffPublisherRequest request = new DataSlotDiffPublisherRequest(100, 200, new HashMap<>()); + for (int i = 0; i < 10; i++) { DatumSummary summary = new DatumSummary("app" + System.currentTimeMillis()); summary.setPublisherDigests(new HashMap<>()); for (int j = 0; j < 1000; j++) { diff --git a/server/server/integration/pom.xml b/server/server/integration/pom.xml index d96716d9b..bc3c7b4fb 100644 --- a/server/server/integration/pom.xml +++ b/server/server/integration/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 @@ -81,4 +81,4 @@ - \ No newline at end of file + diff --git a/server/server/meta/pom.xml b/server/server/meta/pom.xml index 3460620f5..e6270f744 100644 --- a/server/server/meta/pom.xml +++ b/server/server/meta/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/server/pom.xml b/server/server/pom.xml index 7fe829ef7..df7ad0429 100644 --- a/server/server/pom.xml +++ b/server/server/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server-parent - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/server/session/pom.xml b/server/server/session/pom.xml index 2a2f43725..035c2884f 100644 --- a/server/server/session/pom.xml +++ b/server/server/session/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/store/api/pom.xml b/server/store/api/pom.xml index 70179d828..50413f594 100644 --- a/server/store/api/pom.xml +++ b/server/store/api/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-store - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/server/store/jraft/pom.xml b/server/store/jraft/pom.xml index 7ecb15135..cad522918 100644 --- a/server/store/jraft/pom.xml +++ b/server/store/jraft/pom.xml @@ -6,7 +6,7 @@ com.alipay.sofa registry-store - 5.4.5 + 6.0.1 ../pom.xml diff --git a/server/store/pom.xml b/server/store/pom.xml index b02b5c0ef..7d3dfe991 100644 --- a/server/store/pom.xml +++ b/server/store/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-server-parent - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 diff --git a/test/pom.xml b/test/pom.xml index 9f5cd48b9..f7f14c49f 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -5,7 +5,7 @@ com.alipay.sofa registry-parent - 5.4.5 + 6.0.1 ../pom.xml 4.0.0 @@ -80,4 +80,4 @@ test - \ No newline at end of file + diff --git a/tools/change_version.sh b/tools/change_version.sh old mode 100644 new mode 100755 diff --git a/tools/check_format.sh b/tools/check_format.sh old mode 100644 new mode 100755