PullRequest: 2 add test for slot store

Signed-off-by: 向旭 <[email protected]> * What type of PR is this? add test for slot What this PR does / why we need it: Which issue(s) this PR fixes: Special notes for your reviewer: Does this PR introduce a user-facing change?: What type of PR is this? add test for slotstore What this PR does / why we need it: Which issue(s) this PR fixes: Special notes for your reviewer: Does this PR introduce a user-facing change?:
sofastack · Nov 27, 2020 · b67cac0 · b67cac0
1 parent 64e76ff
commit b67cac0
Show file tree

Hide file tree

Showing 40 changed files with 468 additions and 63 deletions.
diff --git a/.aci.yml b/.aci.yml
@@ -0,0 +1,51 @@
+stages: # 定义了一条pipeline
+  - 代码检测 # 定义了stage
+  - 单元测试
+  - 编译
+  #- 发布JAR包
+
+PMD扫描: # 定义了一个job
+  stage: 代码检测 # 绑定到stage
+  plugin: PMD # 指定使用的插件
+  checkRule: # 校验结果
+    - pmd1 = 0 && pmd2 = 0
+
+STC扫描:
+  stage: 代码检测
+  plugin: STC
+  checkRule:
+    - stc = 0
+
+单元测试:
+  stage: 单元测试
+  plugin: UT
+  checkRule:
+    - passRate = 100
+  pluginConfig:
+    markdown: admin/result.md # 向页面输出一些自定义内容
+
+编译:
+  stage: 编译
+  plugin: ANT-BUILD         # 使用插件ANT-BUILD
+  pluginConfig:
+    image: reg.docker.alibaba-inc.com/antb/sofa-build:0.0.2
+    script:
+      - export PATH=$PATH:/opt/taobao/java/bin:/opt/taobao/maven/bin
+      - java -version
+      - mvn -version
+      - mvn clean package -Dmaven.test.skip=true -Pdev
+    outputs:
+      - name: registry.tgz
+        paths: ["./distribution-enterprise/alipay/target/registry.tgz"]
+
+发布JAR包:
+  stage: 发布JAR包
+  plugin: ANT-BUILD
+  pluginConfig:
+    image: reg.docker.alibaba-inc.com/antb/jarbuild:0.0.1
+    script:
+      - java -version
+      - mvn -version
+      - sed -i '/<mirrors>/,/<\/mirrors>/d' /opt/taobao/maven_settings/settings-release.xml
+      - sed -i '/<profiles>/,/<\/profiles>/d' /opt/taobao/maven_settings/settings-release.xml
+      - mvn clean deploy -Dmaven.test.skip=true -s /opt/taobao/maven_settings/settings-release.xml -Pdev -Pantcode-release
diff --git a/.master.aci.yml b/.master.aci.yml
@@ -0,0 +1,51 @@
+stages: # 定义了一条pipeline
+  - 代码检测 # 定义了stage
+  - 单元测试
+  - 编译
+  #- 发布JAR包
+
+PMD扫描: # 定义了一个job
+  stage: 代码检测 # 绑定到stage
+  plugin: PMD # 指定使用的插件
+  checkRule: # 校验结果
+    - pmd1 = 0 && pmd2 = 0
+
+STC扫描:
+  stage: 代码检测
+  plugin: STC
+  checkRule:
+    - stc = 0
+
+单元测试:
+  stage: 单元测试
+  plugin: UT
+  checkRule:
+    - passRate = 100
+  pluginConfig:
+    markdown: admin/result.md # 向页面输出一些自定义内容
+
+编译:
+  stage: 编译
+  plugin: ANT-BUILD         # 使用插件ANT-BUILD
+  pluginConfig:
+    image: reg.docker.alibaba-inc.com/antb/sofa-build:0.0.2
+    script:
+      - export PATH=$PATH:/opt/taobao/java/bin:/opt/taobao/maven/bin
+      - java -version
+      - mvn -version
+      - mvn clean package -Dmaven.test.skip=true -Ptest
+    outputs:
+      - name: registry.tgz
+        paths: ["./distribution-enterprise/alipay/target/registry.tgz"]
+
+发布JAR包:
+  stage: 发布JAR包
+  plugin: ANT-BUILD
+  pluginConfig:
+    image: reg.docker.alibaba-inc.com/antb/jarbuild:0.0.1
+    script:
+      - java -version
+      - mvn -version
+      - sed -i '/<mirrors>/,/<\/mirrors>/d' /opt/taobao/maven_settings/settings-release.xml
+      - sed -i '/<profiles>/,/<\/profiles>/d' /opt/taobao/maven_settings/settings-release.xml
+      - mvn clean deploy -Dmaven.test.skip=true -s /opt/taobao/maven_settings/settings-release.xml -Ptest -Pantcode-release
diff --git a/client/all/pom.xml b/client/all/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.alipay.sofa</groupId>
     <artifactId>registry-client-all</artifactId>
-    <version>5.4.5</version>
+    <version>6.0.1</version>
 
     <name>${project.groupId}:${project.artifactId}</name>
     <url>http://github.com/alipay/sofa-registry</url>

diff --git a/client/api/pom.xml b/client/api/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.alipay.sofa</groupId>
         <artifactId>registry-client-parent</artifactId>
-        <version>5.4.5</version>
+        <version>6.0.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/client/impl/pom.xml b/client/impl/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.alipay.sofa</groupId>
         <artifactId>registry-client-parent</artifactId>
-        <version>5.4.5</version>
+        <version>6.0.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/client/log/pom.xml b/client/log/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.alipay.sofa</groupId>
         <artifactId>registry-client-parent</artifactId>
-        <version>5.4.5</version>
+        <version>6.0.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/client/pom.xml b/client/pom.xml
@@ -7,7 +7,7 @@
     <parent>
         <groupId>com.alipay.sofa</groupId>
         <artifactId>registry-parent</artifactId>
-        <version>5.4.5</version>
+        <version>6.0.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/core/pom.xml b/core/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.alipay.sofa</groupId>
         <artifactId>registry-parent</artifactId>
-        <version>5.4.5</version>
+        <version>6.0.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.alipay.sofa</groupId>
     <artifactId>registry-parent</artifactId>
-    <version>5.4.5</version>
+    <version>6.0.1</version>
     <packaging>pom</packaging>
 
     <name>${project.groupId}:${project.artifactId}</name>

diff --git a/server/common/model/pom.xml b/server/common/model/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.alipay.sofa</groupId>
         <artifactId>registry-common</artifactId>
-        <version>5.4.5</version>
+        <version>6.0.1</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -17,6 +17,12 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>com.alipay.sofa</groupId>
             <artifactId>registry-core</artifactId>

diff --git a/...on/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/DatumSummary.java b/...on/model/src/main/java/com/alipay/sofa/registry/common/model/dataserver/DatumSummary.java
@@ -27,7 +27,7 @@
  */
 public class DatumSummary implements Serializable {
     private String                           dataInfoId;
-    private Map<String/*registerId*/, Long> publisherDigests;
+    private Map<String/*registerId*/, Long> publisherDigests = new HashMap<>();
 
     public DatumSummary(String dataInfoId) {
         this.dataInfoId = dataInfoId;

diff --git a/...mon/model/src/main/java/com/alipay/sofa/registry/common/model/slot/DataSlotDiffUtils.java b/...mon/model/src/main/java/com/alipay/sofa/registry/common/model/slot/DataSlotDiffUtils.java
@@ -20,6 +20,8 @@
 import com.alipay.sofa.registry.common.model.dataserver.DatumSummary;
 import com.alipay.sofa.registry.common.model.store.Publisher;
 import com.alipay.sofa.registry.log.Logger;
+import com.alipay.sofa.registry.util.CollectionUtils;
+import com.google.common.collect.Collections2;
 
 import java.util.*;
 
@@ -32,56 +34,66 @@ public final class DataSlotDiffUtils {
     private DataSlotDiffUtils() {
     }
 
-    public static DataSlotDiffSyncResult diffDataInfoIdsResult(Set<String> dataInfoIds,
-                                                               Map<String, Map<String, Publisher>> existingPublishers,
+    public static DataSlotDiffSyncResult diffDataInfoIdsResult(Set<String> targetDataInfoIds,
+                                                               Map<String, Map<String, Publisher>> sourcePublishers,
                                                                int publisherMaxNum) {
         Map<String, List<Publisher>> adds = new HashMap<>();
-        for (Map.Entry<String, Map<String, Publisher>> e : existingPublishers.entrySet()) {
+        for (Map.Entry<String, Map<String, Publisher>> e : sourcePublishers.entrySet()) {
             final String dataInfoId = e.getKey();
-            if (dataInfoIds.contains(dataInfoId)) {
+            if (targetDataInfoIds.contains(dataInfoId)) {
                 continue;
             }
-            adds.put(dataInfoId, new ArrayList<>(e.getValue().size()));
+            if (CollectionUtils.isEmpty(e.getValue())) {
+                continue;
+            }
+            adds.put(dataInfoId, new ArrayList<>(e.getValue().values()));
         }
 
-        Map<String, List<Publisher>> updateds = new HashMap<>(existingPublishers.size());
+        Map<String, List<Publisher>> updateds = new HashMap<>(sourcePublishers.size());
         int publisherCount = 0;
 
         for (Map.Entry<String, List<Publisher>> add : adds.entrySet()) {
-            updateds.put(add.getKey(), add.getValue());
             publisherCount += add.getValue().size();
-            if (publisherCount > publisherMaxNum) {
+
+            if (updateds.isEmpty()) {
+                // add at lease one
+                updateds.put(add.getKey(), add.getValue());
+                continue;
+            }
+            if (publisherCount >= publisherMaxNum) {
                 // too many publishers, mark has remain
                 break;
             }
+            updateds.put(add.getKey(), add.getValue());
         }
+
         final boolean hasRemain = updateds.size() != adds.size();
         // find the removed dataInfoIds
         List<String> removeds = new ArrayList<>();
-        for (String dataInfoId : dataInfoIds) {
-            if (!existingPublishers.containsKey(dataInfoId)) {
+        for (String dataInfoId : targetDataInfoIds) {
+            if (!sourcePublishers.containsKey(dataInfoId)) {
                 removeds.add(dataInfoId);
             }
         }
         DataSlotDiffSyncResult result = new DataSlotDiffSyncResult(updateds, removeds,
-            new HashMap<>());
+                new HashMap<>());
         result.setHasRemain(hasRemain);
         return result;
     }
 
-    public static DataSlotDiffSyncResult diffPublishersResult(Map<String, DatumSummary> datumSummarys,
-                                                    Map<String, Map<String, Publisher>> existingPublishers,
-                                                    int publisherMaxNum) {
-        Map<String, List<Publisher>> updateds = new HashMap<>(datumSummarys.size());
+    public static DataSlotDiffSyncResult diffPublishersResult(Map<String, DatumSummary> targetDatumSummarys,
+                                                              Map<String, Map<String, Publisher>> sourcePublishers,
+                                                              int publisherMaxNum) {
+        Map<String, List<Publisher>> updateds = new HashMap<>(targetDatumSummarys.size());
         Map<String, List<String>> removedPublishers = new HashMap<>();
         List<String> removedDataInfoIds = new ArrayList<>();
 
         int publisherCount = 0;
         int checkRound = 0;
-        for (Map.Entry<String, DatumSummary> summary : datumSummarys.entrySet()) {
+        for (Map.Entry<String, DatumSummary> summary : targetDatumSummarys.entrySet()) {
             checkRound++;
             final String dataInfoId = summary.getKey();
-            Map<String, Publisher> publisherMap = existingPublishers.get(dataInfoId);
+            Map<String, Publisher> publisherMap = sourcePublishers.get(dataInfoId);
             if (publisherMap == null) {
                 // the dataInfoId has removed
                 removedDataInfoIds.add(dataInfoId);
@@ -90,7 +102,7 @@ public static DataSlotDiffSyncResult diffPublishersResult(Map<String, DatumSumma
             Set<String> registerIds = summary.getValue().getPublisherDigests().keySet();
             for (String registerId : registerIds) {
                 if (!publisherMap.containsKey(registerId)) {
-                    List<String> list = removedPublishers.computeIfAbsent(registerId, k -> new ArrayList<>());
+                    List<String> list = removedPublishers.computeIfAbsent(dataInfoId, k -> new ArrayList<>());
                     list.add(registerId);
                 }
             }
@@ -114,13 +126,13 @@ public static DataSlotDiffSyncResult diffPublishersResult(Map<String, DatumSumma
                 publisherCount += publishers.size();
                 updateds.put(dataInfoId, publishers);
             }
-            if (publisherCount > publisherMaxNum) {
+            if (publisherCount >= publisherMaxNum) {
                 // too many publishers, mark has remain
                 break;
             }
         }
         // the iter has break
-        final boolean hasRemian = checkRound != datumSummarys.size();
+        final boolean hasRemian = checkRound != targetDatumSummarys.size();
         DataSlotDiffSyncResult result = new DataSlotDiffSyncResult(updateds, removedDataInfoIds, removedPublishers);
         result.setHasRemain(hasRemian);
         return result;
@@ -129,17 +141,17 @@ public static DataSlotDiffSyncResult diffPublishersResult(Map<String, DatumSumma
     public static void logDiffResult(DataSlotDiffSyncResult result, int slotId, Logger log) {
         if (!result.getUpdatedPublishers().isEmpty()) {
             log.info("DiffSync, update dataInfoIds for slot={}, remain={}, dataInfoIds={}/{}, {}",
-                slotId, result.isHasRemain(), result.getUpdatedPublishers().size(),
-                result.getUpdatedPublishersCount(), result.getUpdatedPublishers().keySet());
+                    slotId, result.isHasRemain(), result.getUpdatedPublishers().size(),
+                    result.getUpdatedPublishersCount(), result.getUpdatedPublishers().keySet());
         }
         if (!result.getRemovedPublishers().isEmpty()) {
             log.info("DiffSync, remove publishers for slot={}, dataInfoId={}/{}, {}", slotId,
-                result.getRemovedPublishers().size(), result.getRemovedPublishersCount(), result
-                    .getRemovedPublishers().keySet());
+                    result.getRemovedPublishers().size(), result.getRemovedPublishersCount(), result
+                            .getRemovedPublishers().keySet());
         }
         if (!result.getRemovedDataInfoIds().isEmpty()) {
             log.info("DiffSync, remove dataInfoIds for slot={}, dataInfoId={}, {}", slotId, result
-                .getRemovedDataInfoIds().size(), result.getRemovedDataInfoIds());
+                    .getRemovedDataInfoIds().size(), result.getRemovedDataInfoIds());
         }
     }
 }