retry && exit on bootstrap failed

sofastack · Feb 3, 2021 · 375532b · 375532b
1 parent 024bbd4
commit 375532b
Show file tree

Hide file tree

Showing 15 changed files with 155 additions and 34 deletions.
diff --git a/pom.xml b/pom.xml
@@ -80,6 +80,7 @@
         <commons-io.version>2.4</commons-io.version>
         <jetty.version>[9.4.17.v20190418,9.4.19.v20190610]</jetty.version>
         <rocksdbjni.version>6.4.6</rocksdbjni.version>
+        <guava-retrying.version>2.0.0</guava-retrying.version>
         <main.user.dir>${user.dir}</main.user.dir>
         <argLine>-Dnetwork_interface_denylist=docker0</argLine>
         <!-- for linke 添加isSkipUT, isSkipIT参数声明，防止编译时产生错误 -->
@@ -374,6 +375,11 @@
                 <artifactId>rocksdbjni</artifactId>
                 <version>${rocksdbjni.version}</version>
             </dependency>
+            <dependency>
+                <groupId>com.github.rholder</groupId>
+                <artifactId>guava-retrying</artifactId>
+                <version>${guava-retrying.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 

diff --git a/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/DataUtils.java b/server/common/model/src/main/java/com/alipay/sofa/registry/common/model/DataUtils.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package com.alipay.sofa.registry.common.model;
 
 import com.alipay.sofa.registry.common.model.store.BaseInfo;

diff --git a/server/common/util/src/main/java/com/alipay/sofa/registry/util/SystemUtils.java b/server/common/util/src/main/java/com/alipay/sofa/registry/util/SystemUtils.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package com.alipay.sofa.registry.util;
 
 public final class SystemUtils {

diff --git a/server/server/data/pom.xml b/server/server/data/pom.xml
@@ -87,6 +87,10 @@
             <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.github.rholder</groupId>
+            <artifactId>guava-retrying</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/...ata/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerBootstrap.java b/...ata/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerBootstrap.java
@@ -28,6 +28,10 @@
 import com.alipay.sofa.registry.remoting.exchange.Exchange;
 import com.alipay.sofa.registry.server.shared.meta.MetaServerService;
 import com.alipay.sofa.registry.server.shared.remoting.AbstractServerHandler;
+import com.github.rholder.retry.Retryer;
+import com.github.rholder.retry.RetryerBuilder;
+import com.github.rholder.retry.StopStrategies;
+import com.github.rholder.retry.WaitStrategies;
 import org.glassfish.jersey.server.ResourceConfig;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -40,11 +44,10 @@
 import java.util.Collection;
 import java.util.Date;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
- *
- *
  * @author qian.lqlq
  * @version $Id: DataServerBootstrap.java, v 0.1 2017-12-06 20:50 qian.lqlq Exp $
  */
@@ -91,6 +94,20 @@ public class DataServerBootstrap {
 
     private final AtomicBoolean               serverForDataSyncStarted = new AtomicBoolean(false);
 
+    private final Retryer<Boolean>            retryer                  = RetryerBuilder
+                                                                           .<Boolean> newBuilder()
+                                                                           .retryIfException()
+                                                                           .withWaitStrategy(
+                                                                               WaitStrategies
+                                                                                   .exponentialWait(
+                                                                                       1000,
+                                                                                       10000,
+                                                                                       TimeUnit.MILLISECONDS))
+                                                                           .withStopStrategy(
+                                                                               StopStrategies
+                                                                                   .stopAfterAttempt(10))
+                                                                           .build();
+
     /**
      * start dataserver
      */
@@ -108,10 +125,13 @@ public void start() {
 
             openHttpServer();
 
-            startRaftClient();
 
-            renewNode();
+            retryer.call(() -> {
+                startRaftClient();
+                return true;
+            });
 
+            renewNode();
             fetchProviderData();
 
             startScheduler();

diff --git a/...a/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerInitializer.java b/...a/src/main/java/com/alipay/sofa/registry/server/data/bootstrap/DataServerInitializer.java
@@ -16,15 +16,17 @@
  */
 package com.alipay.sofa.registry.server.data.bootstrap;
 
+import com.alipay.sofa.registry.log.Logger;
+import com.alipay.sofa.registry.log.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.SmartLifecycle;
 
 /**
- *
  * @author qian.lqlq
  * @version $Id: DataServerInitializer.java, v 0.1 2018年01月04日 11:08 qian.lqlq Exp $
  */
 public class DataServerInitializer implements SmartLifecycle {
+    private static final Logger LOGGER = LoggerFactory.getLogger(DataServerInitializer.class);
 
     @Autowired
     private DataServerBootstrap dataServerBootstrap;
@@ -44,8 +46,14 @@ public void stop(Runnable runnable) {
 
     @Override
     public void start() {
-        dataServerBootstrap.start();
-        this.isRunning = true;
+        try {
+            dataServerBootstrap.start();
+            this.isRunning = true;
+        } catch (Throwable ex) {
+            this.isRunning = false;
+            LOGGER.error("Could not initalized Data server", ex);
+            System.exit(-1);
+        }
     }
 
     @Override

diff --git a/server/server/data/src/main/resources/logback-spring.xml b/server/server/data/src/main/resources/logback-spring.xml
@@ -188,6 +188,10 @@
             <charset>${LOG_ENCODE}</charset>
         </encoder>
     </appender>
+    <appender name ="ASYNC-DATA-DATUM-CHANGE-APPENDER" class= "ch.qos.logback.classic.AsyncAppender">
+        <queueSize>65536</queueSize>
+        <appender-ref ref ="DATA-DATUM-CHANGE-APPENDER"/>
+    </appender>
 
     <appender name ="ASYNC-DATA-STARTUP-APPENDER" class= "ch.qos.logback.classic.AsyncAppender">
         <queueSize>65536</queueSize>

diff --git a/server/server/session/pom.xml b/server/server/session/pom.xml
@@ -86,6 +86,10 @@
             <groupId>commons-lang</groupId>
             <artifactId>commons-lang</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.github.rholder</groupId>
+            <artifactId>guava-retrying</artifactId>
+        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

diff --git a/...c/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerBootstrap.java b/...c/main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerBootstrap.java
@@ -40,6 +40,7 @@
 import com.alipay.sofa.registry.server.shared.meta.MetaServerService;
 import com.alipay.sofa.registry.server.shared.remoting.AbstractServerHandler;
 import com.alipay.sofa.registry.task.batcher.TaskDispatchers;
+import com.github.rholder.retry.*;
 import org.glassfish.jersey.server.ResourceConfig;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationContext;
@@ -51,10 +52,12 @@
 import java.util.Collection;
 import java.util.Date;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * The type Session server bootstrap.
+ *
  * @author shangyu.wh
  * @version $Id : SessionServerBootstrap.java, v 0.1 2017-11-14 11:44 synex Exp $
  */
@@ -119,6 +122,20 @@ public class SessionServerBootstrap {
 
     private final AtomicBoolean               serverForSessionSyncStart = new AtomicBoolean(false);
 
+    private final Retryer<Boolean>            retryer                   = RetryerBuilder
+                                                                            .<Boolean> newBuilder()
+                                                                            .retryIfRuntimeException()
+                                                                            .withWaitStrategy(
+                                                                                WaitStrategies
+                                                                                    .exponentialWait(
+                                                                                        1000,
+                                                                                        10000,
+                                                                                        TimeUnit.MILLISECONDS))
+                                                                            .withStopStrategy(
+                                                                                StopStrategies
+                                                                                    .stopAfterAttempt(10))
+                                                                            .build();
+
     /**
      * Do initialized.
      */
@@ -131,13 +148,20 @@ public void start() {
 
             openSessionSyncServer();
 
-            connectMetaServer();
+
+            retryer.call(() -> {
+                connectMetaServer();
+                return true;
+            });
 
             startScheduler();
 
             openHttpServer();
 
-            connectDataServer();
+            retryer.call(() -> {
+                connectDataServer();
+                return true;
+            });
 
             registerSerializer();
 
@@ -230,9 +254,8 @@ private void openSessionSyncServer() {
 
     private void connectDataServer() {
         try {
-            if (dataStart.compareAndSet(false, true)) {
-                dataNodeExchanger.connectServer();
-            }
+            dataNodeExchanger.connectServer();
+            dataStart.set(true);
         } catch (Exception e) {
             dataStart.set(false);
             LOGGER.error("Data server connected server error! port:{}",
@@ -243,20 +266,19 @@ private void connectDataServer() {
 
     private void connectMetaServer() {
         try {
-            if (metaStart.compareAndSet(false, true)) {
-                mataNodeService.startRaftClient();
-                // register node as renew node
-                mataNodeService.renewNode();
-                // start sched renew
-                mataNodeService
-                    .startRenewer(sessionServerConfig.getSchedulerHeartbeatIntervalSec() * 1000);
-                fetchStopPushSwitch();
-
-                fetchBlackList();
-
-                LOGGER.info("MetaServer connected meta server! Port:{}",
-                    sessionServerConfig.getMetaServerPort());
-            }
+            mataNodeService.startRaftClient();
+            // register node as renew node
+            mataNodeService.renewNode();
+            // start sched renew
+            mataNodeService
+                .startRenewer(sessionServerConfig.getSchedulerHeartbeatIntervalSec() * 1000);
+            fetchStopPushSwitch();
+
+            fetchBlackList();
+            metaStart.set(true);
+
+            LOGGER.info("MetaServer connected meta server! Port:{}",
+                sessionServerConfig.getMetaServerPort());
         } catch (Exception e) {
             metaStart.set(false);
             LOGGER.error("MetaServer connected server error! Port:{}",

diff --git a/...in/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfiguration.java b/...in/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerConfiguration.java
@@ -410,11 +410,6 @@ public ClientNodeService clientNodeService() {
             return new ClientNodeServiceImpl();
         }
 
-        @Bean
-        public RaftClientManager raftClientManager() {
-            return new RaftClientManager();
-        }
-
         @Bean
         public AppRevisionNodeService appRevisionNodeService() {
             return new AppRevisionNodeServiceImpl();

diff --git a/...main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerInitializer.java b/...main/java/com/alipay/sofa/registry/server/session/bootstrap/SessionServerInitializer.java
@@ -26,6 +26,7 @@
 
 /**
  * SmartLifecycle for SessionServerBootstrap
+ *
  * @author shangyu.wh
  * @version $Id: SessionServerInitializerConfiguration.java, v 0.1 2017-11-14 11:41 synex Exp $
  */
@@ -51,9 +52,10 @@ public void start() {
             LOGGER.info("Started SessionServer");
 
             SessionServerInitializer.this.running.set(true);
-        } catch (Exception ex) {
+        } catch (Throwable ex) {
             SessionServerInitializer.this.running.set(false);
             LOGGER.error("Could not initialize Session server!", ex);
+            System.exit(-1);
         }
     }
 

diff --git a/...src/main/java/com/alipay/sofa/registry/server/session/cache/AppRevisionCacheRegistry.java b/...src/main/java/com/alipay/sofa/registry/server/session/cache/AppRevisionCacheRegistry.java
@@ -21,6 +21,7 @@
 import com.alipay.sofa.registry.core.model.AppRevisionInterface;
 import com.alipay.sofa.registry.log.Logger;
 import com.alipay.sofa.registry.log.LoggerFactory;
+import com.alipay.sofa.registry.server.session.node.RaftClientManager;
 import com.alipay.sofa.registry.server.session.node.service.AppRevisionNodeService;
 import com.alipay.sofa.registry.util.ConcurrentUtils;
 import com.alipay.sofa.registry.util.LoopRunnable;
@@ -43,6 +44,9 @@ public class AppRevisionCacheRegistry {
     @Autowired
     private AppRevisionNodeService                                                  appRevisionNodeService;
 
+    @Autowired
+    private RaftClientManager                                                       raftClientManager;
+
     final private Map<String /*revision*/, AppRevision>                            registry           = new ConcurrentHashMap<>();
     private volatile String                                                         keysDigest         = "";
     final private Map<String /*interface*/, Map<String /*appname*/, Set<String>>> interfaceRevisions = new ConcurrentHashMap<>();
@@ -58,6 +62,10 @@ public void init() {
     private final class RevisionWatchDog extends LoopRunnable {
         @Override
         public void runUnthrowable() {
+            if (raftClientManager.getLeader() == null) {
+                LOG.warn("meta has not leader, skip refresh revisions");
+                return;
+            }
             refreshAll();
         }
 

diff --git a/...java/com/alipay/sofa/registry/server/session/node/service/AppRevisionNodeServiceImpl.java b/...java/com/alipay/sofa/registry/server/session/node/service/AppRevisionNodeServiceImpl.java
@@ -44,7 +44,7 @@ public class AppRevisionNodeServiceImpl implements AppRevisionNodeService {
     protected NodeExchanger       metaNodeExchanger;
 
     @Autowired
-    RaftClientManager             raftClientManager;
+    private RaftClientManager     raftClientManager;
 
     public void register(AppRevision appRevision) {
         Request<AppRevision> request = new Request<AppRevision>() {

diff --git a/...ion/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/RegisterLogs.java b/...ion/src/main/java/com/alipay/sofa/registry/server/session/strategy/impl/RegisterLogs.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package com.alipay.sofa.registry.server.session.strategy.impl;
 
 import com.alipay.sofa.registry.core.model.BaseRegister;

diff --git a/server/server/session/src/main/resources/logback-spring.xml b/server/server/session/src/main/resources/logback-spring.xml
@@ -392,7 +392,7 @@
 
     <logger name="SESSION-PUSH" level="${LOG_LEVEL}" additivity="false">
         <appender-ref ref="ASYNC-SESSION-PUSH-APPENDER"/>
-        <appender-ref ref="ASYNC-SESSION-PUSH-ERROR"/>
+        <appender-ref ref="ASYNC-ERROR-PUSH-APPENDER"/>
     </logger>
 
     <logger name="SESSION-PROFILE-DIGEST" level="${LOG_LEVEL}" additivity="false">