Readiness admin panel resilient to data center unavailability (#1658)

allegro · Apr 3, 2023 · f11cc0f · f11cc0f
1 parent 3221a44
commit f11cc0f
Show file tree

Hide file tree

Showing 41 changed files with 499 additions and 321 deletions.
diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/DatacenterReadiness.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/DatacenterReadiness.java
@@ -7,28 +7,28 @@
 
 public class DatacenterReadiness {
     private final String datacenter;
-    private final boolean isReady;
+    private final ReadinessStatus status;
 
     @JsonCreator
-    public DatacenterReadiness(@JsonProperty("datacenter") String datacenter, @JsonProperty("isReady") boolean isReady) {
+    public DatacenterReadiness(@JsonProperty("datacenter") String datacenter,
+                               @JsonProperty("status") ReadinessStatus status) {
         this.datacenter = datacenter;
-        this.isReady = isReady;
+        this.status = status;
     }
 
     public String getDatacenter() {
         return datacenter;
     }
 
-    @JsonProperty("isReady")
-    public boolean isReady() {
-        return isReady;
+    public ReadinessStatus getStatus() {
+        return status;
     }
 
     @Override
     public String toString() {
         return "DatacenterReadiness{"
                 + "datacenter='" + datacenter + '\''
-                + ", isReady=" + isReady
+                + ", status=" + status
                 + '}';
     }
 
@@ -41,12 +41,18 @@ public boolean equals(Object o) {
             return false;
         }
         DatacenterReadiness that = (DatacenterReadiness) o;
-        return isReady == that.isReady
+        return status == that.status
                 && Objects.equals(datacenter, that.datacenter);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(datacenter, isReady);
+        return Objects.hash(datacenter, status);
+    }
+
+    public enum ReadinessStatus {
+        READY,
+        NOT_READY,
+        UNDEFINED
     }
 }
diff --git a/hermes-api/src/main/java/pl/allegro/tech/hermes/api/endpoints/ModeEndpoint.java b/hermes-api/src/main/java/pl/allegro/tech/hermes/api/endpoints/ModeEndpoint.java
@@ -8,12 +8,13 @@
 import javax.ws.rs.core.Response;
 
 import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
+import static javax.ws.rs.core.MediaType.TEXT_PLAIN;
 
 @Path("mode")
 public interface ModeEndpoint {
 
     @GET
-    @Produces(APPLICATION_JSON)
+    @Produces(TEXT_PLAIN)
     String getMode();
 
     @POST

diff --git a/...c/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperBasedRepository.java b/...c/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperBasedRepository.java
@@ -23,9 +23,9 @@ public abstract class ZookeeperBasedRepository {
 
     private static final Logger logger = LoggerFactory.getLogger(ZookeeperBasedRepository.class);
 
-    protected final CuratorFramework zookeeper;
+    private final CuratorFramework zookeeper;
 
-    protected final ObjectMapper mapper;
+    private final ObjectMapper mapper;
 
     protected final ZookeeperPaths paths;
 
@@ -37,7 +37,7 @@ protected ZookeeperBasedRepository(CuratorFramework zookeeper,
         this.paths = paths;
     }
 
-    protected void ensureConnected() {
+    private void ensureConnected() {
         if (!zookeeper.getZookeeperClient().isConnected()) {
             throw new RepositoryNotAvailableException("Could not establish connection to a Zookeeper instance");
         }
@@ -65,6 +65,7 @@ protected boolean pathExists(String path) {
     }
 
     protected List<String> childrenOf(String path) {
+        ensureConnected();
         try {
             List<String> retrievedNodes = new ArrayList<>(zookeeper.getChildren().forPath(path));
             Collections.sort(retrievedNodes);
@@ -74,6 +75,11 @@ protected List<String> childrenOf(String path) {
         }
     }
 
+    @SuppressWarnings("unchecked")
+    protected byte[] readFrom(String path) {
+        return readWithStatFrom(path, bytes -> bytes, (t, stat) -> {}, false).get();
+    }
+
     @SuppressWarnings("unchecked")
     protected <T> T readFrom(String path, Class<T> clazz) {
         return readFrom(path, clazz, false).get();
@@ -94,6 +100,7 @@ protected <T> Optional<T> readWithStatFrom(String path, Class<T> clazz, BiConsum
     }
 
     private <T> Optional<T> readWithStatFrom(String path, ThrowingReader<T> supplier, BiConsumer<T, Stat> statDecorator, boolean quiet) {
+        ensureConnected();
         try {
             Stat stat = new Stat();
             byte[] data = zookeeper.getData().storingStatIn(stat).forPath(path);
@@ -122,38 +129,52 @@ private void logWarnOrThrowException(String message, RuntimeException e, Boolean
         }
     }
 
-    protected void overwrite(String path, Object value) {
-        try {
-            zookeeper.setData().forPath(path, mapper.writeValueAsBytes(value));
-        } catch (Exception ex) {
-            throw new InternalProcessingException(ex);
-        }
+    protected void overwrite(String path, Object value) throws Exception {
+        ensureConnected();
+        zookeeper.setData().forPath(path, mapper.writeValueAsBytes(value));
     }
 
-    protected void touch(String path) {
-        try {
-            byte[] oldData = zookeeper.getData().forPath(path);
-            zookeeper.setData().forPath(path, oldData);
-        } catch (Exception ex) {
-            throw new InternalProcessingException(ex);
-        }
+    protected void overwrite(String path, byte[] value) throws Exception {
+        ensureConnected();
+        zookeeper.setData().forPath(path, value);
     }
 
-    protected void remove(String path) {
-        try {
-            zookeeper.delete().guaranteed().deletingChildrenIfNeeded().forPath(path);
-        } catch (Exception ex) {
-            throw new InternalProcessingException(ex);
-        }
+    protected void createRecursively(String path, Object value) throws Exception {
+        ensureConnected();
+        zookeeper.create()
+                .creatingParentsIfNeeded()
+                .forPath(path, mapper.writeValueAsBytes(value));
     }
 
-    protected boolean isEmpty(String path) {
-        try {
-            byte[] data = zookeeper.getData().forPath(path);
-            return data.length == 0;
-        } catch (Exception e) {
-            throw new InternalProcessingException(e);
-        }
+    protected void createInTransaction(String path, Object value, String childPath) throws Exception {
+        ensureConnected();
+        zookeeper.inTransaction()
+                .create().forPath(path, mapper.writeValueAsBytes(value))
+                .and()
+                .create().forPath(childPath)
+                .and()
+                .commit();
+    }
+
+    protected void create(String path, Object value) throws Exception {
+        ensureConnected();
+        zookeeper.create().forPath(path, mapper.writeValueAsBytes(value));
+    }
+
+    protected void create(String path, byte[] value) throws Exception {
+        ensureConnected();
+        zookeeper.create().forPath(path, value);
+    }
+
+    protected void touch(String path) throws Exception {
+        ensureConnected();
+        byte[] oldData = zookeeper.getData().forPath(path);
+        zookeeper.setData().forPath(path, oldData);
+    }
+
+    protected void remove(String path) throws Exception {
+        ensureConnected();
+        zookeeper.delete().guaranteed().deletingChildrenIfNeeded().forPath(path);
     }
 
     private interface ThrowingReader<T> {

diff --git a/.../java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperCredentialsRepository.java b/.../java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperCredentialsRepository.java
@@ -2,6 +2,7 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.curator.framework.CuratorFramework;
+import pl.allegro.tech.hermes.common.exception.InternalProcessingException;
 import pl.allegro.tech.hermes.domain.CredentialsRepository;
 import pl.allegro.tech.hermes.domain.NodePassword;
 
@@ -18,6 +19,10 @@ public NodePassword readAdminPassword() {
 
     @Override
     public void overwriteAdminPassword(String password) {
-        overwrite(paths.groupsPath(), new NodePassword(password));
+        try {
+            overwrite(paths.groupsPath(), new NodePassword(password));
+        } catch (Exception e) {
+            throw new InternalProcessingException(e);
+        }
     }
 }
diff --git a/.../allegro/tech/hermes/infrastructure/zookeeper/ZookeeperDatacenterReadinessRepository.java b/.../allegro/tech/hermes/infrastructure/zookeeper/ZookeeperDatacenterReadinessRepository.java
diff --git a/...c/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperGroupRepository.java b/...c/main/java/pl/allegro/tech/hermes/infrastructure/zookeeper/ZookeeperGroupRepository.java
@@ -28,7 +28,6 @@ public ZookeeperGroupRepository(CuratorFramework zookeeper,
 
     @Override
     public boolean groupExists(String groupName) {
-        ensureConnected();
         return pathExists(paths.groupPath(groupName));
     }
 
@@ -41,17 +40,11 @@ public void ensureGroupExists(String groupName) {
 
     @Override
     public void createGroup(Group group) {
-        ensureConnected();
-
         String groupPath = paths.groupPath(group.getGroupName());
         logger.info("Creating group {} for path {}", group.getGroupName(), groupPath);
 
         try {
-            zookeeper.inTransaction()
-                    .create().forPath(groupPath, mapper.writeValueAsBytes(group))
-                    .and()
-                    .create().forPath(paths.topicsPath(group.getGroupName()))
-                    .and().commit();
+            createInTransaction(groupPath, group, paths.topicsPath(group.getGroupName()));
         } catch (KeeperException.NodeExistsException ex) {
             throw new GroupAlreadyExistsException(group.getGroupName(), ex);
         } catch (Exception ex) {
@@ -61,21 +54,27 @@ public void createGroup(Group group) {
 
     @Override
     public void updateGroup(Group group) {
-        ensureConnected();
         ensureGroupExists(group.getGroupName());
 
         logger.info("Updating group {}", group.getGroupName());
-        overwrite(paths.groupPath(group.getGroupName()), group);
+        try {
+            overwrite(paths.groupPath(group.getGroupName()), group);
+        } catch (Exception e) {
+            throw new InternalProcessingException(e);
+        }
     }
 
     @Override
     public void removeGroup(String groupName) {
-        ensureConnected();
         ensureGroupExists(groupName);
         ensureGroupIsEmpty(groupName);
 
         logger.info("Removing group: {}", groupName);
-        remove(paths.groupPath(groupName));
+        try {
+            remove(paths.groupPath(groupName));
+        } catch (Exception e) {
+            throw new InternalProcessingException(e);
+        }
     }
 
     private void ensureGroupIsEmpty(String groupName) {
@@ -86,7 +85,6 @@ private void ensureGroupIsEmpty(String groupName) {
 
     @Override
     public List<String> listGroupNames() {
-        ensureConnected();
         return childrenOf(paths.groupsPath());
     }
 
@@ -105,7 +103,6 @@ public Group getGroupDetails(String groupName) {
     }
 
     private Optional<Group> getGroupDetails(String groupName, boolean quiet) {
-        ensureConnected();
         ensureGroupExists(groupName);
 
         String path = paths.groupPath(groupName);