From 90e9ff2db0bb3443dcf631c889c59b972b0190fa Mon Sep 17 00:00:00 2001 From: cchenax Date: Tue, 19 Jan 2021 19:39:52 +0800 Subject: [PATCH 1/5] Support skip no permission storagedir with multiple volumes --- .../org/apache/ratis/server/impl/ServerState.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java index 29d8320d40..2c53d316f6 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java @@ -67,7 +67,7 @@ class ServerState implements Closeable { /** The thread that applies committed log entries to the state machine */ private final StateMachineUpdater stateMachineUpdater; /** local storage for log and snapshot */ - private final RaftStorageImpl storage; + private RaftStorageImpl storage; private final SnapshotManager snapshotManager; private volatile Timestamp lastNoLeaderTime; private final TimeDuration noLeaderTimeout; @@ -103,10 +103,17 @@ class ServerState implements Closeable { configurationManager = new ConfigurationManager(initialConf); LOG.info("{}: {}", getMemberId(), configurationManager); + List directories = RaftServerConfigKeys.storageDir(prop); // use full uuid string to create a subdirectory - final File dir = chooseStorageDir(RaftServerConfigKeys.storageDir(prop), - group.getGroupId().getUuid().toString()); - storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop)); + File dir = chooseStorageDir(RaftServerConfigKeys.storageDir(prop), + group.getGroupId().getUuid().toString()); + try { + storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop)); + } catch (IOException e) { + directories.remove(dir); + dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString()); + storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop)); + } snapshotManager = new SnapshotManager(storage, id); stateMachine.initialize(server.getRaftServer(), group.getGroupId(), storage); From 3c5f8baad3637a928bbcfacaa6fe788771225fef Mon Sep 17 00:00:00 2001 From: cchenax Date: Wed, 20 Jan 2021 12:21:27 +0800 Subject: [PATCH 2/5] Support skip the storagedir with multiple volumes which cause by bad disk --- .../apache/ratis/server/impl/ServerState.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java index 2c53d316f6..99ba055fa3 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java @@ -104,15 +104,16 @@ class ServerState implements Closeable { LOG.info("{}: {}", getMemberId(), configurationManager); List directories = RaftServerConfigKeys.storageDir(prop); - // use full uuid string to create a subdirectory - File dir = chooseStorageDir(RaftServerConfigKeys.storageDir(prop), - group.getGroupId().getUuid().toString()); - try { - storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop)); - } catch (IOException e) { - directories.remove(dir); - dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString()); - storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop)); + boolean flag = false; + while (!flag) { + // use full uuid string to create a subdirectory + File dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString()); + try { + storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop)); + flag = true; + } catch (IOException e) { + directories.remove(dir); + } } snapshotManager = new SnapshotManager(storage, id); From 1fbc65f7eec746d503f02c3650426f7296aeae6f Mon Sep 17 00:00:00 2001 From: cchenax Date: Wed, 20 Jan 2021 12:33:50 +0800 Subject: [PATCH 3/5] Support skip the storagedir with multiple volumes which cause by bad disk --- .../org/apache/ratis/server/impl/ServerState.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java index 99ba055fa3..ef89a5b208 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java @@ -104,15 +104,23 @@ class ServerState implements Closeable { LOG.info("{}: {}", getMemberId(), configurationManager); List directories = RaftServerConfigKeys.storageDir(prop); - boolean flag = false; - while (!flag) { + // the flag is true means the storagedir create successful or iterate all storagedir whill exit this loop + boolean Flag = false; + int FileCount = 0; + int FileSize = directories.size(); + while (!Flag) { // use full uuid string to create a subdirectory File dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString()); try { storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop)); - flag = true; + Flag = true; } catch (IOException e) { directories.remove(dir); + } finally { + FileCount = FileCount + 1; + if (FileCount == FileSize) { + Flag = true; + } } } snapshotManager = new SnapshotManager(storage, id); From 8c44b6b302c9446b6fd0e39d46026a8663ab7106 Mon Sep 17 00:00:00 2001 From: cchenax Date: Wed, 20 Jan 2021 15:02:30 +0800 Subject: [PATCH 4/5] Support skip the storagedir with multiple volumes which cause by bad disk --- .../apache/ratis/server/impl/ServerState.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java index ef89a5b208..728f7e9c9b 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java @@ -40,6 +40,7 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; +import java.nio.file.AccessDeniedException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -104,23 +105,14 @@ class ServerState implements Closeable { LOG.info("{}: {}", getMemberId(), configurationManager); List directories = RaftServerConfigKeys.storageDir(prop); - // the flag is true means the storagedir create successful or iterate all storagedir whill exit this loop - boolean Flag = false; - int FileCount = 0; - int FileSize = directories.size(); - while (!Flag) { + while (!directories.isEmpty()) { // use full uuid string to create a subdirectory File dir = chooseStorageDir(directories, group.getGroupId().getUuid().toString()); try { storage = new RaftStorageImpl(dir, RaftServerConfigKeys.Log.corruptionPolicy(prop)); - Flag = true; - } catch (IOException e) { + break; + } catch (AccessDeniedException e) { directories.remove(dir); - } finally { - FileCount = FileCount + 1; - if (FileCount == FileSize) { - Flag = true; - } } } snapshotManager = new SnapshotManager(storage, id); From 0863db3692b84e5fd641f609f668175f0b5aed8f Mon Sep 17 00:00:00 2001 From: cchenax Date: Thu, 25 Mar 2021 16:59:03 +0800 Subject: [PATCH 5/5] one of raftgroup initraftlog fail due to raftlog IllegalStateException --- .../apache/ratis/server/impl/RaftServerProxy.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java index e6408f8bc5..6e7f5384ed 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java @@ -349,7 +349,17 @@ private RaftServerImpl getImpl(RaftGroupId groupId) throws IOException { List getImpls() throws IOException { final List list = new ArrayList<>(); for(CompletableFuture f : impls.getAll()) { - list.add(IOUtils.getFromFuture(f, this::getId)); + try { + RaftServerImpl result = f.exceptionally(e->{ + LOG.warn("raft groupid " + this.getId() +" initlog is error"); + return null; + }).get(); + if (result != null) { + list.add(IOUtils.getFromFuture(f, this::getId)); + } + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } } return list; }