curve_ops_tool add check-operator and status command optimize

Change-Id: Ib21935799a9153016f80effd49905bac01ca852e

src/tools/copyset_check.cpp

@@ -6,13 +6,19 @@
  */
 #include "src/tools/copyset_check.h"
 #include "src/tools/common.h"
+#include "src/tools/metric_name.h"
 
 DEFINE_bool(detail, false, "list the copyset detail or not");
 DEFINE_uint32(chunkserverId, 0, "chunkserver id");
 DEFINE_string(chunkserverAddr, "", "if specified, chunkserverId is not required");  // NOLINT
 DEFINE_uint32(serverId, 0, "server id");
 DEFINE_string(serverIp, "", "server ip");
+DEFINE_string(opName, curve::tool::kTotalOpName, "operator name");
 DECLARE_string(mdsAddr);
+DEFINE_uint64(opIntervalExceptLeader, 5, "Operator generation interval other "
+                                         "than transfer leader");
+DEFINE_uint64(leaderOpInterval, 30,
+                        "tranfer leader operator generation interval");
 
 namespace curve {
 namespace tool {
@@ -32,7 +38,8 @@ namespace tool {
 
 bool CopysetCheck::SupportCommand(const std::string& command) {
     return (command == kCheckCopysetCmd || command == kCheckChunnkServerCmd
-            || command == kCheckServerCmd || command == kCheckClusterCmd);
+            || command == kCheckServerCmd || command == kCopysetsStatusCmd
+            || command == kCheckOperatorCmd);
 }
 
 int CopysetCheck::Init() {
@@ -67,8 +74,15 @@ int CopysetCheck::RunCommand(const std::string& command) {
     } else if (command == kCheckServerCmd) {
         CHECK_ONLY_ONE_SHOULD_BE_SPECIFIED(serverIp, serverId);
         return CheckServer();
-    } else if (command == kCheckClusterCmd) {
-        return CheckCluster();
+    } else if (command == kCopysetsStatusCmd) {
+        return CheckCopysetsInCluster();
+    } else if (command == kCheckOperatorCmd) {
+        if (!SupportOpName(FLAGS_opName)) {
+            std::cout << "only support opName: ";
+            PrintSupportOpName();
+            return -1;
+        }
+        return CheckOperator(FLAGS_opName);
     } else {
         PrintHelp(command);
         return -1;
@@ -142,12 +156,12 @@ int CopysetCheck::CheckServer() {
     return res;
 }
 
-int CopysetCheck::CheckCluster() {
+int CopysetCheck::CheckCopysetsInCluster() {
     int res = core_->CheckCopysetsInCluster();
     if (res == 0) {
-        std::cout << "Cluster is healthy!" << std::endl;
+        std::cout << "Copysets are healthy!" << std::endl;
     } else {
-        std::cout << "Cluster is not healthy!" << std::endl;
+        std::cout << "Copysets not healthy!" << std::endl;
     }
     PrintStatistic();
     if (FLAGS_detail) {
@@ -156,6 +170,23 @@ int CopysetCheck::CheckCluster() {
     return res;
 }
 
+int CopysetCheck::CheckOperator(const std::string& opName) {
+    int res;
+    if (opName == kTransferOpName || opName == kTotalOpName) {
+        res = core_->CheckOperator(opName, FLAGS_leaderOpInterval);
+    } else {
+        res = core_->CheckOperator(opName, FLAGS_opIntervalExceptLeader);
+    }
+     if (res < 0) {
+        std::cout << "Check operator fail!" << std::endl;
+    } else {
+        std::cout << "Operator num is "
+                  << res << std::endl;
+        res = 0;
+    }
+    return res;
+}
+
 void CopysetCheck::PrintHelp(const std::string& command) {
     std::cout << "Example: " << std::endl << std::endl;
     if (command == kCheckCopysetCmd) {
@@ -169,8 +200,8 @@ void CopysetCheck::PrintHelp(const std::string& command) {
     } else if (command == kCheckServerCmd) {
         std::cout << "curve_ops_tool check-server -mdsAddr=127.0.0.1:6666 -serverId=1 [-margin=1000]" << std::endl;  // NOLINT
         std::cout << "curve_ops_tool check-server -mdsAddr=127.0.0.1:6666 -serverIp=127.0.0.1 [-margin=1000]" << std::endl;  // NOLINT
-    } else if (command == kCheckClusterCmd) {
-        std::cout << "curve_ops_tool check-cluster -mdsAddr=127.0.0.1:6666 [-margin=1000] [-operatorMaxPeriod=30] [-checkOperator]" << std::endl << std::endl;  // NOLINT
+    } else if (command == kCopysetsStatusCmd) {
+        std::cout << "curve_ops_tool copysets-status -mdsAddr=127.0.0.1:6666 [-margin=1000] [-operatorMaxPeriod=30] [-checkOperator]" << std::endl << std::endl;  // NOLINT
     } else {
         std::cout << "Command not supported!" << std::endl;
     }

src/tools/copyset_check.h

@@ -97,7 +97,13 @@ class CopysetCheck : public CurveTool {
      *  @brief 检查集群所有copyset
      *  @return 健康返回0，其他情况返回-1
      */
-    int CheckCluster();
+    int CheckCopysetsInCluster();
+
+    /**
+     *  @brief 检查mds端的operator
+     *  @return 无operator返回0，其他情况返回-1
+     */
+    int CheckOperator(const std::string& opName);
 
     // 打印copyset检查的详细结果
     void PrintDetail();

src/tools/copyset_check_core.cpp

@@ -4,6 +4,7 @@
  * Author: charisu
  * Copyright (c) 2018 netease
  */
+#include <math.h>
 #include "src/tools/copyset_check_core.h"
 
 DEFINE_uint64(margin, 1000, "The threshold of the gap between peers");
@@ -297,24 +298,37 @@ int CopysetCheckCore::CheckCopysetsInCluster() {
     }
     // 默认不检查operator，在测试脚本之类的要求比较严格的地方才检查operator，不然
     // 每次执行命令等待30秒很不方便
-    if (!FLAGS_checkOperator) {
-        return 0;
+    if (FLAGS_checkOperator) {
+        int res = CheckOperator(kTotalOpName, FLAGS_operatorMaxPeriod);
+        if (res != 0) {
+            std::cout << "Exists operators on mds, scheduling!" << std::endl;
+            return -1;
+        }
     }
+    return 0;
+}
+
+int CopysetCheckCore::CheckOperator(const std::string& opName,
+                                    uint64_t checkTimeSec) {
     uint64_t startTime = curve::common::TimeUtility::GetTimeofDaySec();
-    while (curve::common::TimeUtility::GetTimeofDaySec() -
-                        startTime < FLAGS_operatorMaxPeriod) {
+    std::string metricName = GetOpNumMetricName(opName);
+    do {
         uint64_t opNum = 0;
-        int res = mdsClient_->GetMetric(kOperatorNumMetricName, &opNum);
+        int res = mdsClient_->GetMetric(metricName, &opNum);
         if (res != 0) {
             std::cout << "Get oparator num from mds fail!" << std::endl;
             return -1;
         }
         if (opNum != 0) {
-            std::cout << "Exists operators on mds, scheduling!" << std::endl;
-            return -1;
+            return opNum;
+        }
+        if (curve::common::TimeUtility::GetTimeofDaySec() -
+                                        startTime >= checkTimeSec) {
+            break;
         }
         sleep(1);
-    }
+    } while (curve::common::TimeUtility::GetTimeofDaySec() -
+                                        startTime < checkTimeSec);
     return 0;
 }
 

src/tools/copyset_check_core.h

@@ -149,6 +149,15 @@ class CopysetCheckCore {
     */
     virtual int CheckCopysetsInCluster();
 
+    /**
+    * @brief 检查集群中的operator
+    * @param opName operator的名字
+    * @param checkTimeSec 检查时间
+    * @return 检查正常返回0，检查失败或存在operator返回-1
+    */
+    virtual int CheckOperator(const std::string& opName,
+                              uint64_t checkTimeSec);
+
     /**
      *  @brief 计算不健康的copyset的比例，检查后调用
      *  @return 不健康的copyset的比例

src/tools/curve_tool_define.h

@@ -34,7 +34,8 @@ const char kChunkLocatitonCmd[] = "chunk-location";
 const char kCheckCopysetCmd[] = "check-copyset";
 const char kCheckChunnkServerCmd[] = "check-chunkserver";
 const char kCheckServerCmd[] = "check-server";
-const char kCheckClusterCmd[] = "check-cluster";
+const char kCopysetsStatusCmd[] = "copysets-status";
+const char kCheckOperatorCmd[] = "check-operator";
 
 // 一致性检查命令
 const char kCheckConsistencyCmd[] = "check-consistency";

src/tools/curve_tool_factory.cpp

@@ -41,7 +41,7 @@ std::shared_ptr<StatusTool> CurveToolFactory::GenerateStatusTool() {
     auto versionTool = std::make_shared<VersionTool>(mdsClient, metricClient);
     return std::make_shared<StatusTool>(mdsClient, etcdClient,
                                        nameSpaceTool, copysetCheck,
-                                       versionTool);
+                                       versionTool, metricClient);
 }
 
 std::shared_ptr<NameSpaceTool> CurveToolFactory::GenerateNameSpaceTool() {

src/tools/curve_tool_main.cpp

@@ -16,6 +16,7 @@ const char* kHelpStr = "Usage: curve_ops_tool [Command] [OPTIONS...]\n"
         "mds-status : show the mds status\n"
         "client-status : show the client status\n"
         "etcd-status : show the etcd status\n"
+        "copysets-status : check the health state of all copysets\n"
         "chunkserver-list : show curve chunkserver-list, list all chunkserver infomation\n"  //NOLINT
         "get : show the file info and the actual space of file\n"
         "list : list the file info of files in the directory\n"
@@ -28,10 +29,10 @@ const char* kHelpStr = "Usage: curve_ops_tool [Command] [OPTIONS...]\n"
         "remove-peer : remove the peer from the copyset\n"
         "transfer-leader : transfer the leader of the copyset to the peer\n"  //NOLINT
         "reset-peer : reset the configuration of copyset, only reset to one peer is supported\n" //NOLINT
-        "check-copyset : check the health state of copyset\n"
         "check-chunkserver : check the health state of the chunkserver\n"
         "check-server : check the health state of the server\n"
         "check-cluster : check the health state of the cluster\n"
+        "check-operator : check the operators\n"
         "rapid-leader-schedule: rapid leader schedule in cluster in logicalpool\n\n"  //NOLINT
         "You can specify the config path by -confPath to avoid typing too many options\n";  //NOLINT
 

src/tools/mds_client.cpp

@@ -788,6 +788,9 @@ int MDSClient::GetMdsOnlineStatus(std::map<std::string, bool>* onlineStatus) {
 int MDSClient::GetMetric(const std::string& metricName, uint64_t* value) {
     std::string str;
     int res = GetMetric(metricName, &str);
+    if (res != 0) {
+        return -1;
+    }
     if (!curve::common::StringToUll(str, value)) {
         std::cout << "parse metric as uint64_t fail!" << std::endl;
         return -1;
@@ -809,7 +812,7 @@ int MDSClient::GetMetric(const std::string& metricName, std::string* value) {
             changeTimeLeft--;
         }
     }
-    std::cout << "GetMetric from all mds fail!"
+    std::cout << "GetMetric " << metricName << " from all mds fail!"
               << std::endl;
     return -1;
 }

src/tools/metric_client.cpp

@@ -59,6 +59,23 @@ int MetricClient::GetMetric(const std::string& addr,
     return -1;
 }
 
+int MetricClient::GetMetricUint(const std::string& addr,
+                  const std::string& metricName,
+                  uint64_t* value) {
+    std::string str;
+    int res = GetMetric(addr, metricName, &str);
+    if (res != 0) {
+        std::cout << "get metric " << metricName << " from "
+                  << addr << " fail";
+        return -1;
+    }
+    if (!curve::common::StringToUll(str, value)) {
+        std::cout << "parse metric as uint64_t fail!" << std::endl;
+        return -1;
+    }
+    return 0;
+}
+
 int MetricClient::GetValueFromAttachment(const std::string& attachment,
                                        std::string* value) {
     auto pos = attachment.find(":");

src/tools/metric_client.h

@@ -12,6 +12,7 @@
 #include <iostream>
 #include <string>
 #include "src/tools/common.h"
+#include "src/common/string_util.h"
 
 namespace curve {
 namespace tool {
@@ -31,6 +32,17 @@ class MetricClient {
                   const std::string& metricName,
                   std::string* value);
 
+     /**
+     *  @brief 从指定地址获取metric,并转换成uint
+     *  @param addr 要访问的地址
+     *  @param metricName 要获取的metric name
+     *  @param[out] value metric的值
+     *  @return 成功返回0，失败返回-1
+     */
+    virtual int GetMetricUint(const std::string& addr,
+                  const std::string& metricName,
+                  uint64_t* value);
+
  private:
     // 从response attachment解析出metric值
     int GetValueFromAttachment(const std::string& attachment,

src/tools/metric_name.h

@@ -8,6 +8,7 @@
 #include <bvar/bvar.h>
 
 #include <string>
+#include <map>
 
 #ifndef SRC_TOOLS_METRIC_NAME_H_
 #define SRC_TOOLS_METRIC_NAME_H_
@@ -16,12 +17,19 @@
 namespace curve {
 namespace tool {
 
-const char kOperatorNum[] = "mds_scheduler_metric_operator_num";
 const char kLogicalPoolMetricPrefix[] = "topology_metric_logicalPool_";
-const char kChunkServerMetricPrefix[] = "topology_metric_chunkserver_Id_";
+const char kChunkServerMetricPrefix[] = "chunkserver_";
 const char kOperatorNumMetricName[] = "mds_scheduler_metric_operator_num";
 const char kMdsListenAddrMetricName[] = "mds_config_mds_listen_addr";
 const char kCurveVersionMetricName[] = "curve_version";
+const char kSechduleOpMetricpPrefix[] = "mds_scheduler_metric_";
+
+// operator名称
+const char kTotalOpName[] = "operator";
+const char kChangeOpName[] = "change_peer";
+const char kAddOpName[] = "add_peer";
+const char kRemoveOpName[] = "remove_peer";
+const char kTransferOpName[] = "transfer_leader";
 
 inline std::string GetPoolTotalBytesName(
                             const std::string& poolName) {
@@ -50,13 +58,34 @@ inline std::string GetPoolDiskAllocName(
     return metricName;
 }
 
-inline std::string GetCSLeftBytesName(uint64_t id) {
+inline std::string GetCSLeftChunkName(const std::string& csAddr) {
     std::string tmpName = kChunkServerMetricPrefix +
-                        std::to_string(id) + "_chunkSizeLeftBytes";
+                        csAddr + "_chunkfilepool_left";
+    std::string metricName;
+    bvar::to_underscored_name(&metricName, tmpName);
+    return metricName;
+}
+
+inline std::string GetOpNumMetricName(const std::string& opName) {
+    std::string tmpName = kSechduleOpMetricpPrefix +
+                                opName + "_num";
     std::string metricName;
     bvar::to_underscored_name(&metricName, tmpName);
     return metricName;
 }
+
+inline bool SupportOpName(const std::string& opName) {
+    return opName == kTotalOpName || opName == kChangeOpName
+                || opName == kAddOpName || opName == kRemoveOpName
+                || opName == kTransferOpName;
+}
+
+inline void PrintSupportOpName() {
+    std::cout << kTotalOpName << ", " << kChangeOpName
+              << ", " << kAddOpName << ", " << kRemoveOpName
+              << ", " << kTransferOpName << std::endl;
+}
+
 }  // namespace tool
 }  // namespace curve
 

src/tools/status_tool.cpp

@@ -12,6 +12,7 @@ DEFINE_bool(unhealthy, false, "if true, only list chunkserver that unhealthy "
                               "ratio greater than 0");
 DEFINE_bool(checkHealth, true, "if true, it will check the health "
                                 "state of chunkserver in chunkserver-list");
+DEFINE_uint64(chunkSize, 16777216, "chunk size");
 DECLARE_string(mdsAddr);
 DECLARE_string(etcdAddr);
 DECLARE_string(mdsDummyPort);
@@ -211,12 +212,14 @@ int StatusTool::ChunkServerStatusCmd() {
 }
 
 int StatusTool::PrintClusterStatus() {
+    int ret = 0;
     std::cout << "Cluster status:" << std::endl;
-    int res = copysetCheckCore_->CheckCopysetsInCluster();
-    if (res == 0) {
-        std::cout << "cluster is healthy!" << std::endl;
+    bool healthy = IsClusterHeatlhy();
+    if (healthy) {
+        std::cout << "cluster is healthy" << std::endl;
     } else {
-        std::cout << "cluster is not healthy!" << std::endl;
+        std::cout << "cluster is not healthy" << std::endl;
+        ret = -1;
     }
     const auto& statistics = copysetCheckCore_->GetCopysetStatistics();
     std::cout << "total copysets: " << statistics.totalNum
@@ -225,14 +228,57 @@ int StatusTool::PrintClusterStatus() {
               << statistics.unhealthyRatio * 100 << "%" << std::endl;
     std::vector<PhysicalPoolInfo> phyPools;
     std::vector<LogicalPoolInfo> lgPools;
-    res = GetPoolsInCluster(&phyPools, &lgPools);
+    int res = GetPoolsInCluster(&phyPools, &lgPools);
     if (res != 0) {
         std::cout << "GetPoolsInCluster fail!" << std::endl;
-        return -1;
+        ret = -1;
     }
     std::cout << "physical pool number: " << phyPools.size()
               << ", logical pool number: " << lgPools.size() << std::endl;
-    return SpaceCmd();
+    res = SpaceCmd();
+    if (res != 0) {
+        ret = -1;
+    }
+    return ret;
+}
+
+bool StatusTool::IsClusterHeatlhy() {
+    // 1、检查copyset健康状态
+    int res = copysetCheckCore_->CheckCopysetsInCluster();
+    if (res != 0) {
+        return false;
+    }
+    // 2、检查mds状态
+    std::string mds = mdsClient_->GetCurrentMds();
+    if (mds.empty()) {
+        return false;
+    }
+    std::map<std::string, bool> onlineStatus;
+    res = mdsClient_->GetMdsOnlineStatus(&onlineStatus);
+    if (res != 0) {
+        return false;
+    }
+    for (const auto& item : onlineStatus) {
+        if (!item.second) {
+            return false;
+        }
+    }
+    // 3、检查etcd在线状态
+    std::string leaderAddr;
+    onlineStatus.clear();
+    res = etcdClient_->GetEtcdClusterStatus(&leaderAddr, &onlineStatus);
+    if (res != 0) {
+        return false;
+    }
+    if (leaderAddr.empty()) {
+        return false;
+    }
+    for (const auto& item : onlineStatus) {
+        if (!item.second) {
+            return false;
+        }
+    }
+    return true;
 }
 
 void StatusTool::PrintOnlineStatus(const std::string& name,
@@ -268,13 +314,18 @@ void StatusTool::PrintOnlineStatus(const std::string& name,
 int StatusTool::PrintMdsStatus() {
     std::cout << "MDS status:" << std::endl;
     std::string version;
-    int res = versionTool_->GetAndCheckMdsVersion(&version);
+    std::vector<std::string> failedList;
+    int res = versionTool_->GetAndCheckMdsVersion(&version, &failedList);
     int ret = 0;
     if (res != 0) {
         std::cout << "GetAndCheckMdsVersion fail" << std::endl;
         ret = -1;
     } else {
         std::cout << "version: " << version << std::endl;
+        if (!failedList.empty()) {
+            versionTool_->PrintFailedList(failedList);
+            ret = -1;
+        }
     }
 
     std::cout << "current MDS: " << mdsClient_->GetCurrentMds() << std::endl;
@@ -331,13 +382,19 @@ int StatusTool::PrintClientStatus() {
 int StatusTool::PrintChunkserverStatus(bool checkLeftSize) {
     std::cout << "ChunkServer status:" << std::endl;
     std::string version;
-    int res = versionTool_->GetAndCheckChunkServerVersion(&version);
+    std::vector<std::string> failedList;
+    int res = versionTool_->GetAndCheckChunkServerVersion(&version,
+                                                          &failedList);
     int ret = 0;
     if (res != 0) {
         std::cout << "GetAndCheckChunkserverVersion fail" << std::endl;
         ret = -1;
     } else {
         std::cout << "version: " << version << std::endl;
+        if (!failedList.empty()) {
+            versionTool_->PrintFailedList(failedList);
+            ret = -1;
+        }
     }
     std::vector<ChunkServerInfo> chunkservers;
     res = mdsClient_->ListChunkServersInCluster(&chunkservers);
@@ -362,16 +419,18 @@ int StatusTool::PrintChunkserverStatus(bool checkLeftSize) {
         if (!checkLeftSize) {
             continue;
         }
-        auto csId = chunkserver.chunkserverid();
-        std::string metricName = GetCSLeftBytesName(csId);
-        uint64_t size;
-        int res = mdsClient_->GetMetric(metricName, &size);
+        std::string csAddr = chunkserver.hostip()
+                        + ":" + std::to_string(chunkserver.port());
+        std::string metricName = GetCSLeftChunkName(csAddr);
+        uint64_t chunkNum;
+        int res = metricClient_->GetMetricUint(csAddr, metricName, &chunkNum);
         if (res != 0) {
-            std::cout << "Get left chunk size of chunkserver " << csId
+            std::cout << "Get left chunk size of chunkserver " << csAddr
                       << " fail!" << std::endl;
             ret = -1;
             continue;
         }
+        uint64_t size = chunkNum * FLAGS_chunkSize;
         if (leftSizeNum.count(size) == 0) {
             leftSizeNum[size] = 1;
         } else {

src/tools/status_tool.h

@@ -27,6 +27,8 @@
 #include "src/tools/version_tool.h"
 #include "src/tools/curve_tool.h"
 #include "src/tools/curve_tool_define.h"
+#include "src/tools/metric_client.h"
+#include "src/tools/metric_name.h"
 
 using curve::mds::topology::ChunkServerStatus;
 using curve::mds::topology::DiskState;
@@ -53,11 +55,13 @@ class StatusTool : public CurveTool {
                std::shared_ptr<EtcdClient> etcdClient,
                std::shared_ptr<NameSpaceToolCore> nameSpaceToolCore,
                std::shared_ptr<CopysetCheckCore> copysetCheckCore,
-               std::shared_ptr<VersionTool> versionTool) :
+               std::shared_ptr<VersionTool> versionTool,
+               std::shared_ptr<MetricClient> metricClient) :
                   mdsClient_(mdsClient), etcdClient_(etcdClient),
                   nameSpaceToolCore_(nameSpaceToolCore),
                   copysetCheckCore_(copysetCheckCore),
                   versionTool_(versionTool),
+                  metricClient_(metricClient),
                   mdsInited_(false), etcdInited_(false) {}
     ~StatusTool() = default;
 
@@ -82,6 +86,11 @@ class StatusTool : public CurveTool {
      */
     static bool SupportCommand(const std::string& command);
 
+    /**
+     *  @brief 判断集群是否健康
+     */
+    bool IsClusterHeatlhy();
+
  private:
     int Init(const std::string& command);
     int SpaceCmd();
@@ -133,6 +142,8 @@ class StatusTool : public CurveTool {
     std::shared_ptr<CopysetCheckCore> copysetCheckCore_;
     // etcd client，用于调etcd API获取状态
     std::shared_ptr<EtcdClient> etcdClient_;
+    // 用于获取metric
+    std::shared_ptr<MetricClient> metricClient_;
     // version client，用于获取version信息
     std::shared_ptr<VersionTool> versionTool_;
     // mds是否初始化过

src/tools/version_tool.cpp

@@ -14,20 +14,16 @@ int VersionTool::Init(const std::string& mdsAddr) {
     return mdsClient_->Init(mdsAddr);
 }
 
-int VersionTool::GetAndCheckMdsVersion(std::string* version) {
+int VersionTool::GetAndCheckMdsVersion(std::string* version,
+                                       std::vector<std::string>* failedList) {
     const auto& dummyServerMap = mdsClient_->GetDummyServerMap();
     std::vector<std::string> dummyServers;
     for (const auto& item : dummyServerMap) {
         dummyServers.emplace_back(item.second);
     }
     VersionMapType versionMap;
-    std::vector<std::string> failedList;
-    GetVersionMap(dummyServers, &versionMap, &failedList);
+    GetVersionMap(dummyServers, &versionMap, failedList);
     int ret = 0;
-    if (!failedList.empty()) {
-        PrintFailedList(failedList);
-        ret = -1;
-    }
     if (versionMap.empty()) {
         std::cout << "no version found!" << std::endl;
         ret = -1;
@@ -41,7 +37,8 @@ int VersionTool::GetAndCheckMdsVersion(std::string* version) {
     return ret;
 }
 
-int VersionTool::GetAndCheckChunkServerVersion(std::string* version) {
+int VersionTool::GetAndCheckChunkServerVersion(std::string* version,
+                                        std::vector<std::string>* failedList) {
     std::vector<ChunkServerInfo> chunkServers;
     int res = mdsClient_->ListChunkServersInCluster(&chunkServers);
     if (res != 0) {
@@ -55,13 +52,8 @@ int VersionTool::GetAndCheckChunkServerVersion(std::string* version) {
     }
 
     VersionMapType versionMap;
-    std::vector<std::string> failedList;
-    GetVersionMap(csAddrs, &versionMap, &failedList);
+    GetVersionMap(csAddrs, &versionMap, failedList);
     int ret = 0;
-    if (!failedList.empty()) {
-        PrintFailedList(failedList);
-        ret = -1;
-    }
     if (versionMap.empty()) {
         std::cout << "no version found!" << std::endl;
         ret = -1;
@@ -90,6 +82,7 @@ int VersionTool::GetClientVersion(VersionMapType* versionMap,
 void VersionTool::GetVersionMap(const std::vector<std::string>& addrVec,
                                 VersionMapType* versionMap,
                                 std::vector<std::string>* failedList) {
+    failedList->clear();
     for (const auto& addr : addrVec) {
         std::string version;
         int res = metricClient_->GetMetric(addr, kCurveVersionMetricName,
@@ -127,7 +120,7 @@ void VersionTool::PrintFailedList(const std::vector<std::string>& failedList) {
         }
         std::cout << failedList[i];
     }
-    std::cout << "} fail";
+    std::cout << "} fail" << std::endl;
 }
 
 }  // namespace tool

src/tools/version_tool.h

@@ -39,14 +39,16 @@ class VersionTool {
      *  @param[out] version 版本
      *  @return 成功返回0，失败返回-1
      */
-    virtual int GetAndCheckMdsVersion(std::string* version);
+    virtual int GetAndCheckMdsVersion(std::string* version,
+                                      std::vector<std::string>* failedList);
 
     /**
      *  @brief 获取chunkserver的版本并检查版本一致性
      *  @param[out] version 版本
      *  @return 成功返回0，失败返回-1
      */
-    virtual int GetAndCheckChunkServerVersion(std::string* version);
+    virtual int GetAndCheckChunkServerVersion(std::string* version,
+                                       std::vector<std::string>* failedList);
 
     /**
      *  @brief 获取client的版本
@@ -63,6 +65,12 @@ class VersionTool {
      */
     void PrintVersionMap(const VersionMapType& versionMap);
 
+    /**
+     *  @brief 打印访问失败的地址
+     *  @param failedList 访问失败的地址列表
+     */
+    void PrintFailedList(const std::vector<std::string>& failedList);
+
  private:
     /**
      *  @brief 获取addrVec对应地址的version，并把version和地址对应关系存在map中
@@ -74,12 +82,6 @@ class VersionTool {
                        VersionMapType* versionMap,
                        std::vector<std::string>* failedList);
 
-     /**
-     *  @brief 打印访问失败的地址
-     *  @param failedList 访问失败的地址列表
-     */
-    void PrintFailedList(const std::vector<std::string>& failedList);
-
  private:
     // 向mds发送RPC的client
     std::shared_ptr<MDSClient> mdsClient_;

test/tools/copyset_check_core_test.cpp

@@ -18,22 +18,18 @@ using curve::mds::topology::ChunkServerStatus;
 using curve::mds::topology::DiskState;
 using curve::mds::topology::OnlineState;
 using curve::mds::topology::CopySetServerInfo;
-using curve::tool::kTotal;
-using curve::tool::kInstallingSnapshot;
-using curve::tool::kNoLeader;
-using curve::tool::kLogIndexGapTooBig;
-using curve::tool::kPeersNoSufficient;
-using curve::tool::kMinorityPeerNotOnline;
-using curve::tool::kMajorityPeerNotOnline;
 
 DECLARE_uint64(operatorMaxPeriod);
 DECLARE_bool(checkOperator);
 
+namespace curve {
+namespace tool {
+
 class CopysetCheckCoreTest : public ::testing::Test {
  protected:
     void SetUp() {
-        mdsClient_ = std::make_shared<curve::tool::MockMDSClient>();
-        csClient_ = std::make_shared<curve::tool::MockChunkServerClient>();
+        mdsClient_ = std::make_shared<MockMDSClient>();
+        csClient_ = std::make_shared<MockChunkServerClient>();
         FLAGS_operatorMaxPeriod = 3;
         FLAGS_checkOperator = true;
     }
@@ -150,8 +146,8 @@ class CopysetCheckCoreTest : public ::testing::Test {
         os.move_to(*buf);
     }
 
-    std::shared_ptr<curve::tool::MockMDSClient> mdsClient_;
-    std::shared_ptr<curve::tool::MockChunkServerClient> csClient_;
+    std::shared_ptr<MockMDSClient> mdsClient_;
+    std::shared_ptr<MockChunkServerClient> csClient_;
 };
 
 TEST_F(CopysetCheckCoreTest, Init) {
@@ -159,7 +155,7 @@ TEST_F(CopysetCheckCoreTest, Init) {
         .Times(2)
         .WillOnce(Return(0))
         .WillOnce(Return(-1));
-    curve::tool::CopysetCheckCore copysetCheck(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck(mdsClient_, csClient_);
     ASSERT_EQ(0, copysetCheck.Init("127.0.0.1:6666"));
     ASSERT_EQ(-1, copysetCheck.Init("127.0.0.1:6666"));
 }
@@ -193,7 +189,7 @@ TEST_F(CopysetCheckCoreTest, CheckOneCopysetNormal) {
     EXPECT_CALL(*csClient_, CheckChunkServerOnline())
         .Times(2)
         .WillRepeatedly(Return(true));
-    curve::tool::CopysetCheckCore copysetCheck(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck(mdsClient_, csClient_);
     ASSERT_EQ(0, copysetCheck.CheckOneCopyset(1, 100));
     butil::IOBuf iobuf;
     iobuf.append("\r\n");
@@ -226,7 +222,7 @@ TEST_F(CopysetCheckCoreTest, CheckOneCopysetError) {
     EXPECT_CALL(*mdsClient_, GetChunkServerListInCopySet(_, _, _))
         .Times(1)
         .WillOnce(Return(-1));
-    curve::tool::CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck1.CheckOneCopyset(1, 100));
 
     // 2、copyset不健康
@@ -242,7 +238,7 @@ TEST_F(CopysetCheckCoreTest, CheckOneCopysetError) {
         .Times(3)
         .WillRepeatedly(DoAll(SetArgPointee<0>(followerBuf),
                         Return(0)));
-    curve::tool::CopysetCheckCore copysetCheck2(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck2(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck2.CheckOneCopyset(1, 100));
 
     // 3、第一个peer不在线
@@ -263,7 +259,7 @@ TEST_F(CopysetCheckCoreTest, CheckOneCopysetError) {
                         Return(0)))
         .WillOnce(DoAll(SetArgPointee<0>(followerBuf),
                         Return(0)));
-    curve::tool::CopysetCheckCore copysetCheck3(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck3(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck3.CheckOneCopyset(1, 100));
 }
 
@@ -287,7 +283,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsOnChunkServerHealthy) {
         .Times(1)
         .WillOnce(DoAll(SetArgPointee<1>(csInfo),
                         Return(0)));
-    curve::tool::CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
     ASSERT_EQ(0, copysetCheck1.CheckCopysetsOnChunkServer(csId));
     ASSERT_DOUBLE_EQ(0, copysetCheck1.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedRes, copysetCheck1.GetCopysetsRes());
@@ -311,7 +307,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsOnChunkServerHealthy) {
     EXPECT_CALL(*csClient_, CheckChunkServerOnline())
         .Times(1)
         .WillRepeatedly(Return(true));
-    curve::tool::CopysetCheckCore copysetCheck2(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck2(mdsClient_, csClient_);
     ASSERT_EQ(0, copysetCheck2.CheckCopysetsOnChunkServer(csId));
     ASSERT_DOUBLE_EQ(0, copysetCheck2.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedRes, copysetCheck2.GetCopysetsRes());
@@ -333,7 +329,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsOnChunkServerHealthy) {
     EXPECT_CALL(*csClient_, CheckChunkServerOnline())
         .Times(1)
         .WillRepeatedly(Return(true));
-    curve::tool::CopysetCheckCore copysetCheck3(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck3(mdsClient_, csClient_);
     ASSERT_EQ(0, copysetCheck3.CheckCopysetsOnChunkServer(csAddr));
     ASSERT_DOUBLE_EQ(0, copysetCheck3.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedRes, copysetCheck3.GetCopysetsRes());
@@ -358,7 +354,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsOnChunkServerError) {
     std::map<std::string, std::set<std::string>> expectedRes;
 
     // 1、GetChunkServerInfo失败的情况
-    curve::tool::CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
     EXPECT_CALL(*mdsClient_, GetChunkServerInfo(csId, _))
         .Times(1)
         .WillOnce(Return(-1));
@@ -400,7 +396,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsOnChunkServerError) {
         .Times(1)
         .WillOnce(DoAll(SetArgPointee<2>(csServerInfos),
                         Return(0)));
-    curve::tool::CopysetCheckCore copysetCheck2(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck2(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck2.CheckCopysetsOnChunkServer(csId));
     ASSERT_DOUBLE_EQ(1, copysetCheck2.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedExcepCs, copysetCheck2.GetServiceExceptionChunkServer());
@@ -419,7 +415,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsOnChunkServerError) {
     EXPECT_CALL(*mdsClient_, GetCopySetsInChunkServer(csAddr, _))
         .Times(1)
         .WillOnce(Return(-1));
-    curve::tool::CopysetCheckCore copysetCheck3(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck3(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck3.CheckCopysetsOnChunkServer(csId));
     ASSERT_DOUBLE_EQ(0, copysetCheck3.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedExcepCs, copysetCheck3.GetServiceExceptionChunkServer());
@@ -444,7 +440,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsOnChunkServerError) {
     EXPECT_CALL(*mdsClient_, GetChunkServerListInCopySets(_, _, _))
         .Times(1)
         .WillOnce(Return(-1));
-    curve::tool::CopysetCheckCore copysetCheck4(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck4(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck4.CheckCopysetsOnChunkServer(csId));
     ASSERT_DOUBLE_EQ(0, copysetCheck4.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedExcepCs, copysetCheck4.GetServiceExceptionChunkServer());
@@ -578,7 +574,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsOnChunkServerUnhealthy) {
 
     // 检查结果
     std::set<std::string> expectedExcepCs = {csAddr1, csAddr2};
-    curve::tool::CopysetCheckCore copysetCheck(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck.CheckCopysetsOnChunkServer(csId));
     ASSERT_DOUBLE_EQ(0.5, copysetCheck.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedRes, copysetCheck.GetCopysetsRes());
@@ -627,7 +623,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsOnServerNormal) {
     EXPECT_CALL(*csClient_, CheckChunkServerOnline())
         .Times(2)
         .WillRepeatedly(Return(true));
-    curve::tool::CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
     ASSERT_EQ(0, copysetCheck1.CheckCopysetsOnServer(serverId, &unhealthyCs));
     ASSERT_EQ(0, unhealthyCs.size());
     ASSERT_EQ(0, copysetCheck1.GetCopysetStatistics().unhealthyRatio);
@@ -651,7 +647,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsOnServerNormal) {
         .Times(2)
         .WillRepeatedly(Return(true));
     // 通过ip查询
-    curve::tool::CopysetCheckCore copysetCheck2(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck2(mdsClient_, csClient_);
     ASSERT_EQ(0, copysetCheck2.CheckCopysetsOnServer(serverIp, &unhealthyCs));
     ASSERT_EQ(0, unhealthyCs.size());
     ASSERT_EQ(0, copysetCheck2.GetCopysetStatistics().unhealthyRatio);
@@ -685,7 +681,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsOnServerError) {
     EXPECT_CALL(*mdsClient_, ListChunkServersOnServer(serverId, _))
         .Times(1)
         .WillOnce(Return(-1));
-    curve::tool::CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck1.CheckCopysetsOnServer(serverId));
     ASSERT_EQ(0, copysetCheck1.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedRes, copysetCheck1.GetCopysetsRes());
@@ -715,7 +711,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsOnServerError) {
         .WillOnce(DoAll(SetArgPointee<1>(copysets),
                         Return(0)));
     std::vector<std::string> unhealthyCs;
-    curve::tool::CopysetCheckCore copysetCheck2(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck2(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck2.CheckCopysetsOnServer(serverId, &unhealthyCs));
     ASSERT_EQ(1, copysetCheck2.GetCopysetStatistics().unhealthyRatio);
     std::vector<std::string> unhealthyCsExpected =
@@ -761,7 +757,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsInClusterNormal) {
         .Times(3)
         .WillRepeatedly(DoAll(SetArgPointee<1>(0),
                         Return(0)));
-    curve::tool::CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
     ASSERT_EQ(0, copysetCheck1.CheckCopysetsInCluster());
     ASSERT_EQ(0, copysetCheck1.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedRes, copysetCheck1.GetCopysetsRes());
@@ -782,7 +778,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsInClusterError) {
     EXPECT_CALL(*mdsClient_, ListServersInCluster(_))
         .Times(1)
         .WillOnce(Return(-1));
-    curve::tool::CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck1(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck1.CheckCopysetsInCluster());
     ASSERT_EQ(0, copysetCheck1.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedRes, copysetCheck1.GetCopysetsRes());
@@ -795,7 +791,7 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsInClusterError) {
     EXPECT_CALL(*mdsClient_, ListChunkServersOnServer(1, _))
         .Times(1)
         .WillOnce(Return(-1));
-    curve::tool::CopysetCheckCore copysetCheck2(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck2(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck2.CheckCopysetsInCluster());
     ASSERT_EQ(0, copysetCheck2.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedRes, copysetCheck2.GetCopysetsRes());
@@ -826,13 +822,38 @@ TEST_F(CopysetCheckCoreTest, CheckCopysetsInClusterError) {
         .WillRepeatedly(DoAll(SetArgPointee<1>(10),
                         Return(0)));
     // 获取operator失败
-    curve::tool::CopysetCheckCore copysetCheck3(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck3(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck3.CheckCopysetsInCluster());
     ASSERT_EQ(0, copysetCheck3.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedRes, copysetCheck3.GetCopysetsRes());
     // operator数量大于0
-    curve::tool::CopysetCheckCore copysetCheck4(mdsClient_, csClient_);
+    CopysetCheckCore copysetCheck4(mdsClient_, csClient_);
     ASSERT_EQ(-1, copysetCheck4.CheckCopysetsInCluster());
     ASSERT_EQ(0, copysetCheck4.GetCopysetStatistics().unhealthyRatio);
     ASSERT_EQ(expectedRes, copysetCheck4.GetCopysetsRes());
 }
+
+TEST_F(CopysetCheckCoreTest, CheckOperator) {
+    CopysetCheckCore copysetCheck(mdsClient_, csClient_);
+    std::string opName = "change_peer";
+    uint64_t checkTime = 3;
+    // 1、获取metric失败
+    EXPECT_CALL(*mdsClient_, GetMetric(_, _))
+        .Times(1)
+        .WillOnce(Return(-1));
+    ASSERT_EQ(-1, copysetCheck.CheckOperator(opName, checkTime));
+    // 2、operator数量不为0
+    EXPECT_CALL(*mdsClient_, GetMetric(_, _))
+        .Times(1)
+        .WillOnce(DoAll(SetArgPointee<1>(10),
+                        Return(0)));
+    ASSERT_EQ(10, copysetCheck.CheckOperator(opName, checkTime));
+    // 3、operator数量为0
+    EXPECT_CALL(*mdsClient_, GetMetric(_, _))
+        .WillRepeatedly(DoAll(SetArgPointee<1>(0),
+                        Return(0)));
+    ASSERT_EQ(0, copysetCheck.CheckOperator(opName, checkTime));
+}
+
+}  // namespace tool
+}  // namespace curve

test/tools/copyset_check_test.cpp

@@ -23,15 +23,21 @@ DECLARE_uint32(chunkserverId);
 DECLARE_string(chunkserverAddr);
 DECLARE_uint32(serverId);
 DECLARE_string(serverIp);
+DECLARE_uint64(leaderOpInterval);
+DECLARE_uint64(opIntervalExceptLeader);
+DECLARE_string(opName);
+
+namespace curve {
+namespace tool {
 
 class CopysetCheckTest : public ::testing::Test {
  protected:
     CopysetCheckTest() {
-        statistics1 = curve::tool::CopysetStatistics(2, 0);
-        statistics2 = curve::tool::CopysetStatistics(6, 5);
+        statistics1 = CopysetStatistics(2, 0);
+        statistics2 = CopysetStatistics(6, 5);
     }
     void SetUp() {
-        core_ = std::make_shared<curve::tool::MockCopysetCheckCore>();
+        core_ = std::make_shared<MockCopysetCheckCore>();
         FLAGS_detail = true;
     }
     void TearDown() {
@@ -101,22 +107,22 @@ class CopysetCheckTest : public ::testing::Test {
     std::set<std::string> serviceExcepCs = {"127.0.0.1:9092"};
     std::set<std::string> emptySet;
 
-    curve::tool::CopysetStatistics statistics1;
-    curve::tool::CopysetStatistics statistics2;
-    std::shared_ptr<curve::tool::MockCopysetCheckCore> core_;
+    CopysetStatistics statistics1;
+    CopysetStatistics statistics2;
+    std::shared_ptr<MockCopysetCheckCore> core_;
 };
 
 TEST_F(CopysetCheckTest, SupportCommand) {
-    curve::tool::CopysetCheck copysetCheck(core_);
+    CopysetCheck copysetCheck(core_);
     ASSERT_TRUE(copysetCheck.SupportCommand("check-copyset"));
     ASSERT_TRUE(copysetCheck.SupportCommand("check-chunkserver"));
     ASSERT_TRUE(copysetCheck.SupportCommand("check-server"));
-    ASSERT_TRUE(copysetCheck.SupportCommand("check-cluster"));
+    ASSERT_TRUE(copysetCheck.SupportCommand("copysets-status"));
     ASSERT_FALSE(copysetCheck.SupportCommand("check-nothing"));
 }
 
 TEST_F(CopysetCheckTest, CheckOneCopyset) {
-    curve::tool::CopysetCheck copysetCheck(core_);
+    CopysetCheck copysetCheck(core_);
     butil::IOBuf iobuf;
     GetIoBufForTest(&iobuf, "4294967396", true);
     std::vector<std::string> peersInCopyset =
@@ -167,7 +173,7 @@ TEST_F(CopysetCheckTest, CheckOneCopyset) {
 }
 
 TEST_F(CopysetCheckTest, testCheckChunkServer) {
-    curve::tool::CopysetCheck copysetCheck(core_);
+    CopysetCheck copysetCheck(core_);
     EXPECT_CALL(*core_, Init(_))
         .Times(1)
         .WillOnce(Return(0));
@@ -227,7 +233,7 @@ TEST_F(CopysetCheckTest, testCheckChunkServer) {
 }
 
 TEST_F(CopysetCheckTest, testCheckServer) {
-    curve::tool::CopysetCheck copysetCheck(core_);
+    CopysetCheck copysetCheck(core_);
     std::vector<std::string> chunkservers =
             {"127.0.0.1:9091", "127.0.0.1:9092", "127.0.0.1:9093"};
     EXPECT_CALL(*core_, Init(_))
@@ -292,7 +298,7 @@ TEST_F(CopysetCheckTest, testCheckServer) {
 }
 
 TEST_F(CopysetCheckTest, testCheckCluster) {
-    curve::tool::CopysetCheck copysetCheck(core_);
+    CopysetCheck copysetCheck(core_);
     EXPECT_CALL(*core_, Init(_))
         .Times(1)
         .WillOnce(Return(0));
@@ -310,7 +316,7 @@ TEST_F(CopysetCheckTest, testCheckCluster) {
     EXPECT_CALL(*core_, GetServiceExceptionChunkServer())
         .Times(1)
         .WillOnce(ReturnRef(emptySet));
-    ASSERT_EQ(0, copysetCheck.RunCommand("check-cluster"));
+    ASSERT_EQ(0, copysetCheck.RunCommand(kCopysetsStatusCmd));
 
     // 不健康的情况
     EXPECT_CALL(*core_, CheckCopysetsInCluster())
@@ -325,5 +331,39 @@ TEST_F(CopysetCheckTest, testCheckCluster) {
     EXPECT_CALL(*core_, GetServiceExceptionChunkServer())
         .Times(1)
         .WillOnce(ReturnRef(serviceExcepCs));
-    ASSERT_EQ(-1, copysetCheck.RunCommand("check-cluster"));
+    ASSERT_EQ(-1, copysetCheck.RunCommand(kCopysetsStatusCmd));
 }
+
+TEST_F(CopysetCheckTest, testCheckOperator) {
+    CopysetCheck copysetCheck(core_);
+    EXPECT_CALL(*core_, Init(_))
+        .Times(1)
+        .WillOnce(Return(0));
+
+    // 1、不支持的operator
+    FLAGS_opName = "no_operator";
+    ASSERT_EQ(-1, copysetCheck.RunCommand(kCheckOperatorCmd));
+    // 2、transfer leader的operator和total的
+    EXPECT_CALL(*core_, CheckOperator(_, FLAGS_leaderOpInterval))
+        .Times(2)
+        .WillOnce(Return(0))
+        .WillOnce(Return(-1));
+    FLAGS_opName = kTransferOpName;
+    ASSERT_EQ(0, copysetCheck.RunCommand(kCheckOperatorCmd));
+    FLAGS_opName = kTotalOpName;
+    ASSERT_EQ(-1, copysetCheck.RunCommand(kCheckOperatorCmd));
+    // 2、其他operator
+    EXPECT_CALL(*core_, CheckOperator(_, FLAGS_opIntervalExceptLeader))
+        .Times(3)
+        .WillOnce(Return(10))
+        .WillRepeatedly(Return(0));
+    FLAGS_opName = kChangeOpName;
+    ASSERT_EQ(0, copysetCheck.RunCommand(kCheckOperatorCmd));
+    FLAGS_opName = kAddOpName;
+    ASSERT_EQ(0, copysetCheck.RunCommand(kCheckOperatorCmd));
+    FLAGS_opName = kRemoveOpName;
+    ASSERT_EQ(0, copysetCheck.RunCommand(kCheckOperatorCmd));
+}
+
+}  // namespace tool
+}  // namespace curve

test/tools/curve_tool_factory_test.cpp

@@ -24,6 +24,8 @@ TEST(CurveToolFactoryTest, GetStatusTool) {
     ASSERT_TRUE(dynamic_cast<StatusTool *>(curveTool.get()) != nullptr);
     curveTool = CurveToolFactory::GenerateCurveTool("etcd-status");
     ASSERT_TRUE(dynamic_cast<StatusTool *>(curveTool.get()) != nullptr);
+    curveTool = CurveToolFactory::GenerateCurveTool("client-status");
+    ASSERT_TRUE(dynamic_cast<StatusTool *>(curveTool.get()) != nullptr);
     curveTool = CurveToolFactory::GenerateCurveTool("nothing");
     ASSERT_TRUE(curveTool.get() == nullptr);
 }
@@ -66,7 +68,9 @@ TEST(CurveToolFactoryTest, GetCopysetCheck) {
     ASSERT_TRUE(dynamic_cast<CopysetCheck *>(curveTool.get()) != nullptr);
     curveTool = CurveToolFactory::GenerateCurveTool("check-server");
     ASSERT_TRUE(dynamic_cast<CopysetCheck *>(curveTool.get()) != nullptr);
-    curveTool = CurveToolFactory::GenerateCurveTool("check-cluster");
+    curveTool = CurveToolFactory::GenerateCurveTool("copysets-status");
+    ASSERT_TRUE(dynamic_cast<CopysetCheck *>(curveTool.get()) != nullptr);
+    curveTool = CurveToolFactory::GenerateCurveTool("check-operator");
     ASSERT_TRUE(dynamic_cast<CopysetCheck *>(curveTool.get()) != nullptr);
 }
 }  // namespace tool

test/tools/mock_copyset_check_core.h

@@ -42,6 +42,7 @@ class MockCopysetCheckCore : public CopysetCheckCore {
     MOCK_CONST_METHOD0(GetCopysetDetail, const std::string&());
     MOCK_CONST_METHOD0(GetServiceExceptionChunkServer,
                        const std::set<std::string>&());
+    MOCK_METHOD2(CheckOperator, int(const std::string&, uint64_t));
 };
 }  // namespace tool
 }  // namespace curve

test/tools/mock_metric_client.h

@@ -23,6 +23,8 @@ class MockMetricClient : public MetricClient {
  public:
     MOCK_METHOD3(GetMetric, int(const std::string&, const std::string&,
                                 std::string*));
+    MOCK_METHOD3(GetMetricUint, int(const std::string&, const std::string&,
+                                uint64_t*));
 };
 }  // namespace tool
 }  // namespace curve