add system health check

ggits-etlp-server/src/main/java/com/sig/ggits/etlp/server/cluster/ClusterMasterService.java

@@ -30,7 +30,8 @@ public class ClusterMasterService extends AbstractClusterMasterService {
     @Override
     public void election(int clusterId, boolean isMaster, boolean isMasterChanged) {
         if (isMasterChanged) {
-            log.info("ClusterMasterService:election: master state changed: clusterId: {}, master: {}", clusterId, isMaster);
+            log.info("ClusterNodeId: {}, ClusterMasterService:election: master state changed: clusterId: {}, master: {}",
+                    this.clusterId, clusterId, isMaster);
         }
     }
 
@@ -38,8 +39,8 @@ public class ClusterMasterService extends AbstractClusterMasterService {
     public void onClusterMessage(ClusterMessage message) {
         // 슬래이브로 부터 수신되는 메시지 처리
         if (message.getInfos() == null) {
-            log.warn("onClusterMessage: clusterId: {}, master: {}, time: {}: message infos not found, receive clusterNodeId: {}",
-                    message.getClusterId(), message.isMaster(), ClusterUtils.timeToString(message.getCurrentTimeMillis()), message.getClusterId());
+            log.warn("ClusterNodeId: {}, ClusterMasterService.onClusterMessage: clusterId: {}, master: {}, time: {}: message infos not found, receive clusterNodeId: {}",
+                    this.clusterId, message.getClusterId(), message.isMaster(), ClusterUtils.timeToString(message.getCurrentTimeMillis()), message.getClusterId());
             return;
         }
         List<ClusterEtlpDto> infos = new ArrayList<>();
@@ -48,11 +49,19 @@ public class ClusterMasterService extends AbstractClusterMasterService {
                 infos.add((ClusterEtlpDto)info);
             }
         }
-        log.info("onClusterMessage: {}", infos);
+        if (infos.isEmpty()) {
+            log.warn("ClusterNodeId: {}, ClusterMasterService.onClusterMessage: fromClusterNodeId: {}, Data Empty.",
+                    this.clusterId, message.getClusterId());
+            return;
+        }
+
+        log.info("ClusterNodeId: {}, ClusterMasterService.onClusterMessage: fromClusterNodeId: {}, {} Centers.",
+                this.clusterId, message.getClusterId(), infos.get(0).getCenters().size());
+
         for (ClusterEtlpDto clusterTodpDto : infos){
             if (this.clusterId == clusterTodpDto.getClusterId()) {
-                log.warn("onClusterMessage: clusterId: {}, master: {}, time: {}: clusterId error, this: {}, receive: {}",
-                        message.getClusterId(), message.isMaster(), ClusterUtils.timeToString(message.getCurrentTimeMillis()), this.clusterId, clusterTodpDto.getClusterId());
+                log.warn("ClusterNodeId: {}, ClusterMasterService.onClusterMessage: clusterId: {}, master: {}, time: {}: clusterId error, this: {}, receive: {}",
+                        this.clusterId, message.getClusterId(), message.isMaster(), ClusterUtils.timeToString(message.getCurrentTimeMillis()), this.clusterId, clusterTodpDto.getClusterId());
             }
 
             for (RegionCenter rcvCenter: clusterTodpDto.getCenters()) {

ggits-etlp-server/src/main/java/com/sig/ggits/etlp/server/cluster/ClusterSlaveService.java

@@ -47,7 +47,8 @@ public class ClusterSlaveService extends AbstractClusterSlaveService {
                 }
             });
         }
-        log.info("getClusterMessageData: {}", result);
+        log.info("ClusterNodeId: {}, ClusterSlaveService.getClusterMessageData: {} Centers.",
+                this.clusterId, clusterInfo.getCenters().size());
         return result;
     }
 

ggits-etlp-server/src/main/java/com/sig/ggits/etlp/server/controller/GgitsEtlpServerRestController.java

@@ -60,23 +60,25 @@ public class GgitsEtlpServerRestController {
             sb.append(heading).append(sep);
             sb.append(String.format("Cluster Running Information: clusterNodeId: %d, %d EA", this.clusterConfig.getId(), this.clusterConfig.getClusterMap().size())).append(sep);
             sb.append(heading).append(sep);
-            sb.append("SEQ nodeId  IP Address       Port     isMaster  netStat  lastSyncRecvTime     lastSyncSendTime").append(sep);
+            sb.append("SEQ nodeId  IP Address       Port     isMaster  netStat  connectTime          lastSyncRecvTime     lastSyncSendTime").append(sep);
             sb.append(heading).append(sep);
 
             for (Integer clusterId : clusterIds) {
                 ClusterNetState netState;
                 ClusterNode clusterNode = this.clusterConfig.getClusterMap().get(clusterId);
                 netState = clusterNode.getSyncState();
+                final String connectTimeString = netState.getConnectTimeString();
                 if (clusterNode.getId() == this.clusterConfig.getId()) {
                     netState = this.clusterConfig.getMyCluster().getElectionState();
                 }
-                sb.append(String.format("%3d %6.6s  %-15.15s  %-7.7s  %-8.8s  %-7.7s  %s  %s",
+                sb.append(String.format("%3d %6.6s  %-15.15s  %-7.7s  %-8.8s  %-7.7s  %s  %s  %s",
                         ii++,
                         this.clusterConfig.getId() == clusterNode.getId() ? "*"+clusterNode.getId() : ""+clusterNode.getId(),
                         clusterNode.getIp(),
                         ""+clusterNode.getPort(),
                         clusterNode.isMaster(),
                         netState.getState() != ClusterNET.CLOSED ? "Up" : "Down",
+                        connectTimeString,
                         netState.getLastRecvTimeString(),
                         netState.getLastSendTimeString()
                 )).append(sep);

ggits-etlp-server/src/main/java/com/sig/ggits/etlp/server/service/GgitsEtlpService.java

@@ -111,7 +111,7 @@ public class GgitsEtlpService {
                 center.setEtlp(true);
                 center.setRealClusterId(this.clusterConfig.getId());
                 this.runCenterList.add(regionCd);
-                // 클러스터가 동일하면 추가하고 리턴
+                // 클러스터가 동일하면 추가
                 return;
             }
             if (!this.clusterConfig.isMaster()) {
@@ -121,7 +121,7 @@ public class GgitsEtlpService {
             int clusterId = center.getClusterId();
             ClusterNode clusterNode = this.clusterConfig.getClusterMap().get(clusterId);
             if (clusterNode != null && clusterNode.getElectionState().getState() == ClusterNET.CLOSED) {
-                // 통신이 이상이면 추가하고 리턴
+                // 통신이 이상이면 추가
                 center.setEtlp(true);
                 center.setRealClusterId(this.clusterConfig.getId());
                 this.runCenterList.add(regionCd);
@@ -151,10 +151,12 @@ public class GgitsEtlpService {
         this.running = true;
 
         try {
+            // 초기화 작업을 먼저 수해
+            initEtlp();
+
             initRunCenterList();
             makeValidCenters();
 
-            initEtlp();
 
             //교차로구성_연계, SCS_T_CON_INTLC
             //etlpIntLc();

ggits-etlp-server/src/main/resources/logback-spring-appender.xml

@@ -8,6 +8,20 @@
         </encoder>
     </appender>
 
+    <appender name="FILE_CLUSTER" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${LOG_PATH}${LOG_FILE_NAME_CLUSTER}</file>
+        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+            <charset>${FILE_LOG_CHARSET}</charset>
+            <pattern>${LOG_PATTERN_CLUSTER}</pattern>
+            <immediateFlush>true</immediateFlush>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${LOG_BACKUP_PATH}${LOG_FILE_NAME_CLUSTER}.${LOG_FILE_NAME_BACKUP}</fileNamePattern>
+            <maxFileSize>${MAX_FILESIZE}</maxFileSize>
+            <maxHistory>${MAX_HISTORY}</maxHistory>
+        </rollingPolicy>
+    </appender>
+
     <appender name="FILE_PACKET" class="ch.qos.logback.classic.sift.SiftingAppender">
         <discriminator>
             <key>id</key>
@@ -22,7 +36,7 @@
                 </encoder>
 
                 <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
-                    <FileNamePattern>${LOG_BACKUP_PATH}etlp/${id}.${LOG_FILE_NAME_PATTERN}</FileNamePattern>
+                    <FileNamePattern>${LOG_BACKUP_PATH}etlp/${id}.${LOG_FILE_NAME_BACKUP}</FileNamePattern>
                     <minIndex>1</minIndex>
                     <maxIndex>${MAX_HISTORY}</maxIndex>
                 </rollingPolicy>
@@ -46,7 +60,7 @@
         </encoder>
 
         <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
-            <fileNamePattern>${LOG_BACKUP_PATH}${LOG_FILE_NAME_ERROR}.${LOG_FILE_NAME_PATTERN}</fileNamePattern>
+            <fileNamePattern>${LOG_BACKUP_PATH}${LOG_FILE_NAME_ERROR}.${LOG_FILE_NAME_BACKUP}</fileNamePattern>
             <minIndex>1</minIndex>
             <maxIndex>${MAX_HISTORY}</maxIndex>
         </rollingPolicy>
@@ -63,7 +77,7 @@
         </encoder>
 
         <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
-            <fileNamePattern>${LOG_BACKUP_PATH}${LOG_FILE_NAME}.${LOG_FILE_NAME_PATTERN}</fileNamePattern>
+            <fileNamePattern>${LOG_BACKUP_PATH}${LOG_FILE_NAME}.${LOG_FILE_NAME_BACKUP}</fileNamePattern>
             <minIndex>1</minIndex>
             <maxIndex>${MAX_HISTORY}</maxIndex>
         </rollingPolicy>
@@ -80,7 +94,7 @@
         </encoder>
 
         <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
-            <fileNamePattern>${LOG_BACKUP_PATH}${LOG_FILE_NAME_SCHEDULE}.${LOG_FILE_NAME_PATTERN}</fileNamePattern>
+            <fileNamePattern>${LOG_BACKUP_PATH}${LOG_FILE_NAME_SCHEDULE}.${LOG_FILE_NAME_BACKUP}</fileNamePattern>
             <minIndex>1</minIndex>
             <maxIndex>${MAX_HISTORY}</maxIndex>
         </rollingPolicy>
@@ -89,83 +103,4 @@
         </triggeringPolicy>
     </appender>
 
-<!--    <appender name="FILE_PACKET" class="ch.qos.logback.classic.sift.SiftingAppender">-->
-<!--        <discriminator>-->
-<!--            <key>id</key>-->
-<!--            <defaultValue>${LOG_FILE_NAME_PACKET}</defaultValue>-->
-<!--        </discriminator>-->
-<!--        <sift>-->
-<!--            <appender name="FILE-${id}" class="ch.qos.logback.core.rolling.RollingFileAppender">-->
-<!--                <file>${LOG_PATH}etlp/${id}.log</file>-->
-<!--                <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">-->
-<!--                    <charset>${FILE_LOG_CHARSET}</charset>-->
-<!--                    <Pattern>${LOG_PATTERN_PACKET}</Pattern>-->
-<!--                </encoder>-->
-
-<!--                <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">-->
-<!--                    <FileNamePattern>${LOG_BACKUP_PATH}etlp/${id}.${LOG_FILE_NAME_PATTERN}</FileNamePattern>-->
-<!--                    <maxFileSize>${MAX_FILESIZE}</maxFileSize>-->
-<!--                    <maxHistory>${MAX_HISTORY}</maxHistory>-->
-<!--                </rollingPolicy>-->
-<!--            </appender>-->
-<!--        </sift>-->
-<!--    </appender>-->
-
-    <!--    <appender name="FILE_ERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">-->
-<!--        <filter class="ch.qos.logback.classic.filter.LevelFilter">-->
-<!--            <level>error</level>-->
-<!--            <onMatch>ACCEPT</onMatch>-->
-<!--            <onMismatch>DENY</onMismatch>-->
-<!--        </filter>-->
-<!--        <file>${LOG_PATH}${LOG_FILE_NAME_ERROR}</file>-->
-<!--        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">-->
-<!--            <charset>${FILE_LOG_CHARSET}</charset>-->
-<!--            <pattern>${LOG_PATTERN_ERROR}</pattern>-->
-<!--        </encoder>-->
-<!--        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">-->
-<!--            <fileNamePattern>${LOG_BACKUP_PATH}${LOG_FILE_NAME_ERROR}.${LOG_FILE_NAME_PATTERN}</fileNamePattern>-->
-<!--            <maxFileSize>${MAX_FILESIZE}</maxFileSize>-->
-<!--            <maxHistory>${MAX_HISTORY}</maxHistory>-->
-<!--        </rollingPolicy>-->
-<!--    </appender>-->
-
-<!--    <appender name="FILE_LOG" class="ch.qos.logback.core.rolling.RollingFileAppender">-->
-<!--        <file>${LOG_PATH}${LOG_FILE_NAME}</file>-->
-<!--        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">-->
-<!--            <charset>${FILE_LOG_CHARSET}</charset>-->
-<!--            <pattern>${LOG_PATTERN_FILE}</pattern>-->
-<!--        </encoder>-->
-<!--        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">-->
-<!--            <fileNamePattern>${LOG_BACKUP_PATH}${LOG_FILE_NAME}.${LOG_FILE_NAME_PATTERN}</fileNamePattern>-->
-<!--            <maxFileSize>${MAX_FILESIZE}</maxFileSize>-->
-<!--            <maxHistory>${MAX_HISTORY}</maxHistory>-->
-<!--        </rollingPolicy>-->
-<!--    </appender>-->
-
-<!--    <appender name="FILE_SCHEDULE" class="ch.qos.logback.core.rolling.RollingFileAppender">-->
-<!--        <file>${LOG_PATH}${LOG_FILE_NAME_SCHEDULE}</file>-->
-<!--        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">-->
-<!--            <charset>${FILE_LOG_CHARSET}</charset>-->
-<!--            <pattern>${LOG_PATTERN_SCHEDULE}</pattern>-->
-<!--        </encoder>-->
-<!--        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">-->
-<!--            <fileNamePattern>${LOG_BACKUP_PATH}${LOG_FILE_NAME_SCHEDULE}.${LOG_FILE_NAME_PATTERN}</fileNamePattern>-->
-<!--            <maxFileSize>${MAX_FILESIZE}</maxFileSize>-->
-<!--            <maxHistory>${MAX_HISTORY}</maxHistory>-->
-<!--        </rollingPolicy>-->
-<!--    </appender>-->
-
-    <appender name="FILE_SQL" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>${LOG_PATH}${LOG_FILE_NAME_SQL}</file>
-        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
-            <charset>${FILE_LOG_CHARSET}</charset>
-            <pattern>${LOG_PATTERN_SQL}</pattern>
-        </encoder>
-        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-            <fileNamePattern>${LOG_BACKUP_PATH}${LOG_FILE_NAME_SQL}.${LOG_FILE_NAME_PATTERN}</fileNamePattern>
-            <maxFileSize>${MAX_FILESIZE}</maxFileSize>
-            <maxHistory>${MAX_HISTORY}</maxHistory>
-        </rollingPolicy>
-    </appender>
-
 </included>

ggits-etlp-server/src/main/resources/logback-spring.xml

@@ -15,18 +15,18 @@
 
     <property name="LOG_FILE_NAME"             value="${PROJECT_NAME}.log"/>
     <property name="LOG_FILE_NAME_ERROR"       value="${PROJECT_NAME}.err.log"/>
-    <property name="LOG_FILE_NAME_PATTERN"     value="%d{yyyyMMdd}_%i.log.gz"/>
+    <property name="LOG_FILE_NAME_BACKUP"      value="%d{yyyyMMdd}_%i.log.gz"/>
     <property name="LOG_FILE_NAME_PACKET"      value="${PROJECT_PREFIX}-service"/>
-    <property name="LOG_FILE_NAME_SQL"         value="${PROJECT_PREFIX}-sql.log"/>
     <property name="LOG_FILE_NAME_SCHEDULE"    value="${PROJECT_PREFIX}-schedule.log"/>
+    <property name="LOG_FILE_NAME_CLUSTER"     value="${PROJECT_PREFIX}-cluster.log"/>
 
-    <property name="MAX_FILESIZE" value="10MB"/>
+    <property name="MAX_FILESIZE" value="80MB"/>
     <property name="MAX_HISTORY"  value="10"/>
     <property name="LOG_PATTERN_FILE"        value="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%-5level] %msg%n"/>
     <!--    <property name="LOG_PATTERN_ERROR"       value="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%30t] [%5level] %42logger{35}.%-20M ${PID:-} %n%msg%n"/>-->
     <property name="LOG_PATTERN_ERROR"       value="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%-5level] %msg%n"/>
     <property name="LOG_PATTERN_PACKET"      value="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%-5level] %msg%n"/>
-    <property name="LOG_PATTERN_SQL"         value="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%-5level] %msg%n"/>
+    <property name="LOG_PATTERN_CLUSTER"     value="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%-5level] %msg%n"/>
     <property name="LOG_PATTERN_SCHEDULE"    value="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%-5level] %msg%n"/>
     <property name="LOG_PATTERN_CONSOLE"     value="[%d{HH:mm:ss.SSS}] [%5level] %msg %n"/>
 
@@ -46,6 +46,16 @@
             <appender-ref ref="FILE_LOG"/>
             <appender-ref ref="FILE_ERROR"/>
         </root>
+        <logger name="com.its.common.cluster" level="INFO" additivity="false">
+            <appender-ref ref="CONSOLE"/>
+            <appender-ref ref="FILE_CLUSTER"/>
+            <appender-ref ref="FILE_ERROR"/>
+        </logger>
+        <logger name="${APP_CLASS_PATH}.cluster" level="INFO" additivity="false">
+            <appender-ref ref="CONSOLE"/>
+            <appender-ref ref="FILE_CLUSTER"/>
+            <appender-ref ref="FILE_ERROR"/>
+        </logger>
 
         <logger name="${APP_CLASS_PATH}.service" level="INFO" additivity="false">
             <appender-ref ref="CONSOLE"/>
@@ -66,6 +76,14 @@
             <appender-ref ref="FILE_LOG"/>
             <appender-ref ref="FILE_ERROR"/>
         </root>
+        <logger name="com.its.common.cluster" level="INFO" additivity="false">
+            <appender-ref ref="FILE_CLUSTER"/>
+            <appender-ref ref="FILE_ERROR"/>
+        </logger>
+        <logger name="${APP_CLASS_PATH}.cluster" level="INFO" additivity="false">
+            <appender-ref ref="FILE_CLUSTER"/>
+            <appender-ref ref="FILE_ERROR"/>
+        </logger>
 
         <logger name="${APP_CLASS_PATH}.service" level="INFO" additivity="false">
             <appender-ref ref="FILE_PACKET"/>