|
@@ -3,6 +3,7 @@ package com.utic.its.rota.utic.client.scheduler;
|
|
|
import com.utic.its.common.dto.CenterDto;
|
|
import com.utic.its.common.dto.CenterDto;
|
|
|
import com.utic.its.common.dto.NET;
|
|
import com.utic.its.common.dto.NET;
|
|
|
import com.utic.its.common.repository.ApplicationRepository;
|
|
import com.utic.its.common.repository.ApplicationRepository;
|
|
|
|
|
+import com.utic.its.common.utils.SystemHealth;
|
|
|
import com.utic.its.common.xnet.process.dbms.DbmsData;
|
|
import com.utic.its.common.xnet.process.dbms.DbmsData;
|
|
|
import com.utic.its.rota.utic.client.cluster.ClusterConfig;
|
|
import com.utic.its.rota.utic.client.cluster.ClusterConfig;
|
|
|
import com.utic.its.rota.utic.client.config.ApplicationConfig;
|
|
import com.utic.its.rota.utic.client.config.ApplicationConfig;
|
|
@@ -10,12 +11,21 @@ import com.utic.its.rota.utic.client.config.SchedulingConfig;
|
|
|
import com.utic.its.rota.utic.client.config.TraceConfig;
|
|
import com.utic.its.rota.utic.client.config.TraceConfig;
|
|
|
import com.utic.its.rota.utic.client.process.DbmsDataProcessService;
|
|
import com.utic.its.rota.utic.client.process.DbmsDataProcessService;
|
|
|
import com.utic.its.rota.utic.client.service.ProcessStateService;
|
|
import com.utic.its.rota.utic.client.service.ProcessStateService;
|
|
|
|
|
+import com.zaxxer.hikari.HikariDataSource;
|
|
|
|
|
+import com.zaxxer.hikari.HikariPoolMXBean;
|
|
|
import lombok.RequiredArgsConstructor;
|
|
import lombok.RequiredArgsConstructor;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
import org.springframework.scheduling.annotation.EnableScheduling;
|
|
import org.springframework.scheduling.annotation.EnableScheduling;
|
|
|
import org.springframework.scheduling.annotation.Scheduled;
|
|
import org.springframework.scheduling.annotation.Scheduled;
|
|
|
import org.springframework.stereotype.Component;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
|
|
|
|
|
+import java.lang.management.ManagementFactory;
|
|
|
|
|
+import java.lang.management.ThreadInfo;
|
|
|
|
|
+import java.lang.management.ThreadMXBean;
|
|
|
|
|
+import java.text.DecimalFormat;
|
|
|
|
|
+import java.util.EnumMap;
|
|
|
|
|
+import java.util.Map;
|
|
|
|
|
+
|
|
|
@Slf4j
|
|
@Slf4j
|
|
|
@RequiredArgsConstructor
|
|
@RequiredArgsConstructor
|
|
|
@EnableScheduling
|
|
@EnableScheduling
|
|
@@ -32,8 +42,29 @@ public class ApplicationScheduler {
|
|
|
private final DbmsData delRcvLinkTraffic = new DbmsData(0, null, DbmsData.DBMS_DELETE_RCV_LINK_TRAFFIC, null, null, 0);
|
|
private final DbmsData delRcvLinkTraffic = new DbmsData(0, null, DbmsData.DBMS_DELETE_RCV_LINK_TRAFFIC, null, null, 0);
|
|
|
private final DbmsData delRcvIncident = new DbmsData(0, null, DbmsData.DBMS_DELETE_RCV_INCIDENT, null, null, 0);
|
|
private final DbmsData delRcvIncident = new DbmsData(0, null, DbmsData.DBMS_DELETE_RCV_INCIDENT, null, null, 0);
|
|
|
|
|
|
|
|
|
|
+ private final HikariDataSource dataSource;
|
|
|
|
|
+ private final Map<Thread.State, Integer> stateCountMap = new EnumMap<>(Thread.State.class);
|
|
|
|
|
+ private final SystemHealth systemHealth = new SystemHealth();
|
|
|
|
|
+ private final DecimalFormat df = new DecimalFormat("#.##");
|
|
|
|
|
+
|
|
|
@Scheduled(cron = "5 * * * * *") // 1분주기 작업 실행
|
|
@Scheduled(cron = "5 * * * * *") // 1분주기 작업 실행
|
|
|
public void updateProcessState() {
|
|
public void updateProcessState() {
|
|
|
|
|
+ try {
|
|
|
|
|
+ this.systemHealth.checkHealth(false);
|
|
|
|
|
+ loggingHealthCheck();
|
|
|
|
|
+
|
|
|
|
|
+ double cpuUsage = this.systemHealth.getCpuUsage();
|
|
|
|
|
+ if (cpuUsage > this.config.getCpuLimits()) {
|
|
|
|
|
+ ApplicationRepository.processState.setErrDesc("CPU 사용율이 너무 높음: " + String.format("%.2f", cpuUsage));
|
|
|
|
|
+ log.warn("[SKIP] ApplicationScheduler.updateProcessState: High CPU Usage, Limit({} %), Current({} %), Schedule Job SKIP...",
|
|
|
|
|
+ this.config.getCpuLimits(), String.format("%.2f", cpuUsage));
|
|
|
|
|
+ loggingThreads();
|
|
|
|
|
+ return;
|
|
|
|
|
+ }
|
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
|
+ log.error("ApplicationScheduler.updateProcessState: System Health Check Exception {}", e.getMessage());
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
try {
|
|
try {
|
|
|
if (this.processStateService.isStateUpdateTimeExpired()) {
|
|
if (this.processStateService.isStateUpdateTimeExpired()) {
|
|
|
// 60초 이상 경과 시 프로세스 상태 업데이트
|
|
// 60초 이상 경과 시 프로세스 상태 업데이트
|
|
@@ -113,4 +144,68 @@ public class ApplicationScheduler {
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+ private void loggingHealthCheck() {
|
|
|
|
|
+ log.info("----------------------------------------------------------------------------------------------------------");
|
|
|
|
|
+ loggingSystemHealth();
|
|
|
|
|
+ logSessionStatus();
|
|
|
|
|
+ loggingThreadState();
|
|
|
|
|
+ log.info("----------------------------------------------------------------------------------------------------------");
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private void loggingSystemHealth() {
|
|
|
|
|
+ double loadAvg = this.systemHealth.getLoadAverage();
|
|
|
|
|
+ String loadAvgStr = (loadAvg < 0) ? "N/A" : this.df.format(loadAvg);
|
|
|
|
|
+ log.info("SYSTEM HEALTH: CPU {} Cores[{} %, LoadAvg({})], Memory[{} %, Used({} MB), Max({} MB)], Threads[{}, Peak({})], GC[{}, {} ms]",
|
|
|
|
|
+ this.systemHealth.getCpuCores(),
|
|
|
|
|
+ this.df.format(this.systemHealth.getCpuUsage()),
|
|
|
|
|
+ loadAvgStr,
|
|
|
|
|
+ this.df.format(this.systemHealth.getMemUsage()),
|
|
|
|
|
+ this.systemHealth.getUsedMemory(),
|
|
|
|
|
+ this.systemHealth.getMaxMemory(),
|
|
|
|
|
+ this.systemHealth.getThreadCount(),
|
|
|
|
|
+ this.systemHealth.getPeakThreadCount(),
|
|
|
|
|
+ this.systemHealth.getGcCount(),
|
|
|
|
|
+ this.systemHealth.getGcTime());
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private void logSessionStatus() {
|
|
|
|
|
+ HikariPoolMXBean poolStats = this.dataSource.getHikariPoolMXBean();
|
|
|
|
|
+ int totalConnections = poolStats.getTotalConnections();
|
|
|
|
|
+ int activeConnections = poolStats.getActiveConnections();
|
|
|
|
|
+ int idleConnections = poolStats.getIdleConnections();
|
|
|
|
|
+ int threadsAwaiting = poolStats.getThreadsAwaitingConnection();
|
|
|
|
|
+ log.info(" DB SESSION: Total: {}, Active: {}, Idle: {}, Waiting: {}",
|
|
|
|
|
+ totalConnections, activeConnections, idleConnections, threadsAwaiting);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private void loggingThreadState() {
|
|
|
|
|
+ ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
|
|
|
|
|
+ ThreadInfo[] infos = threadBean.dumpAllThreads(false, false);
|
|
|
|
|
+ this.stateCountMap.clear();
|
|
|
|
|
+ for (ThreadInfo info : infos) {
|
|
|
|
|
+ Thread.State state = info.getThreadState();
|
|
|
|
|
+ this.stateCountMap.put(state, this.stateCountMap.getOrDefault(state, 0) + 1);
|
|
|
|
|
+ }
|
|
|
|
|
+ StringBuilder sb = new StringBuilder();
|
|
|
|
|
+ for (Map.Entry<Thread.State, Integer> entry : stateCountMap.entrySet()) {
|
|
|
|
|
+ sb.append(String.format("%s(%d), ", entry.getKey().name(), entry.getValue()));
|
|
|
|
|
+ }
|
|
|
|
|
+ // 마지막 쉼표 제거
|
|
|
|
|
+ if (sb.length() > 0) {
|
|
|
|
|
+ sb.setLength(sb.length() - 2);
|
|
|
|
|
+ }
|
|
|
|
|
+ log.info(" THREAD STATE: {}", sb.toString());
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private void loggingThreads() {
|
|
|
|
|
+ ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
|
|
|
|
|
+ ThreadInfo[] infos = threadBean.dumpAllThreads(false, false);
|
|
|
|
|
+ this.stateCountMap.clear();
|
|
|
|
|
+ for (ThreadInfo info : infos) {
|
|
|
|
|
+ Thread.State state = info.getThreadState();
|
|
|
|
|
+ this.stateCountMap.put(state, this.stateCountMap.getOrDefault(state, 0) + 1);
|
|
|
|
|
+ log.info(" [" + info.getThreadId() + "] " + info.getThreadName() + " - " + info.getThreadState());
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
}
|
|
}
|