|
|
@@ -1,32 +1,23 @@
|
|
|
package com.utic.center.utic.traf.server.scheduler;
|
|
|
|
|
|
-import com.utic.center.common.utils.SystemHealth;
|
|
|
import com.utic.center.common.utils.TimeUtils;
|
|
|
import com.utic.center.utic.traf.server.config.ApplicationConfig;
|
|
|
import com.utic.center.utic.traf.server.config.TraceConfig;
|
|
|
import com.utic.center.utic.traf.server.controller.UticTrafServerController;
|
|
|
-import com.utic.center.utic.traf.server.repository.ApplicationRepository;
|
|
|
+import com.utic.center.utic.traf.server.health.SystemHealthService;
|
|
|
import com.utic.center.utic.traf.server.service.ProcessStateService;
|
|
|
-import com.zaxxer.hikari.HikariDataSource;
|
|
|
-import com.zaxxer.hikari.HikariPoolMXBean;
|
|
|
+import lombok.RequiredArgsConstructor;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
-import org.springframework.beans.factory.annotation.Qualifier;
|
|
|
import org.springframework.scheduling.annotation.EnableScheduling;
|
|
|
import org.springframework.scheduling.annotation.Scheduled;
|
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
|
-import javax.sql.DataSource;
|
|
|
-import java.lang.management.ManagementFactory;
|
|
|
-import java.lang.management.ThreadInfo;
|
|
|
-import java.lang.management.ThreadMXBean;
|
|
|
-import java.text.DecimalFormat;
|
|
|
import java.time.LocalDateTime;
|
|
|
-import java.util.EnumMap;
|
|
|
-import java.util.Map;
|
|
|
import java.util.concurrent.atomic.AtomicBoolean;
|
|
|
|
|
|
@Slf4j
|
|
|
@EnableScheduling
|
|
|
+@RequiredArgsConstructor
|
|
|
@Component
|
|
|
public class ApplicationScheduler {
|
|
|
|
|
|
@@ -34,26 +25,11 @@ public class ApplicationScheduler {
|
|
|
private final TraceConfig traceConfig;
|
|
|
private final ProcessStateService processStateService;
|
|
|
private final UticTrafServerController controller;
|
|
|
+ private final SystemHealthService healthService;
|
|
|
|
|
|
private final AtomicBoolean isScheduleRunning = new AtomicBoolean(false);
|
|
|
-
|
|
|
- private final DataSource dataSource;
|
|
|
- private final Map<Thread.State, Integer> stateCountMap = new EnumMap<>(Thread.State.class);
|
|
|
- private final SystemHealth systemHealth = new SystemHealth();
|
|
|
- private final DecimalFormat df = new DecimalFormat("#.##");
|
|
|
private String scheduleTime;
|
|
|
|
|
|
- public ApplicationScheduler(@Qualifier("dataSource") DataSource dataSource,
|
|
|
- ApplicationConfig config,
|
|
|
- TraceConfig traceConfig,
|
|
|
- ProcessStateService processStateService,
|
|
|
- UticTrafServerController controller) {
|
|
|
- this.dataSource = dataSource;
|
|
|
- this.config = config;
|
|
|
- this.traceConfig = traceConfig;
|
|
|
- this.processStateService = processStateService;
|
|
|
- this.controller = controller;
|
|
|
- }
|
|
|
|
|
|
@Scheduled(cron = "0 * * * * *")
|
|
|
public void scheduleProcess() {
|
|
|
@@ -61,20 +37,9 @@ public class ApplicationScheduler {
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
- try {
|
|
|
- this.systemHealth.checkHealth(false);
|
|
|
- loggingHealthCheck();
|
|
|
-
|
|
|
- double cpuUsage = this.systemHealth.getCpuUsage();
|
|
|
- if (cpuUsage > this.config.getCpuLimits()) {
|
|
|
- ApplicationRepository.processState.setErrDesc("CPU 사용율이 너무 높음: " + String.format("%.2f", cpuUsage));
|
|
|
- log.warn("[SKIP] ApplicationScheduler.scheduleProcess: High CPU Usage, Limit({} %), Current({} %), Schedule Job SKIP...",
|
|
|
- this.config.getCpuLimits(), String.format("%.2f", cpuUsage));
|
|
|
- loggingThreads();
|
|
|
- return;
|
|
|
- }
|
|
|
- } catch (Exception e) {
|
|
|
- log.error("ApplicationScheduler.scheduleProcess: System Health Check Exception {}", e.getMessage());
|
|
|
+ if (!this.healthService.checkSystemHealth()) {
|
|
|
+ // 시스템이 과부하이면 처리하지 않는다.
|
|
|
+ return;
|
|
|
}
|
|
|
|
|
|
// 이전 작업이 아직 실행 중이면 스킵(비동기 처리로 변경할 경우 대비-보혐)
|
|
|
@@ -106,69 +71,4 @@ public class ApplicationScheduler {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- private void loggingHealthCheck() {
|
|
|
- log.info("----------------------------------------------------------------------------------------------------------");
|
|
|
- loggingSystemHealth();
|
|
|
- logSessionStatus();
|
|
|
- loggingThreadState();
|
|
|
- log.info("----------------------------------------------------------------------------------------------------------");
|
|
|
- }
|
|
|
-
|
|
|
- private void loggingSystemHealth() {
|
|
|
- double loadAvg = this.systemHealth.getLoadAverage();
|
|
|
- String loadAvgStr = (loadAvg < 0) ? "N/A" : this.df.format(loadAvg);
|
|
|
- log.info("SYSTEM HEALTH: CPU {} Cores[{} %, LoadAvg({})], Memory[{} %, Used({} MB), Max({} MB)], Threads[{}, Peak({})], GC(T/R)[{}/{} ms]",
|
|
|
- this.systemHealth.getCpuCores(),
|
|
|
- this.df.format(this.systemHealth.getCpuUsage()),
|
|
|
- loadAvgStr,
|
|
|
- this.df.format(this.systemHealth.getMemUsage()),
|
|
|
- this.systemHealth.getUsedMemory(),
|
|
|
- this.systemHealth.getMaxMemory(),
|
|
|
- this.systemHealth.getThreadCount(),
|
|
|
- this.systemHealth.getPeakThreadCount(),
|
|
|
- this.systemHealth.getGcTotalAvgTime(),
|
|
|
- this.systemHealth.getGcRecentAvgTime());
|
|
|
- }
|
|
|
-
|
|
|
- private void logSessionStatus() {
|
|
|
- HikariDataSource hikariDataSource = (HikariDataSource) this.dataSource;
|
|
|
- HikariPoolMXBean poolStats = hikariDataSource.getHikariPoolMXBean();
|
|
|
- int totalConnections = poolStats.getTotalConnections();
|
|
|
- int activeConnections = poolStats.getActiveConnections();
|
|
|
- int idleConnections = poolStats.getIdleConnections();
|
|
|
- int threadsAwaiting = poolStats.getThreadsAwaitingConnection();
|
|
|
- log.info(" DB SESSION: UTIS, Total: {}, Active: {}, Idle: {}, Waiting: {}",
|
|
|
- totalConnections, activeConnections, idleConnections, threadsAwaiting);
|
|
|
- }
|
|
|
-
|
|
|
- private void loggingThreadState() {
|
|
|
- ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
|
|
|
- ThreadInfo[] infos = threadBean.dumpAllThreads(false, false);
|
|
|
- this.stateCountMap.clear();
|
|
|
- for (ThreadInfo info : infos) {
|
|
|
- Thread.State state = info.getThreadState();
|
|
|
- this.stateCountMap.put(state, this.stateCountMap.getOrDefault(state, 0) + 1);
|
|
|
- }
|
|
|
- StringBuilder sb = new StringBuilder();
|
|
|
- for (Map.Entry<Thread.State, Integer> entry : stateCountMap.entrySet()) {
|
|
|
- sb.append(String.format("%s(%d), ", entry.getKey().name(), entry.getValue()));
|
|
|
- }
|
|
|
- // 마지막 쉼표 제거
|
|
|
- if (sb.length() > 0) {
|
|
|
- sb.setLength(sb.length() - 2);
|
|
|
- }
|
|
|
- log.info(" THREAD STATE: {}", sb.toString());
|
|
|
- }
|
|
|
-
|
|
|
- private void loggingThreads() {
|
|
|
- ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
|
|
|
- ThreadInfo[] infos = threadBean.dumpAllThreads(false, false);
|
|
|
- this.stateCountMap.clear();
|
|
|
- for (ThreadInfo info : infos) {
|
|
|
- Thread.State state = info.getThreadState();
|
|
|
- this.stateCountMap.put(state, this.stateCountMap.getOrDefault(state, 0) + 1);
|
|
|
- log.info(" [" + info.getThreadId() + "] " + info.getThreadName() + " - " + info.getThreadState());
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
}
|