diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/AdminGpuUsageResponseDTO.java b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/AdminGpuUsageResponseDTO.java index 9a9c7eb8..dcc31613 100644 --- a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/AdminGpuUsageResponseDTO.java +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/AdminGpuUsageResponseDTO.java @@ -33,6 +33,7 @@ public record Process( @Schema(description = "소유자 사용자 번호. 미확인이거나 옛 Docker 컨테이너면 null", nullable = true) Long userId, @Schema(description = "소유 신청 번호. 미확인이거나 옛 Docker 컨테이너면 null", nullable = true) Long requestId, @Schema(description = "컨테이너 ID 앞 12자") String containerId, + @Schema(description = "이 컨테이너(Pod) 전체 RAM(MiB, working set). 이 프로세스만의 값이 아니라 같은 컨테이너의 행마다 같다. 모르면 null", nullable = true) Double containerRamMiB, @Schema(description = "프로세스 이름(파일 이름만)") String processName, @Schema(description = "GPU 메모리(MiB)") double memoryUsedMiB, @Schema(description = "실행된 지 지난 시간(초)") long elapsedSeconds) { diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/MyGpuUsageResponseDTO.java b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/MyGpuUsageResponseDTO.java index 9c0eec30..2428ac0d 100644 --- a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/MyGpuUsageResponseDTO.java +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/MyGpuUsageResponseDTO.java @@ -15,7 +15,8 @@ public record MyGpuUsageResponseDTO( @Schema(description = "서버(노드) 이름", example = "farm1") String nodeName, @Schema(description = "GPU별 현황") List gpus, @Schema(description = "프로세스·다른 사용자 정보를 읽었는지. false면 내 프로세스 없음·다른 사용자 0명이 '없음'이 아니라 '모름'이다") - boolean processesAvailable) { + boolean processesAvailable, + @Schema(description = "내 Pod 전체 RAM(MiB, working set). 모르면 null. 다른 사용자의 RAM은 담지 않는다", nullable = true) Double myRamMiB) { @Builder public record Gpu( diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageService.java b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageService.java index c0a878fe..24fd4698 100644 --- a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageService.java +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageService.java @@ -49,6 +49,11 @@ public List getGpuUsage(String node) { .toList(); } + private static Double ramMiB(Snapshot snap, String containerId) { + Double bytes = snap.podRamBytesByContainer().get(containerId); + return bytes == null ? null : bytes / MyGpuUsageService.MIB; + } + private AdminGpuUsageResponseDTO usageOf(String node, boolean failHard) { Snapshot snap; try { @@ -71,6 +76,7 @@ private AdminGpuUsageResponseDTO usageOf(String node, boolean failHard) { .userId(owner == null ? null : owner.userId()) .requestId(owner == null ? null : owner.requestId()) .containerId(containerId == null || containerId.isBlank() ? null : MyGpuUsageService.shortId(containerId)) + .containerRamMiB(containerId == null ? null : ramMiB(snap, containerId)) .processName(MyGpuUsageService.baseName(s.labels().get("process_name"))) .memoryUsedMiB(s.value() / MyGpuUsageService.MIB) .elapsedSeconds(snap.elapsed().getOrDefault(MyGpuUsageService.procKey(s), 0.0).longValue()) diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageService.java b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageService.java index dad5b42d..73c4f319 100644 --- a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageService.java +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageService.java @@ -47,6 +47,7 @@ @RequiredArgsConstructor public class MyGpuUsageService { + private static final java.util.regex.Pattern POD_NAME = java.util.regex.Pattern.compile("[A-Za-z0-9.-]+"); static final double MIB = 1024.0 * 1024.0; static final long CACHE_TTL_MS = 10_000; private static final ExecutorService POOL = Executors.newCachedThreadPool(r -> { @@ -62,11 +63,11 @@ public class MyGpuUsageService { private final Map cache = new ConcurrentHashMap<>(); private final Map locks = new ConcurrentHashMap<>(); - /** 노드 하나의 Prometheus 결과와 소유자 해석. processesAvailable=false면 프로세스 쪽 조회가 실패한 것이다. */ + /** 노드 하나의 Prometheus 결과와 소유자 해석. podRamBytesByContainer: 컨테이너 ID → 그 Pod 전체 RAM(바이트), 모르면 키 없음. processesAvailable=false면 프로세스 쪽 조회가 실패한 것이다. */ record Snapshot(List util, Map used, Map free, List procMem, Map elapsed, Map podOwnerByContainer, Map dockerOwnerByContainerGpu, - boolean processesAvailable) { + Map podRamBytesByContainer, boolean processesAvailable) { } /** @@ -91,7 +92,7 @@ public MyGpuUsageResponseDTO getMyGpuUsage(Long requestId, Long userId) { } String node = request.getNodeName(); if (request.getPodName() == null || node == null) { - return response(node, List.of(), true); + return response(node, List.of(), true, null); } Snapshot snap; @@ -153,11 +154,25 @@ public MyGpuUsageResponseDTO getMyGpuUsage(Long requestId, Long userId) { }) .sorted(Comparator.comparingInt(Gpu::index)) .toList(); - return response(node, gpus, snap.processesAvailable()); + return response(node, gpus, snap.processesAvailable(), podRamMiB(request.getPodName())); } - private static MyGpuUsageResponseDTO response(String node, List gpus, boolean processesAvailable) { - return MyGpuUsageResponseDTO.builder().nodeName(node).gpus(gpus).processesAvailable(processesAvailable).build(); + private static MyGpuUsageResponseDTO response(String node, List gpus, boolean processesAvailable, Double myRamMiB) { + return MyGpuUsageResponseDTO.builder().nodeName(node).gpus(gpus).processesAvailable(processesAvailable) + .myRamMiB(myRamMiB).build(); + } + + /** 내 Pod 전체 RAM(MiB). GPU를 안 써도 보이도록 별도로 조회하며, 실패·없음은 GPU 정보를 막지 않고 null(모름)이다. */ + private Double podRamMiB(String pod) { + if (!POD_NAME.matcher(pod).matches()) return null; + try { + List r = prometheusClient.querySeries( + "sum(container_memory_working_set_bytes{pod=\"" + pod + "\",container!=\"\",container!=\"POD\"})"); + return r.isEmpty() ? null : r.get(0).value() / MIB; + } catch (Exception e) { + log.warn("[MY-GPU] Pod RAM 조회 실패: pod={} {}", pod, e.toString()); + return null; + } } Set nodeModels(String node) { @@ -222,6 +237,7 @@ private Snapshot load(String node) { Map elapsed = new HashMap<>(); Map podOwners = new HashMap<>(); Map dockerOwners = new HashMap<>(); + Map ramByContainer = new HashMap<>(); boolean available = true; try { // exporter가 이 노드에서 프로세스 목록을 읽고 있는지: 없으면(옛 exporter·미배포) 또는 0이면 '프로세스 없음'이 아니라 '모름'이다. @@ -229,7 +245,7 @@ private Snapshot load(String node) { .stream().anyMatch(s -> s.value() == 1.0); if (!up) { log.warn("[MY-GPU] 프로세스 지표 없음(gpu_process_metrics_up): node={}", node); - return new Snapshot(util, used, free, List.of(), elapsed, podOwners, dockerOwners, false); + return new Snapshot(util, used, free, List.of(), elapsed, podOwners, dockerOwners, ramByContainer, false); } String uuids = util.stream().map(s -> s.labels().get("UUID")).filter(u -> u != null && u.matches("[A-Za-z0-9-]+")) .collect(Collectors.joining("|")); @@ -241,7 +257,7 @@ private Snapshot load(String node) { () -> prometheusClient.querySeries("gpu_process_elapsed_seconds" + psel), POOL); procMem = memF.join(); for (Sample s : elF.join()) elapsed.put(procKey(s), s.value()); - resolveOwners(procMem, podOwners, dockerOwners); + resolveOwners(procMem, podOwners, dockerOwners, ramByContainer); } } catch (Exception e) { // 프로세스 정보 없이도 GPU별 사용률은 보여 준다. 대신 '모름'임을 응답에 남긴다. @@ -249,7 +265,7 @@ private Snapshot load(String node) { procMem = List.of(); available = false; } - return new Snapshot(util, used, free, procMem, elapsed, podOwners, dockerOwners, available); + return new Snapshot(util, used, free, procMem, elapsed, podOwners, dockerOwners, ramByContainer, available); } /** @@ -267,7 +283,8 @@ static String hostSelector(String label, String node) { * Pod는 kube_pod_container_info의 container_id로 Pod 이름을 찾아 신청의 사용자로, * 옛 Docker 컨테이너는 docker_gpu_user_container_info의 ubuntu_username으로 찾는다. */ - private void resolveOwners(List procMem, Map podOwners, Map dockerOwners) { + private void resolveOwners(List procMem, Map podOwners, Map dockerOwners, + Map ramByContainer) { Set ids = procMem.stream().map(s -> s.labels().get("container_id")) .filter(id -> id != null && id.matches("[0-9a-f]{64}")).collect(Collectors.toSet()); if (ids.isEmpty()) return; @@ -280,6 +297,7 @@ private void resolveOwners(List procMem, Map podOwners, M if (cid != null && pod != null) podByContainer.put(cid.substring(cid.lastIndexOf('/') + 1), pod); } if (!podByContainer.isEmpty()) { + fillPodRam(podByContainer, ramByContainer); Map ownerByPod = new HashMap<>(); for (Request r : requestRepository.findAllByPodNameIn(new HashSet<>(podByContainer.values()))) { ownerByPod.put(r.getPodName(), new Owner(ownerKey(r), r.getUser().getUserId(), r.getUser().getName(), @@ -303,6 +321,24 @@ private void resolveOwners(List procMem, Map podOwners, M } } + /** Pod 전체 RAM(cAdvisor working set 합계)을 컨테이너 ID에 붙인다. 실패해도 소유자 해석은 계속한다. */ + private void fillPodRam(Map podByContainer, Map ramByContainer) { + Set pods = podByContainer.values().stream().filter(p -> POD_NAME.matcher(p).matches()).collect(Collectors.toSet()); + if (pods.isEmpty()) return; + try { + Map ramByPod = new HashMap<>(); + for (Sample s : prometheusClient.querySeries("sum by (pod) (container_memory_working_set_bytes{pod=~\"" + + String.join("|", pods) + "\",container!=\"\",container!=\"POD\"})")) { + if (s.labels().get("pod") != null) ramByPod.put(s.labels().get("pod"), s.value()); + } + podByContainer.forEach((cid, pod) -> { + if (ramByPod.containsKey(pod)) ramByContainer.put(cid, ramByPod.get(pod)); + }); + } catch (Exception e) { + log.warn("[MY-GPU] Pod RAM 조회 실패: {}", e.toString()); + } + } + /** 같은 사용자는 Pod든 옛 Docker 컨테이너든 같은 키가 되도록 우분투 계정을 쓴다. */ static String ownerKey(Request r) { String name = r.getUser().getUbuntuUsername(); diff --git a/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageServiceTest.java b/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageServiceTest.java index f9526742..efec48fa 100644 --- a/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageServiceTest.java +++ b/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageServiceTest.java @@ -49,11 +49,16 @@ private static Sample s(double v, String... kv) { } private static Snapshot snap(boolean available, List procMem, Map podOwners) { + return snap(available, procMem, podOwners, Map.of()); + } + + private static Snapshot snap(boolean available, List procMem, Map podOwners, + Map ramBytes) { List util = List.of(s(40, "UUID", "GPU-1", "gpu", "0", "modelName", "RTX 3090")); return new Snapshot(util, Map.of("GPU-1", s(10240, "UUID", "GPU-1")), Map.of("GPU-1", s(14336, "UUID", "GPU-1")), - procMem, Map.of("GPU-1/" + C_KNOWN + "/11", 3600.0), podOwners, Map.of(), available); + procMem, Map.of("GPU-1/" + C_KNOWN + "/11", 3600.0), podOwners, Map.of(), ramBytes, available); } @Test @@ -94,6 +99,21 @@ void unresolvedOwnerIsNull() { assertThat(p.elapsedSeconds()).isZero(); } + @Test + @DisplayName("컨테이너 RAM은 아는 컨테이너에만 MiB로 채우고 모르면 null이다") + void containerRamOnlyWhenKnown() { + Owner owner = new Owner("u7", 7L, "홍길동", "u7", 70L); + Sample known = s(5 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_KNOWN, "pid", "11", "process_name", "/b/python"); + Sample unknown = s(1 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_UNKNOWN, "pid", "1", "process_name", "/a"); + when(myGpuUsageService.snapshot("farm6")).thenReturn( + snap(true, List.of(known, unknown), Map.of(C_KNOWN, owner), Map.of(C_KNOWN, 3 * 1048576.0))); + + List procs = service.getGpuUsage("farm6").get(0).gpus().get(0).processes(); + + assertThat(procs.get(0).containerRamMiB()).isEqualTo(3.0); + assertThat(procs.get(1).containerRamMiB()).isNull(); + } + @Test @DisplayName("소유자를 찾으면 이름·계정·사용자·신청 번호를 채우고, 프로세스는 GPU 메모리 큰 순이다") void resolvedOwnerAndSortedByMemory() { diff --git a/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageServiceTest.java b/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageServiceTest.java index 632fd84c..19add89b 100644 --- a/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageServiceTest.java +++ b/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageServiceTest.java @@ -103,8 +103,14 @@ void aggregatesOthersWithoutIdentity() throws Exception { Request amy = request(3L, "pod-secret-y", "node-a"); when(requestRepository.findAllByPodNameIn(anyCollection())).thenReturn(List.of(mine, bob, amy)); + // 내 Pod의 RAM(512MiB)만 응답에 담긴다. 다른 사용자 Pod의 RAM은 어디에도 나오지 않는다. + when(prometheus.querySeries(contains("pod=\"pod-mine\""))).thenReturn(List.of(s(512 * 1048576.0))); + when(prometheus.querySeries(contains("sum by (pod)"))).thenReturn(List.of( + s(512 * 1048576.0, "pod", "pod-mine"), s(777 * 1048576.0, "pod", "pod-secret-x"))); + MyGpuUsageResponseDTO dto = service.getMyGpuUsage(10L, 1L); + assertThat(dto.myRamMiB()).isEqualTo(512.0); assertThat(dto.gpus()).hasSize(2); MyGpuUsageResponseDTO.Gpu g0 = dto.gpus().get(0); assertThat(g0.utilizationPercent()).isEqualTo(40); @@ -116,7 +122,22 @@ void aggregatesOthersWithoutIdentity() throws Exception { String json = new ObjectMapper().writeValueAsString(dto); assertThat(json).doesNotContain("secret").doesNotContain("bob").doesNotContain("amy") - .doesNotContain("train").doesNotContain("dave").doesNotContain("\"pid\""); + .doesNotContain("train").doesNotContain("dave").doesNotContain("\"pid\"").doesNotContain("777"); + } + + @Test + @DisplayName("RAM 조회가 실패해도 GPU 정보는 주고 myRamMiB만 null이다") + void ramFailureDoesNotBreakGpuView() { + Request mine = request(1L, "pod-mine", "node-a"); + when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(mine)); + when(prometheus.querySeries(contains("DCGM_FI_DEV_GPU_UTIL"))).thenReturn(List.of( + s(40, "UUID", "GPU-1", "gpu", "0", "modelName", "RTX 3090"))); + when(prometheus.querySeries(contains("container_memory_working_set_bytes"))).thenThrow(new RuntimeException("down")); + + MyGpuUsageResponseDTO dto = service.getMyGpuUsage(10L, 1L); + + assertThat(dto.myRamMiB()).isNull(); + assertThat(dto.gpus()).hasSize(1); } @Test