Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ public record Process(
@Schema(description = "소유자 사용자 번호. 미확인이거나 옛 Docker 컨테이너면 null", nullable = true) Long userId,
@Schema(description = "소유 신청 번호. 미확인이거나 옛 Docker 컨테이너면 null", nullable = true) Long requestId,
@Schema(description = "컨테이너 ID 앞 12자") String containerId,
@Schema(description = "이 컨테이너(Pod) 전체 RAM(MiB, working set). 이 프로세스만의 값이 아니라 같은 컨테이너의 행마다 같다. 모르면 null", nullable = true) Double containerRamMiB,
@Schema(description = "프로세스 이름(파일 이름만)") String processName,
@Schema(description = "GPU 메모리(MiB)") double memoryUsedMiB,
@Schema(description = "실행된 지 지난 시간(초)") long elapsedSeconds) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ public record MyGpuUsageResponseDTO(
@Schema(description = "서버(노드) 이름", example = "farm1") String nodeName,
@Schema(description = "GPU별 현황") List<Gpu> gpus,
@Schema(description = "프로세스·다른 사용자 정보를 읽었는지. false면 내 프로세스 없음·다른 사용자 0명이 '없음'이 아니라 '모름'이다")
boolean processesAvailable) {
boolean processesAvailable,
@Schema(description = "내 Pod 전체 RAM(MiB, working set). 모르면 null. 다른 사용자의 RAM은 담지 않는다", nullable = true) Double myRamMiB) {

@Builder
public record Gpu(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@ public List<AdminGpuUsageResponseDTO> getGpuUsage(String node) {
.toList();
}

private static Double ramMiB(Snapshot snap, String containerId) {
Double bytes = snap.podRamBytesByContainer().get(containerId);
return bytes == null ? null : bytes / MyGpuUsageService.MIB;
}

private AdminGpuUsageResponseDTO usageOf(String node, boolean failHard) {
Snapshot snap;
try {
Expand All @@ -71,6 +76,7 @@ private AdminGpuUsageResponseDTO usageOf(String node, boolean failHard) {
.userId(owner == null ? null : owner.userId())
.requestId(owner == null ? null : owner.requestId())
.containerId(containerId == null || containerId.isBlank() ? null : MyGpuUsageService.shortId(containerId))
.containerRamMiB(containerId == null ? null : ramMiB(snap, containerId))
.processName(MyGpuUsageService.baseName(s.labels().get("process_name")))
.memoryUsedMiB(s.value() / MyGpuUsageService.MIB)
.elapsedSeconds(snap.elapsed().getOrDefault(MyGpuUsageService.procKey(s), 0.0).longValue())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
@RequiredArgsConstructor
public class MyGpuUsageService {

private static final java.util.regex.Pattern POD_NAME = java.util.regex.Pattern.compile("[A-Za-z0-9.-]+");
static final double MIB = 1024.0 * 1024.0;
static final long CACHE_TTL_MS = 10_000;
private static final ExecutorService POOL = Executors.newCachedThreadPool(r -> {
Expand All @@ -62,11 +63,11 @@ public class MyGpuUsageService {
private final Map<String, Cached> cache = new ConcurrentHashMap<>();
private final Map<String, Object> locks = new ConcurrentHashMap<>();

/** 노드 하나의 Prometheus 결과와 소유자 해석. processesAvailable=false면 프로세스 쪽 조회가 실패한 것이다. */
/** 노드 하나의 Prometheus 결과와 소유자 해석. podRamBytesByContainer: 컨테이너 ID → 그 Pod 전체 RAM(바이트), 모르면 키 없음. processesAvailable=false면 프로세스 쪽 조회가 실패한 것이다. */
record Snapshot(List<Sample> util, Map<String, Sample> used, Map<String, Sample> free,
List<Sample> procMem, Map<String, Double> elapsed,
Map<String, Owner> podOwnerByContainer, Map<String, Owner> dockerOwnerByContainerGpu,
boolean processesAvailable) {
Map<String, Double> podRamBytesByContainer, boolean processesAvailable) {
}

/**
Expand All @@ -91,7 +92,7 @@ public MyGpuUsageResponseDTO getMyGpuUsage(Long requestId, Long userId) {
}
String node = request.getNodeName();
if (request.getPodName() == null || node == null) {
return response(node, List.of(), true);
return response(node, List.of(), true, null);
}

Snapshot snap;
Expand Down Expand Up @@ -153,11 +154,25 @@ public MyGpuUsageResponseDTO getMyGpuUsage(Long requestId, Long userId) {
})
.sorted(Comparator.comparingInt(Gpu::index))
.toList();
return response(node, gpus, snap.processesAvailable());
return response(node, gpus, snap.processesAvailable(), podRamMiB(request.getPodName()));
}

private static MyGpuUsageResponseDTO response(String node, List<Gpu> gpus, boolean processesAvailable) {
return MyGpuUsageResponseDTO.builder().nodeName(node).gpus(gpus).processesAvailable(processesAvailable).build();
private static MyGpuUsageResponseDTO response(String node, List<Gpu> gpus, boolean processesAvailable, Double myRamMiB) {
return MyGpuUsageResponseDTO.builder().nodeName(node).gpus(gpus).processesAvailable(processesAvailable)
.myRamMiB(myRamMiB).build();
}

/** 내 Pod 전체 RAM(MiB). GPU를 안 써도 보이도록 별도로 조회하며, 실패·없음은 GPU 정보를 막지 않고 null(모름)이다. */
private Double podRamMiB(String pod) {
if (!POD_NAME.matcher(pod).matches()) return null;
try {
List<Sample> r = prometheusClient.querySeries(
"sum(container_memory_working_set_bytes{pod=\"" + pod + "\",container!=\"\",container!=\"POD\"})");
return r.isEmpty() ? null : r.get(0).value() / MIB;
} catch (Exception e) {
log.warn("[MY-GPU] Pod RAM 조회 실패: pod={} {}", pod, e.toString());
return null;
}
}

Set<String> nodeModels(String node) {
Expand Down Expand Up @@ -222,14 +237,15 @@ private Snapshot load(String node) {
Map<String, Double> elapsed = new HashMap<>();
Map<String, Owner> podOwners = new HashMap<>();
Map<String, Owner> dockerOwners = new HashMap<>();
Map<String, Double> ramByContainer = new HashMap<>();
boolean available = true;
try {
// exporter가 이 노드에서 프로세스 목록을 읽고 있는지: 없으면(옛 exporter·미배포) 또는 0이면 '프로세스 없음'이 아니라 '모름'이다.
boolean up = prometheusClient.querySeries("gpu_process_metrics_up" + hostSelector("hostname", node))
.stream().anyMatch(s -> s.value() == 1.0);
if (!up) {
log.warn("[MY-GPU] 프로세스 지표 없음(gpu_process_metrics_up): node={}", node);
return new Snapshot(util, used, free, List.of(), elapsed, podOwners, dockerOwners, false);
return new Snapshot(util, used, free, List.of(), elapsed, podOwners, dockerOwners, ramByContainer, false);
}
String uuids = util.stream().map(s -> s.labels().get("UUID")).filter(u -> u != null && u.matches("[A-Za-z0-9-]+"))
.collect(Collectors.joining("|"));
Expand All @@ -241,15 +257,15 @@ private Snapshot load(String node) {
() -> prometheusClient.querySeries("gpu_process_elapsed_seconds" + psel), POOL);
procMem = memF.join();
for (Sample s : elF.join()) elapsed.put(procKey(s), s.value());
resolveOwners(procMem, podOwners, dockerOwners);
resolveOwners(procMem, podOwners, dockerOwners, ramByContainer);
}
} catch (Exception e) {
// 프로세스 정보 없이도 GPU별 사용률은 보여 준다. 대신 '모름'임을 응답에 남긴다.
log.warn("[MY-GPU] 프로세스 지표 조회 실패: node={} {}", node, e.toString());
procMem = List.of();
available = false;
}
return new Snapshot(util, used, free, procMem, elapsed, podOwners, dockerOwners, available);
return new Snapshot(util, used, free, procMem, elapsed, podOwners, dockerOwners, ramByContainer, available);
}

/**
Expand All @@ -267,7 +283,8 @@ static String hostSelector(String label, String node) {
* Pod는 kube_pod_container_info의 container_id로 Pod 이름을 찾아 신청의 사용자로,
* 옛 Docker 컨테이너는 docker_gpu_user_container_info의 ubuntu_username으로 찾는다.
*/
private void resolveOwners(List<Sample> procMem, Map<String, Owner> podOwners, Map<String, Owner> dockerOwners) {
private void resolveOwners(List<Sample> procMem, Map<String, Owner> podOwners, Map<String, Owner> dockerOwners,
Map<String, Double> ramByContainer) {
Set<String> ids = procMem.stream().map(s -> s.labels().get("container_id"))
.filter(id -> id != null && id.matches("[0-9a-f]{64}")).collect(Collectors.toSet());
if (ids.isEmpty()) return;
Expand All @@ -280,6 +297,7 @@ private void resolveOwners(List<Sample> procMem, Map<String, Owner> podOwners, M
if (cid != null && pod != null) podByContainer.put(cid.substring(cid.lastIndexOf('/') + 1), pod);
}
if (!podByContainer.isEmpty()) {
fillPodRam(podByContainer, ramByContainer);
Map<String, Owner> ownerByPod = new HashMap<>();
for (Request r : requestRepository.findAllByPodNameIn(new HashSet<>(podByContainer.values()))) {
ownerByPod.put(r.getPodName(), new Owner(ownerKey(r), r.getUser().getUserId(), r.getUser().getName(),
Expand All @@ -303,6 +321,24 @@ private void resolveOwners(List<Sample> procMem, Map<String, Owner> podOwners, M
}
}

/** Pod 전체 RAM(cAdvisor working set 합계)을 컨테이너 ID에 붙인다. 실패해도 소유자 해석은 계속한다. */
private void fillPodRam(Map<String, String> podByContainer, Map<String, Double> ramByContainer) {
Set<String> pods = podByContainer.values().stream().filter(p -> POD_NAME.matcher(p).matches()).collect(Collectors.toSet());
if (pods.isEmpty()) return;
try {
Map<String, Double> ramByPod = new HashMap<>();
for (Sample s : prometheusClient.querySeries("sum by (pod) (container_memory_working_set_bytes{pod=~\""
+ String.join("|", pods) + "\",container!=\"\",container!=\"POD\"})")) {
if (s.labels().get("pod") != null) ramByPod.put(s.labels().get("pod"), s.value());
}
podByContainer.forEach((cid, pod) -> {
if (ramByPod.containsKey(pod)) ramByContainer.put(cid, ramByPod.get(pod));
});
} catch (Exception e) {
log.warn("[MY-GPU] Pod RAM 조회 실패: {}", e.toString());
}
}

/** 같은 사용자는 Pod든 옛 Docker 컨테이너든 같은 키가 되도록 우분투 계정을 쓴다. */
static String ownerKey(Request r) {
String name = r.getUser().getUbuntuUsername();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,16 @@ private static Sample s(double v, String... kv) {
}

private static Snapshot snap(boolean available, List<Sample> procMem, Map<String, Owner> podOwners) {
return snap(available, procMem, podOwners, Map.of());
}

private static Snapshot snap(boolean available, List<Sample> procMem, Map<String, Owner> podOwners,
Map<String, Double> ramBytes) {
List<Sample> util = List.of(s(40, "UUID", "GPU-1", "gpu", "0", "modelName", "RTX 3090"));
return new Snapshot(util,
Map.of("GPU-1", s(10240, "UUID", "GPU-1")),
Map.of("GPU-1", s(14336, "UUID", "GPU-1")),
procMem, Map.of("GPU-1/" + C_KNOWN + "/11", 3600.0), podOwners, Map.of(), available);
procMem, Map.of("GPU-1/" + C_KNOWN + "/11", 3600.0), podOwners, Map.of(), ramBytes, available);
}

@Test
Expand Down Expand Up @@ -94,6 +99,21 @@ void unresolvedOwnerIsNull() {
assertThat(p.elapsedSeconds()).isZero();
}

@Test
@DisplayName("컨테이너 RAM은 아는 컨테이너에만 MiB로 채우고 모르면 null이다")
void containerRamOnlyWhenKnown() {
Owner owner = new Owner("u7", 7L, "홍길동", "u7", 70L);
Sample known = s(5 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_KNOWN, "pid", "11", "process_name", "/b/python");
Sample unknown = s(1 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_UNKNOWN, "pid", "1", "process_name", "/a");
when(myGpuUsageService.snapshot("farm6")).thenReturn(
snap(true, List.of(known, unknown), Map.of(C_KNOWN, owner), Map.of(C_KNOWN, 3 * 1048576.0)));

List<AdminGpuUsageResponseDTO.Process> procs = service.getGpuUsage("farm6").get(0).gpus().get(0).processes();

assertThat(procs.get(0).containerRamMiB()).isEqualTo(3.0);
assertThat(procs.get(1).containerRamMiB()).isNull();
}

@Test
@DisplayName("소유자를 찾으면 이름·계정·사용자·신청 번호를 채우고, 프로세스는 GPU 메모리 큰 순이다")
void resolvedOwnerAndSortedByMemory() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,14 @@ void aggregatesOthersWithoutIdentity() throws Exception {
Request amy = request(3L, "pod-secret-y", "node-a");
when(requestRepository.findAllByPodNameIn(anyCollection())).thenReturn(List.of(mine, bob, amy));

// 내 Pod의 RAM(512MiB)만 응답에 담긴다. 다른 사용자 Pod의 RAM은 어디에도 나오지 않는다.
when(prometheus.querySeries(contains("pod=\"pod-mine\""))).thenReturn(List.of(s(512 * 1048576.0)));
when(prometheus.querySeries(contains("sum by (pod)"))).thenReturn(List.of(
s(512 * 1048576.0, "pod", "pod-mine"), s(777 * 1048576.0, "pod", "pod-secret-x")));

MyGpuUsageResponseDTO dto = service.getMyGpuUsage(10L, 1L);

assertThat(dto.myRamMiB()).isEqualTo(512.0);
assertThat(dto.gpus()).hasSize(2);
MyGpuUsageResponseDTO.Gpu g0 = dto.gpus().get(0);
assertThat(g0.utilizationPercent()).isEqualTo(40);
Expand All @@ -116,7 +122,22 @@ void aggregatesOthersWithoutIdentity() throws Exception {

String json = new ObjectMapper().writeValueAsString(dto);
assertThat(json).doesNotContain("secret").doesNotContain("bob").doesNotContain("amy")
.doesNotContain("train").doesNotContain("dave").doesNotContain("\"pid\"");
.doesNotContain("train").doesNotContain("dave").doesNotContain("\"pid\"").doesNotContain("777");
}

@Test
@DisplayName("RAM 조회가 실패해도 GPU 정보는 주고 myRamMiB만 null이다")
void ramFailureDoesNotBreakGpuView() {
Request mine = request(1L, "pod-mine", "node-a");
when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(mine));
when(prometheus.querySeries(contains("DCGM_FI_DEV_GPU_UTIL"))).thenReturn(List.of(
s(40, "UUID", "GPU-1", "gpu", "0", "modelName", "RTX 3090")));
when(prometheus.querySeries(contains("container_memory_working_set_bytes"))).thenThrow(new RuntimeException("down"));

MyGpuUsageResponseDTO dto = service.getMyGpuUsage(10L, 1L);

assertThat(dto.myRamMiB()).isNull();
assertThat(dto.gpus()).hasSize(1);
}

@Test
Expand Down
Loading