Skip to content

HDDS-15837. Add health-aware container ID listing in SCM #10736

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
szetszwo merged 2 commits into from
Jul 14, 2026
Merged

HDDS-15837. Add health-aware container ID listing in SCM #10736

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f50fd47
HDDS-15837. Add health-aware container ID listing in SCM
sarvekshayr Jul 13, 2026
f60457d
Refactor getContainerIDs method signature
sarvekshayr Jul 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion ...-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,12 @@ default List<ContainerInfo> getContainers() {
* Usually the count will be replaced with a very big
* value instead of being unlimited in case the db is very big.
* @param state container state
* @param healthState container health state
*
* @return a list of container IDs.
*/
List<ContainerID> getContainerIDs(ContainerID startID, int count, LifeCycleState state);
List<ContainerID> getContainerIDs(ContainerID startID, int count, LifeCycleState state,
ContainerHealthState healthState);

/**
* Returns containers under certain conditions.
Expand Down
5 changes: 3 additions & 2 deletions ...s/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,10 @@ public List<ContainerInfo> getContainers(ReplicationType type) {
@Override
public List<ContainerID> getContainerIDs(final ContainerID startID,
final int count,
final LifeCycleState state) {
final LifeCycleState state,
final ContainerHealthState healthState) {
scmContainerManagerMetrics.incNumListContainersOps();
return containerStateManager.getContainerIDs(state, startID, count);
return containerStateManager.getContainerIDs(state, healthState, startID, count);
}

@Override
Expand Down
5 changes: 3 additions & 2 deletions .../server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,14 @@ public interface ContainerStateManager extends SCMHandler {
boolean contains(ContainerID containerID);

/**
* Get {@link ContainerID}s for the given state.
* Get {@link ContainerID}s for the given optional lifeCycleState and healthState.
*
* @param start the start {@link ContainerID} (inclusive)
* @param count the size limit
* @return a list of {@link ContainerID};
*/
List<ContainerID> getContainerIDs(LifeCycleState state, ContainerID start, int count);
List<ContainerID> getContainerIDs(LifeCycleState state, ContainerHealthState healthState,
ContainerID start, int count);

/**
* Get {@link ContainerInfo}s.
Expand Down
5 changes: 3 additions & 2 deletions ...ver-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,9 +281,10 @@ private void initialize() throws IOException {
}

@Override
public List<ContainerID> getContainerIDs(LifeCycleState state, ContainerID start, int count) {
public List<ContainerID> getContainerIDs(LifeCycleState state, ContainerHealthState healthState,
ContainerID start, int count) {
try (AutoCloseableLock ignored = readLock()) {
return containers.getContainerIDs(state, start, count);
return containers.getContainerIDs(state, healthState, start, count);
}
}

Expand Down
49 changes: 43 additions & 6 deletions ...rver-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.hadoop.hdds.scm.container.states;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.NavigableMap;
import java.util.Objects;
Expand All @@ -26,6 +28,7 @@
import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.container.ContainerHealthState;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
Expand Down Expand Up @@ -118,6 +121,23 @@ List<ContainerInfo> getInfos(ContainerID start, int count) {
.collect(Collectors.toList());
}

List<ContainerID> getContainerIDs(ContainerID start, int count, ContainerHealthState healthState) {
Objects.requireNonNull(start, "start == null");
Preconditions.assertTrue(count >= 0, "count < 0");

final List<ContainerID> result = new ArrayList<>(1024);
for (ContainerEntry entry : map.tailMap(start).values()) {
ContainerInfo info = entry.getInfo();
if (healthState == null || info.getHealthState() == healthState) {
result.add(info.containerID());
if (result.size() >= count) {
break;
}
}
}
return result;
}

Set<ContainerReplica> getReplicas(ContainerID id) {
Objects.requireNonNull(id, "id == null");
final ContainerEntry entry = map.get(id);
Expand Down Expand Up @@ -261,17 +281,34 @@ public void updateState(ContainerID containerID, LifeCycleState currentState,
}

/**
* Returns container IDs matching given optional lifeCycleState and healthState,
* in ascending {@link ContainerID} order starting from {@code start} (inclusive).
*
* @param state the state of the containers
* @param start the start id
* @param count the maximum size of the returned list
* @return a list of sorted {@link ContainerID}s
*/
public List<ContainerID> getContainerIDs(LifeCycleState state, ContainerID start, int count) {
Preconditions.assertTrue(count >= 0, "count < 0");
return lifeCycleStateMap.tailMap(state, start).keySet().stream()
.limit(count)
.collect(Collectors.toList());
public List<ContainerID> getContainerIDs(LifeCycleState lifeCycleState,
ContainerHealthState healthState, ContainerID start, int count) {
if (count == 0) {
return Collections.emptyList();
}
Preconditions.assertTrue(count > 0, "count < 0");

if (lifeCycleState == null) {
return containerMap.getContainerIDs(start, count, healthState);
}

final List<ContainerID> result = new ArrayList<>(Math.min(count, 1024));
for (ContainerInfo info : lifeCycleStateMap.tailMap(lifeCycleState, start).values()) {
if (healthState == null || info.getHealthState() == healthState) {
result.add(info.containerID());
if (result.size() >= count) {
break;
}
}
}
return result;
}

public List<ContainerInfo> getContainerInfos(ContainerID start, int count) {
Expand Down
83 changes: 43 additions & 40 deletions ...scm/src/main/java/org/apache/hadoop/hdds/scm/ha/invoker/ContainerStateManagerInvoker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.container.ContainerHealthState;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
Expand Down Expand Up @@ -109,8 +110,9 @@ public int getContainerCount(LifeCycleState arg0) {
}

@Override
public List<ContainerID> getContainerIDs(LifeCycleState arg0, ContainerID arg1, int arg2) {
return invoker.getImpl().getContainerIDs(arg0, arg1, arg2);
public List<ContainerID> getContainerIDs(LifeCycleState arg0, ContainerHealthState arg1, ContainerID arg2, int
arg3) {
return invoker.getImpl().getContainerIDs(arg0, arg1, arg2, arg3);
}

@Override
Expand Down Expand Up @@ -217,94 +219,95 @@ public Message invokeLocal(String methodName, Object[] p) throws Exception {

case "getContainerIDs":
final LifeCycleState arg4 = p.length > 0 ? (LifeCycleState) p[0] : null;
final ContainerID arg5 = p.length > 1 ? (ContainerID) p[1] : null;
final int arg6 = p.length > 2 ? (int) p[2] : 0;
final ContainerHealthState arg5 = p.length > 1 ? (ContainerHealthState) p[1] : null;
final ContainerID arg6 = p.length > 2 ? (ContainerID) p[2] : null;
final int arg7 = p.length > 3 ? (int) p[3] : 0;
returnType = List.class;
returnValue = getImpl().getContainerIDs(arg4, arg5, arg6);
returnValue = getImpl().getContainerIDs(arg4, arg5, arg6, arg7);
break;

case "getContainerInfos":
if (p.length == 1 && (p[0] == null || LifeCycleState.class.isInstance(p[0]))) {
final LifeCycleState arg7 = (LifeCycleState) p[0];
final LifeCycleState arg8 = (LifeCycleState) p[0];
returnType = List.class;
returnValue = getImpl().getContainerInfos(arg7);
returnValue = getImpl().getContainerInfos(arg8);
break;
}
if (p.length == 1 && (p[0] == null || ReplicationType.class.isInstance(p[0]))) {
final ReplicationType arg8 = (ReplicationType) p[0];
final ReplicationType arg9 = (ReplicationType) p[0];
returnType = List.class;
returnValue = getImpl().getContainerInfos(arg8);
returnValue = getImpl().getContainerInfos(arg9);
break;
}
if (p.length == 2 && (p[0] == null || ContainerID.class.isInstance(p[0])) && p[1] instanceof Integer) {
final ContainerID arg9 = (ContainerID) p[0];
final int arg10 = (int) p[1];
final ContainerID arg10 = (ContainerID) p[0];
final int arg11 = (int) p[1];
returnType = List.class;
returnValue = getImpl().getContainerInfos(arg9, arg10);
returnValue = getImpl().getContainerInfos(arg10, arg11);
break;
}
if (p.length == 3 && (p[0] == null || LifeCycleState.class.isInstance(p[0])) && (p[1] == null ||
ContainerID.class.isInstance(p[1])) && p[2] instanceof Integer) {
final LifeCycleState arg11 = (LifeCycleState) p[0];
final ContainerID arg12 = (ContainerID) p[1];
final int arg13 = (int) p[2];
final LifeCycleState arg12 = (LifeCycleState) p[0];
final ContainerID arg13 = (ContainerID) p[1];
final int arg14 = (int) p[2];
returnType = List.class;
returnValue = getImpl().getContainerInfos(arg11, arg12, arg13);
returnValue = getImpl().getContainerInfos(arg12, arg13, arg14);
break;
}
throw new IllegalArgumentException("Method not found: " + methodName + " in ContainerStateManager");

case "getContainerReplicas":
final ContainerID arg14 = p.length > 0 ? (ContainerID) p[0] : null;
final ContainerID arg15 = p.length > 0 ? (ContainerID) p[0] : null;
returnType = Set.class;
returnValue = getImpl().getContainerReplicas(arg14);
returnValue = getImpl().getContainerReplicas(arg15);
break;

case "getMatchingContainer":
final long arg15 = p.length > 0 ? (long) p[0] : 0L;
final String arg16 = p.length > 1 ? (String) p[1] : null;
final PipelineID arg17 = p.length > 2 ? (PipelineID) p[2] : null;
final NavigableSet arg18 = p.length > 3 ? (NavigableSet) p[3] : null;
final long arg16 = p.length > 0 ? (long) p[0] : 0L;
final String arg17 = p.length > 1 ? (String) p[1] : null;
final PipelineID arg18 = p.length > 2 ? (PipelineID) p[2] : null;
final NavigableSet arg19 = p.length > 3 ? (NavigableSet) p[3] : null;
returnType = ContainerInfo.class;
returnValue = getImpl().getMatchingContainer(arg15, arg16, arg17, arg18);
returnValue = getImpl().getMatchingContainer(arg16, arg17, arg18, arg19);
break;

case "reinitialize":
final Table arg19 = p.length > 0 ? (Table) p[0] : null;
getImpl().reinitialize(arg19);
final Table arg20 = p.length > 0 ? (Table) p[0] : null;
getImpl().reinitialize(arg20);
return Message.EMPTY;

case "removeContainer":
final HddsProtos.ContainerID arg20 = p.length > 0 ? (HddsProtos.ContainerID) p[0] : null;
getImpl().removeContainer(arg20);
final HddsProtos.ContainerID arg21 = p.length > 0 ? (HddsProtos.ContainerID) p[0] : null;
getImpl().removeContainer(arg21);
return Message.EMPTY;

case "removeContainerReplica":
final ContainerReplica arg21 = p.length > 0 ? (ContainerReplica) p[0] : null;
getImpl().removeContainerReplica(arg21);
final ContainerReplica arg22 = p.length > 0 ? (ContainerReplica) p[0] : null;
getImpl().removeContainerReplica(arg22);
return Message.EMPTY;

case "transitionDeletingOrDeletedToTargetState":
final HddsProtos.ContainerID arg22 = p.length > 0 ? (HddsProtos.ContainerID) p[0] : null;
final LifeCycleState arg23 = p.length > 1 ? (LifeCycleState) p[1] : null;
getImpl().transitionDeletingOrDeletedToTargetState(arg22, arg23);
final HddsProtos.ContainerID arg23 = p.length > 0 ? (HddsProtos.ContainerID) p[0] : null;
final LifeCycleState arg24 = p.length > 1 ? (LifeCycleState) p[1] : null;
getImpl().transitionDeletingOrDeletedToTargetState(arg23, arg24);
return Message.EMPTY;

case "updateContainerInfo":
final ContainerInfoProto arg24 = p.length > 0 ? (ContainerInfoProto) p[0] : null;
getImpl().updateContainerInfo(arg24);
final ContainerInfoProto arg25 = p.length > 0 ? (ContainerInfoProto) p[0] : null;
getImpl().updateContainerInfo(arg25);
return Message.EMPTY;

case "updateContainerReplica":
final ContainerReplica arg25 = p.length > 0 ? (ContainerReplica) p[0] : null;
getImpl().updateContainerReplica(arg25);
final ContainerReplica arg26 = p.length > 0 ? (ContainerReplica) p[0] : null;
getImpl().updateContainerReplica(arg26);
return Message.EMPTY;

case "updateContainerStateWithSequenceId":
final HddsProtos.ContainerID arg26 = p.length > 0 ? (HddsProtos.ContainerID) p[0] : null;
final LifeCycleEvent arg27 = p.length > 1 ? (LifeCycleEvent) p[1] : null;
final Long arg28 = p.length > 2 ? (Long) p[2] : null;
getImpl().updateContainerStateWithSequenceId(arg26, arg27, arg28);
final HddsProtos.ContainerID arg27 = p.length > 0 ? (HddsProtos.ContainerID) p[0] : null;
final LifeCycleEvent arg28 = p.length > 1 ? (LifeCycleEvent) p[1] : null;
final Long arg29 = p.length > 2 ? (Long) p[2] : null;
getImpl().updateContainerStateWithSequenceId(arg27, arg28, arg29);
return Message.EMPTY;

default:
Expand Down
2 changes: 1 addition & 1 deletion ...s/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1569,7 +1569,7 @@ public List<ContainerID> getListOfContainerIDs(
auditMap.put("state", String.valueOf(state));
try {
List<ContainerID> results = scm.getContainerManager().getContainerIDs(
startContainerID, count, state);
startContainerID, count, state, null);
AUDIT.logReadSuccess(buildAuditMessageForSuccess(
SCMAction.LIST_CONTAINER_IDS, auditMap));
return results;
Expand Down
2 changes: 1 addition & 1 deletion ...ver-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -405,7 +405,7 @@ public void testGetContainerIDs() throws IOException {
containerStateManager.addContainer(closedContainerInfo.getProtobuf());

assertEquals(1, containerStateManager.getContainerIDs(
HddsProtos.LifeCycleState.CLOSED, ContainerID.MIN, 10).size());
HddsProtos.LifeCycleState.CLOSED, ContainerHealthState.HEALTHY, ContainerID.MIN, 10).size());
}

@Test
Expand Down
Loading