feat(wcs): recover execution state after restart
This commit is contained in:
@@ -0,0 +1,420 @@
|
||||
package com.ruoyi.wcs.application.recovery;
|
||||
|
||||
import com.ruoyi.wcs.application.device.ReconcileResult;
|
||||
import com.ruoyi.wcs.application.device.ReconcileStatus;
|
||||
import com.ruoyi.wcs.domain.recovery.ControlStatus;
|
||||
import com.ruoyi.wcs.domain.recovery.RecoveryCase;
|
||||
import com.ruoyi.wcs.domain.recovery.RecoveryEvidence;
|
||||
import com.ruoyi.wcs.domain.recovery.StartupRecoveryCandidate;
|
||||
import com.ruoyi.wcs.domain.recovery.StartupRecoveryResult;
|
||||
import com.ruoyi.wcs.domain.shared.DomainRuleViolation;
|
||||
import com.ruoyi.wcs.domain.shared.IdGenerator;
|
||||
import com.ruoyi.wcs.domain.execution.DeviceSnapshot;
|
||||
import com.ruoyi.wcs.port.in.DeviceSnapshotUseCase;
|
||||
import com.ruoyi.wcs.port.out.ControlLeaseRepository;
|
||||
import com.ruoyi.wcs.port.out.DeviceSnapshotGateway;
|
||||
import com.ruoyi.wcs.port.out.RecoveryCaseRepository;
|
||||
import com.ruoyi.wcs.port.out.StartupRecoveryDataProvider;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
class StartupRecoveryServiceTest {
|
||||
|
||||
@Test
|
||||
void startupKeepsDispatchPausedUntilAllActiveDevicesAreReconciled() {
|
||||
FakeRecoveryCases cases = new FakeRecoveryCases();
|
||||
RecoveryApplicationService recovery = recoveryService(cases);
|
||||
StartupRecoveryDataProvider data = new StartupRecoveryDataProvider() {
|
||||
@Override
|
||||
public List<StartupRecoveryCandidate> loadActiveCandidates() {
|
||||
return Arrays.asList(
|
||||
candidate("ORDER-1", "VEH-1", "CMD-1", "NODE:A"),
|
||||
candidate("ORDER-2", "VEH-2", "CMD-2", "LIFT:L1"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> findUnexplainedGlobalResourceKeys() {
|
||||
return Collections.singleton("LIFT:L1");
|
||||
}
|
||||
};
|
||||
DeviceSnapshotUseCase reconciler = new DeviceSnapshotUseCase() {
|
||||
@Override
|
||||
public ReconcileResult reconcile(DeviceSnapshot snapshot) {
|
||||
return new ReconcileResult("VEH-1".equals(snapshot.getDeviceId())
|
||||
? ReconcileStatus.CONSISTENT
|
||||
: ReconcileStatus.RECOVERY_REQUIRED,
|
||||
snapshot.getCommandKey(), Collections.<String>emptyList(),
|
||||
"VEH-1".equals(snapshot.getDeviceId())
|
||||
? null : "TASK_NUMBER_MISMATCH");
|
||||
}
|
||||
};
|
||||
StartupRecoveryService service = new StartupRecoveryService(
|
||||
new FakeLease(), data, new FakeSnapshots(), reconciler,
|
||||
recovery, cases);
|
||||
|
||||
StartupRecoveryResult result = service.recover();
|
||||
|
||||
assertEquals(ControlStatus.PAUSED, result.getControlStatus());
|
||||
assertEquals(Collections.singleton("VEH-2"),
|
||||
result.getIsolatedDeviceIds());
|
||||
assertEquals(Collections.singleton("ORDER-1"), result.getResumedOrderIds());
|
||||
assertEquals(Collections.singleton("LIFT:L1"),
|
||||
result.getIsolatedResourceKeys());
|
||||
assertEquals(1, result.getUnresolvedRecoveryCaseIds().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void startupPartiallyEnablesWhenOnlyDeviceScopedWorkIsIsolated() {
|
||||
FakeRecoveryCases cases = new FakeRecoveryCases();
|
||||
RecoveryApplicationService recovery = recoveryService(cases);
|
||||
StartupRecoveryDataProvider data = new StartupRecoveryDataProvider() {
|
||||
@Override
|
||||
public List<StartupRecoveryCandidate> loadActiveCandidates() {
|
||||
return Collections.singletonList(candidate(
|
||||
"ORDER-1", "VEH-1", "CMD-1", "NODE:A"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> findUnexplainedGlobalResourceKeys() {
|
||||
return Collections.emptySet();
|
||||
}
|
||||
};
|
||||
DeviceSnapshotUseCase reconciler = new DeviceSnapshotUseCase() {
|
||||
@Override
|
||||
public ReconcileResult reconcile(DeviceSnapshot snapshot) {
|
||||
return new ReconcileResult(ReconcileStatus.RECOVERY_REQUIRED,
|
||||
snapshot.getCommandKey(), Collections.<String>emptyList(),
|
||||
"TASK_NUMBER_MISMATCH");
|
||||
}
|
||||
};
|
||||
|
||||
StartupRecoveryResult result = new StartupRecoveryService(
|
||||
new FakeLease(), data, new FakeSnapshots(), reconciler,
|
||||
recovery, cases).recover();
|
||||
|
||||
assertEquals(ControlStatus.PARTIALLY_ENABLED, result.getControlStatus());
|
||||
assertEquals(Collections.singleton("VEH-1"),
|
||||
result.getIsolatedDeviceIds());
|
||||
assertEquals(Collections.singleton("NODE:A"),
|
||||
result.getIsolatedResourceKeys());
|
||||
}
|
||||
|
||||
@Test
|
||||
void startupEnablesOnlyAfterReadingAndReconcilingEveryActiveDevice() {
|
||||
FakeRecoveryCases cases = new FakeRecoveryCases();
|
||||
final Set<String> readDevices = new LinkedHashSet<String>();
|
||||
final Set<String> reconciledDevices = new LinkedHashSet<String>();
|
||||
StartupRecoveryDataProvider data = new StartupRecoveryDataProvider() {
|
||||
@Override
|
||||
public List<StartupRecoveryCandidate> loadActiveCandidates() {
|
||||
return Arrays.asList(
|
||||
candidate("ORDER-1", "VEH-1", "CMD-1", "NODE:A"),
|
||||
candidate("ORDER-2", "VEH-2", "CMD-2", "NODE:B"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> findUnexplainedGlobalResourceKeys() {
|
||||
return Collections.emptySet();
|
||||
}
|
||||
};
|
||||
DeviceSnapshotGateway snapshots = new DeviceSnapshotGateway() {
|
||||
@Override
|
||||
public DeviceSnapshot readSnapshot(String deviceId) {
|
||||
readDevices.add(deviceId);
|
||||
return snapshot(deviceId);
|
||||
}
|
||||
};
|
||||
DeviceSnapshotUseCase reconciler = new DeviceSnapshotUseCase() {
|
||||
@Override
|
||||
public ReconcileResult reconcile(DeviceSnapshot snapshot) {
|
||||
reconciledDevices.add(snapshot.getDeviceId());
|
||||
return new ReconcileResult(ReconcileStatus.CONSISTENT,
|
||||
snapshot.getCommandKey(), Collections.<String>emptyList(), null);
|
||||
}
|
||||
};
|
||||
|
||||
StartupRecoveryResult result = new StartupRecoveryService(
|
||||
new FakeLease(), data, snapshots, reconciler,
|
||||
recoveryService(cases), cases).recover();
|
||||
|
||||
assertEquals(ControlStatus.ENABLED, result.getControlStatus());
|
||||
assertEquals(99L, result.getFencingToken());
|
||||
assertEquals(new LinkedHashSet<String>(Arrays.asList("VEH-1", "VEH-2")),
|
||||
readDevices);
|
||||
assertEquals(readDevices, reconciledDevices);
|
||||
assertEquals(new LinkedHashSet<String>(Arrays.asList("ORDER-1", "ORDER-2")),
|
||||
result.getResumedOrderIds());
|
||||
}
|
||||
|
||||
@Test
|
||||
void staleStartupSnapshotIsIsolatedInsteadOfResumingTheOrder() {
|
||||
FakeRecoveryCases cases = new FakeRecoveryCases();
|
||||
StartupRecoveryDataProvider data = dataFor(Collections.singletonList(
|
||||
candidate("ORDER-1", "VEH-1", "CMD-1", "NODE:A")));
|
||||
DeviceSnapshotUseCase reconciler = new DeviceSnapshotUseCase() {
|
||||
@Override
|
||||
public ReconcileResult reconcile(DeviceSnapshot snapshot) {
|
||||
return new ReconcileResult(ReconcileStatus.IGNORED_STALE,
|
||||
snapshot.getCommandKey(), Collections.<String>emptyList(),
|
||||
null);
|
||||
}
|
||||
};
|
||||
|
||||
StartupRecoveryResult result = new StartupRecoveryService(
|
||||
new FakeLease(), data, new FakeSnapshots(), reconciler,
|
||||
recoveryService(cases), cases).recover();
|
||||
|
||||
assertEquals(ControlStatus.PARTIALLY_ENABLED, result.getControlStatus());
|
||||
assertEquals(Collections.emptySet(), result.getResumedOrderIds());
|
||||
assertEquals(Collections.singleton("VEH-1"),
|
||||
result.getIsolatedDeviceIds());
|
||||
}
|
||||
|
||||
@Test
|
||||
void snapshotForAnotherDeviceCannotResumeTheCandidateOrder() {
|
||||
FakeRecoveryCases cases = new FakeRecoveryCases();
|
||||
final int[] reconcileCalls = {0};
|
||||
DeviceSnapshotGateway snapshots = new DeviceSnapshotGateway() {
|
||||
@Override
|
||||
public DeviceSnapshot readSnapshot(String deviceId) {
|
||||
return snapshot("VEH-2");
|
||||
}
|
||||
};
|
||||
DeviceSnapshotUseCase reconciler = new DeviceSnapshotUseCase() {
|
||||
@Override
|
||||
public ReconcileResult reconcile(DeviceSnapshot snapshot) {
|
||||
reconcileCalls[0]++;
|
||||
return new ReconcileResult(ReconcileStatus.CONSISTENT,
|
||||
snapshot.getCommandKey(), Collections.<String>emptyList(), null);
|
||||
}
|
||||
};
|
||||
|
||||
StartupRecoveryResult result = new StartupRecoveryService(
|
||||
new FakeLease(), dataFor(Collections.singletonList(candidate(
|
||||
"ORDER-1", "VEH-1", "CMD-1", "NODE:A"))),
|
||||
snapshots, reconciler, recoveryService(cases), cases).recover();
|
||||
|
||||
assertEquals(0, reconcileCalls[0]);
|
||||
assertEquals(Collections.emptySet(), result.getResumedOrderIds());
|
||||
assertEquals(Collections.singleton("VEH-1"),
|
||||
result.getIsolatedDeviceIds());
|
||||
}
|
||||
|
||||
@Test
|
||||
void automaticCaseWithExpectedFactsCanBeOperatorConfirmed() {
|
||||
FakeRecoveryCases cases = new FakeRecoveryCases();
|
||||
RecoveryApplicationService recovery = recoveryService(cases);
|
||||
StartupRecoveryCandidate candidate = new StartupRecoveryCandidate(
|
||||
"ORDER-1", "VEH-1", "CMD-1",
|
||||
Collections.singletonList("NODE:A"),
|
||||
"TASK-1", "SITE-B", "LOADED");
|
||||
DeviceSnapshotUseCase reconciler = new DeviceSnapshotUseCase() {
|
||||
@Override
|
||||
public ReconcileResult reconcile(DeviceSnapshot snapshot) {
|
||||
return new ReconcileResult(ReconcileStatus.RECOVERY_REQUIRED,
|
||||
snapshot.getCommandKey(), Collections.<String>emptyList(),
|
||||
"SITE_MISMATCH");
|
||||
}
|
||||
};
|
||||
new StartupRecoveryService(new FakeLease(),
|
||||
dataFor(Collections.singletonList(candidate)),
|
||||
new FakeSnapshots(), reconciler, recovery, cases).recover();
|
||||
RecoveryCase recoveryCase = cases.findOpen("VEH-1", "CMD-1");
|
||||
|
||||
recovery.confirmFacts(recoveryCase.getCaseId(), "OP-1",
|
||||
new RecoveryEvidence("SITE-B", "LOADED", "TASK-1", "AUTO",
|
||||
Instant.EPOCH.plusSeconds(1), "PLC_AND_VISUAL",
|
||||
"matched expected startup facts"));
|
||||
|
||||
assertEquals("CONFIRMED", recoveryCase.getState().name());
|
||||
}
|
||||
|
||||
@Test
|
||||
void operatorCannotResolveCaseWithoutMatchingEvidence() {
|
||||
FakeRecoveryCases cases = new FakeRecoveryCases();
|
||||
RecoveryCase recoveryCase = RecoveryCase.open(
|
||||
"RC-1", "VEH-1", "CMD-1", "TASK_NUMBER_MISMATCH",
|
||||
Collections.singletonList("SNAP-1"), "TASK-1", "SITE-B",
|
||||
"LOADED", Instant.EPOCH);
|
||||
cases.save(recoveryCase);
|
||||
RecoveryApplicationService service = recoveryService(cases);
|
||||
|
||||
assertThrows(DomainRuleViolation.class, () -> service.confirmFacts(
|
||||
"RC-1", "OP-1", new RecoveryEvidence("SITE-A", "EMPTY",
|
||||
"WRONG-TASK", "AUTO", Instant.EPOCH.plusSeconds(1),
|
||||
"PLC_SNAPSHOT", "checked on site")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void operatorCannotConfirmCaseWhoseExpectedFactsWereNotPersisted() {
|
||||
FakeRecoveryCases cases = new FakeRecoveryCases();
|
||||
RecoveryApplicationService service = recoveryService(cases);
|
||||
service.open("VEH-1", "CMD-1", "AMBIGUOUS_SEND",
|
||||
Collections.singletonList("SEND-1"));
|
||||
RecoveryCase recoveryCase = cases.findOpen("VEH-1", "CMD-1");
|
||||
|
||||
assertThrows(DomainRuleViolation.class, () -> service.confirmFacts(
|
||||
recoveryCase.getCaseId(), "OP-1", new RecoveryEvidence(
|
||||
"SITE-A", "EMPTY", "TASK-1", "AUTO",
|
||||
Instant.EPOCH.plusSeconds(1), "PLC_SNAPSHOT",
|
||||
"facts were guessed")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void matchingOperatorEvidenceClosesTheCaseWithAnAuditTrail() {
|
||||
FakeRecoveryCases cases = new FakeRecoveryCases();
|
||||
cases.save(RecoveryCase.open("RC-1", "VEH-1", "CMD-1",
|
||||
"TASK_NUMBER_MISMATCH", Collections.singletonList("SNAP-1"),
|
||||
"TASK-1", "SITE-B", "LOADED", Instant.EPOCH));
|
||||
RecoveryApplicationService service = recoveryService(cases);
|
||||
|
||||
service.confirmFacts("RC-1", "OP-1", new RecoveryEvidence(
|
||||
"SITE-B", "LOADED", "TASK-1", "AUTO",
|
||||
Instant.EPOCH.plusSeconds(2), "PLC_AND_VISUAL",
|
||||
"task and load matched"));
|
||||
|
||||
assertEquals("OP-1", cases.findById("RC-1").getResolvedBy());
|
||||
assertEquals("CONFIRMED", cases.findById("RC-1").getState().name());
|
||||
}
|
||||
|
||||
@Test
|
||||
void operatorAbortClosesTheCaseWithReasonAndOperatorAudit() {
|
||||
FakeRecoveryCases cases = new FakeRecoveryCases();
|
||||
cases.save(RecoveryCase.open("RC-1", "VEH-1", "CMD-1",
|
||||
"TASK_NUMBER_MISMATCH", Collections.singletonList("SNAP-1"),
|
||||
"TASK-1", "SITE-B", "LOADED", Instant.EPOCH));
|
||||
RecoveryApplicationService service = recoveryService(cases);
|
||||
|
||||
service.abort("RC-1", "OP-2", "physical task was cancelled");
|
||||
|
||||
RecoveryCase aborted = cases.findById("RC-1");
|
||||
assertEquals("ABORTED", aborted.getState().name());
|
||||
assertEquals("OP-2", aborted.getResolvedBy());
|
||||
assertEquals("physical task was cancelled", aborted.getResolutionReason());
|
||||
assertEquals(Instant.EPOCH, aborted.getResolvedAt());
|
||||
}
|
||||
|
||||
private static RecoveryApplicationService recoveryService(
|
||||
RecoveryCaseRepository repository) {
|
||||
return new RecoveryApplicationService(repository, new IdGenerator() {
|
||||
private int sequence;
|
||||
|
||||
@Override
|
||||
public String nextId() {
|
||||
return "RC-" + (++sequence + 1);
|
||||
}
|
||||
}, new java.util.function.Supplier<Instant>() {
|
||||
@Override
|
||||
public Instant get() {
|
||||
return Instant.EPOCH;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private static StartupRecoveryCandidate candidate(String orderId,
|
||||
String deviceId,
|
||||
String commandKey,
|
||||
String resourceKey) {
|
||||
return new StartupRecoveryCandidate(orderId, deviceId, commandKey,
|
||||
Collections.singletonList(resourceKey));
|
||||
}
|
||||
|
||||
private static StartupRecoveryDataProvider dataFor(
|
||||
final List<StartupRecoveryCandidate> candidates) {
|
||||
return new StartupRecoveryDataProvider() {
|
||||
@Override
|
||||
public List<StartupRecoveryCandidate> loadActiveCandidates() {
|
||||
return candidates;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> findUnexplainedGlobalResourceKeys() {
|
||||
return Collections.emptySet();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static DeviceSnapshot snapshot(String deviceId) {
|
||||
String suffix = deviceId.substring(deviceId.length() - 1);
|
||||
return new DeviceSnapshot("SNAP-" + deviceId, deviceId,
|
||||
"CMD-" + suffix, "TASK-1", 1L, Instant.EPOCH, true,
|
||||
"BOOT-1", 1L, 1L, "SITE-A",
|
||||
Collections.singletonList("SITE-A"), false);
|
||||
}
|
||||
|
||||
private static final class FakeLease implements ControlLeaseRepository {
|
||||
@Override
|
||||
public long acquireNewFencingToken() {
|
||||
return 99L;
|
||||
}
|
||||
}
|
||||
|
||||
private static final class FakeSnapshots implements DeviceSnapshotGateway {
|
||||
@Override
|
||||
public DeviceSnapshot readSnapshot(String deviceId) {
|
||||
return snapshot(deviceId);
|
||||
}
|
||||
}
|
||||
|
||||
private static final class FakeRecoveryCases
|
||||
implements RecoveryCaseRepository {
|
||||
private final Map<String, RecoveryCase> values =
|
||||
new LinkedHashMap<String, RecoveryCase>();
|
||||
|
||||
@Override
|
||||
public RecoveryCase findById(String caseId) {
|
||||
return values.get(caseId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public RecoveryCase findOpen(String deviceId, String commandKey) {
|
||||
for (RecoveryCase value : values.values()) {
|
||||
if (value.isOpen() && value.getDeviceId().equals(deviceId)
|
||||
&& value.getCommandKey().equals(commandKey)) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<RecoveryCase> findOpenCases() {
|
||||
List<RecoveryCase> result = new ArrayList<RecoveryCase>();
|
||||
for (RecoveryCase value : values.values()) {
|
||||
if (value.isOpen()) {
|
||||
result.add(value);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void save(RecoveryCase recoveryCase) {
|
||||
values.put(recoveryCase.getCaseId(), recoveryCase);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized RecoveryCase saveIfAbsentOpen(
|
||||
RecoveryCase recoveryCase) {
|
||||
RecoveryCase existing = findOpen(recoveryCase.getDeviceId(),
|
||||
recoveryCase.getCommandKey());
|
||||
if (existing != null) {
|
||||
return existing;
|
||||
}
|
||||
save(recoveryCase);
|
||||
return recoveryCase;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user