feat(wcs): recover execution state after restart

2026-08-11 11:09:51 +08:00
parent ef3cac9e3e
commit 9782800b0b
16 changed files with 1006 additions and 1 deletions
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/application/device/FeedbackReconciler.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/application/device/FeedbackReconciler.java
@@ -8,6 +8,7 @@ import com.ruoyi.wcs.domain.execution.DeviceEvent;
 import com.ruoyi.wcs.domain.execution.DeviceSnapshot;
 import com.ruoyi.wcs.domain.shared.DomainRuleViolation;
 import com.ruoyi.wcs.port.in.DeviceEventUseCase;
+import com.ruoyi.wcs.port.in.DeviceSnapshotUseCase;
 import com.ruoyi.wcs.port.out.DeviceCommandRepository;
 import com.ruoyi.wcs.port.out.DeviceEventRepository;
 import com.ruoyi.wcs.port.out.DeviceExecutionRepository;
@@ -19,7 +20,8 @@ import java.time.Instant;
 import java.util.Collections;
 import java.util.List;

-public final class FeedbackReconciler implements DeviceEventUseCase {
+public final class FeedbackReconciler
+        implements DeviceEventUseCase, DeviceSnapshotUseCase {
    private final DeviceCommandRepository commandRepository;
    private final DeviceExecutionRepository executionRepository;
    private final DeviceEventRepository eventRepository;
@@ -114,6 +116,7 @@ public final class FeedbackReconciler implements DeviceEventUseCase {
        return EventAcceptanceResult.APPLIED;
    }

+    @Override
    @Transactional
    public ReconcileResult reconcile(DeviceSnapshot snapshot) {
        CommittedDeviceExecution execution = executionRepository
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/application/recovery/RecoveryApplicationService.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/application/recovery/RecoveryApplicationService.java
@@ -0,0 +1,79 @@
+package com.ruoyi.wcs.application.recovery;
+
+import com.ruoyi.wcs.domain.recovery.RecoveryCase;
+import com.ruoyi.wcs.domain.recovery.RecoveryEvidence;
+import com.ruoyi.wcs.domain.shared.DomainRuleViolation;
+import com.ruoyi.wcs.domain.shared.IdGenerator;
+import com.ruoyi.wcs.port.in.RecoveryUseCase;
+import com.ruoyi.wcs.port.out.RecoveryCaseOpener;
+import com.ruoyi.wcs.port.out.RecoveryCaseRepository;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+public final class RecoveryApplicationService
+        implements RecoveryUseCase, RecoveryCaseOpener {
+    private final RecoveryCaseRepository repository;
+    private final IdGenerator idGenerator;
+    private final Supplier<Instant> clock;
+
+    public RecoveryApplicationService(RecoveryCaseRepository repository,
+                                      IdGenerator idGenerator,
+                                      Supplier<Instant> clock) {
+        this.repository = Objects.requireNonNull(repository, "repository");
+        this.idGenerator = Objects.requireNonNull(idGenerator, "idGenerator");
+        this.clock = Objects.requireNonNull(clock, "clock");
+    }
+
+    @Override
+    @Transactional
+    public void open(String deviceId, String commandKey, String reason,
+                     List<String> evidenceIds) {
+        open(deviceId, commandKey, reason, evidenceIds, null, null, null);
+    }
+
+    @Override
+    @Transactional
+    public void open(String deviceId, String commandKey, String reason,
+                     List<String> evidenceIds, String expectedTaskNumber,
+                     String expectedSite, String expectedLoadState) {
+        if (repository.findOpen(deviceId, commandKey) != null) {
+            return;
+        }
+        repository.saveIfAbsentOpen(RecoveryCase.open(idGenerator.nextId(),
+                deviceId, commandKey, reason, evidenceIds,
+                expectedTaskNumber, expectedSite, expectedLoadState, now()));
+    }
+
+    @Override
+    @Transactional
+    public void confirmFacts(String caseId, String operatorId,
+                             RecoveryEvidence evidence) {
+        RecoveryCase recoveryCase = requireCase(caseId);
+        recoveryCase.confirm(operatorId, evidence);
+        repository.save(recoveryCase);
+    }
+
+    @Override
+    @Transactional
+    public void abort(String caseId, String operatorId, String reason) {
+        RecoveryCase recoveryCase = requireCase(caseId);
+        recoveryCase.abort(operatorId, reason, now());
+        repository.save(recoveryCase);
+    }
+
+    private RecoveryCase requireCase(String caseId) {
+        RecoveryCase recoveryCase = repository.findById(caseId);
+        if (recoveryCase == null) {
+            throw new DomainRuleViolation("recovery case not found: " + caseId);
+        }
+        return recoveryCase;
+    }
+
+    private Instant now() {
+        return Objects.requireNonNull(clock.get(), "clock value");
+    }
+}
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/application/recovery/StartupRecoveryService.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/application/recovery/StartupRecoveryService.java
@@ -0,0 +1,148 @@
+package com.ruoyi.wcs.application.recovery;
+
+import com.ruoyi.wcs.application.device.ReconcileResult;
+import com.ruoyi.wcs.application.device.ReconcileStatus;
+import com.ruoyi.wcs.domain.execution.DeviceSnapshot;
+import com.ruoyi.wcs.domain.recovery.ControlStatus;
+import com.ruoyi.wcs.domain.recovery.RecoveryCase;
+import com.ruoyi.wcs.domain.recovery.StartupRecoveryCandidate;
+import com.ruoyi.wcs.domain.recovery.StartupRecoveryResult;
+import com.ruoyi.wcs.port.in.DeviceSnapshotUseCase;
+import com.ruoyi.wcs.port.out.ControlLeaseRepository;
+import com.ruoyi.wcs.port.out.DeviceSnapshotGateway;
+import com.ruoyi.wcs.port.out.RecoveryCaseOpener;
+import com.ruoyi.wcs.port.out.RecoveryCaseRepository;
+import com.ruoyi.wcs.port.out.StartupRecoveryDataProvider;
+
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+public final class StartupRecoveryService {
+    private final ControlLeaseRepository leaseRepository;
+    private final StartupRecoveryDataProvider dataProvider;
+    private final DeviceSnapshotGateway snapshotGateway;
+    private final DeviceSnapshotUseCase snapshotUseCase;
+    private final RecoveryCaseOpener recoveryCaseOpener;
+    private final RecoveryCaseRepository recoveryCaseRepository;
+
+    public StartupRecoveryService(ControlLeaseRepository leaseRepository,
+                                  StartupRecoveryDataProvider dataProvider,
+                                  DeviceSnapshotGateway snapshotGateway,
+                                  DeviceSnapshotUseCase snapshotUseCase,
+                                  RecoveryCaseOpener recoveryCaseOpener,
+                                  RecoveryCaseRepository recoveryCaseRepository) {
+        this.leaseRepository = Objects.requireNonNull(
+                leaseRepository, "leaseRepository");
+        this.dataProvider = Objects.requireNonNull(dataProvider, "dataProvider");
+        this.snapshotGateway = Objects.requireNonNull(
+                snapshotGateway, "snapshotGateway");
+        this.snapshotUseCase = Objects.requireNonNull(
+                snapshotUseCase, "snapshotUseCase");
+        this.recoveryCaseOpener = Objects.requireNonNull(
+                recoveryCaseOpener, "recoveryCaseOpener");
+        this.recoveryCaseRepository = Objects.requireNonNull(
+                recoveryCaseRepository, "recoveryCaseRepository");
+    }
+
+    public StartupRecoveryResult recover() {
+        long fencingToken = leaseRepository.acquireNewFencingToken();
+        List<StartupRecoveryCandidate> candidates = Objects.requireNonNull(
+                dataProvider.loadActiveCandidates(), "active candidates");
+        Set<String> isolatedDevices = new LinkedHashSet<String>();
+        Set<String> isolatedResources = new LinkedHashSet<String>();
+        Set<String> resumedOrders = new LinkedHashSet<String>();
+
+        for (StartupRecoveryCandidate candidate : candidates) {
+            recoverCandidate(candidate, fencingToken, isolatedDevices,
+                    isolatedResources, resumedOrders);
+        }
+
+        Set<String> unexplainedGlobalResources = Objects.requireNonNull(
+                dataProvider.findUnexplainedGlobalResourceKeys(),
+                "unexplained global resources");
+        isolatedResources.addAll(unexplainedGlobalResources);
+
+        Set<String> unresolvedCases = new LinkedHashSet<String>();
+        List<RecoveryCase> openCases = Objects.requireNonNull(
+                recoveryCaseRepository.findOpenCases(), "open recovery cases");
+        for (RecoveryCase recoveryCase : openCases) {
+            unresolvedCases.add(recoveryCase.getCaseId());
+        }
+
+        ControlStatus status = controlStatus(unexplainedGlobalResources,
+                isolatedDevices);
+        return new StartupRecoveryResult(status, fencingToken, isolatedDevices,
+                isolatedResources, resumedOrders, unresolvedCases);
+    }
+
+    private void recoverCandidate(StartupRecoveryCandidate candidate,
+                                  long fencingToken,
+                                  Set<String> isolatedDevices,
+                                  Set<String> isolatedResources,
+                                  Set<String> resumedOrders) {
+        DeviceSnapshot snapshot;
+        ReconcileResult result;
+        try {
+            snapshot = Objects.requireNonNull(
+                    snapshotGateway.readSnapshot(candidate.getDeviceId()),
+                    "device snapshot");
+            if (!candidate.getDeviceId().equals(snapshot.getDeviceId())) {
+                isolate(candidate, "SNAPSHOT_DEVICE_MISMATCH",
+                        snapshot.getSnapshotId(), isolatedDevices,
+                        isolatedResources);
+                return;
+            }
+            result = Objects.requireNonNull(snapshotUseCase.reconcile(snapshot),
+                    "reconcile result");
+        } catch (RuntimeException failure) {
+            isolate(candidate, "STARTUP_RECONCILIATION_FAILED",
+                    "STARTUP-" + fencingToken + "-" + candidate.getDeviceId(),
+                    isolatedDevices, isolatedResources);
+            return;
+        }
+
+        if (result.getStatus() == ReconcileStatus.RECOVERY_REQUIRED) {
+            String reason = textOrDefault(result.getRecoveryReason(),
+                    "STARTUP_RECONCILIATION_FAILED");
+            isolate(candidate, reason, snapshot.getSnapshotId(),
+                    isolatedDevices, isolatedResources);
+            return;
+        }
+        if (result.getStatus() == ReconcileStatus.CONSISTENT
+                || result.getStatus() == ReconcileStatus.ADVANCED) {
+            resumedOrders.add(candidate.getOrderId());
+            return;
+        }
+        isolate(candidate, "STALE_STARTUP_SNAPSHOT", snapshot.getSnapshotId(),
+                isolatedDevices, isolatedResources);
+    }
+
+    private void isolate(StartupRecoveryCandidate candidate, String reason,
+                         String evidenceId, Set<String> isolatedDevices,
+                         Set<String> isolatedResources) {
+        isolatedDevices.add(candidate.getDeviceId());
+        isolatedResources.addAll(candidate.getResourceKeys());
+        recoveryCaseOpener.open(candidate.getDeviceId(),
+                candidate.getCommandKey(), reason,
+                Collections.singletonList(evidenceId),
+                candidate.getExpectedTaskNumber(), candidate.getExpectedSite(),
+                candidate.getExpectedLoadState());
+    }
+
+    private static ControlStatus controlStatus(Set<String> globalResources,
+                                               Set<String> isolatedDevices) {
+        if (!globalResources.isEmpty()) {
+            return ControlStatus.PAUSED;
+        }
+        return isolatedDevices.isEmpty()
+                ? ControlStatus.ENABLED : ControlStatus.PARTIALLY_ENABLED;
+    }
+
+    private static String textOrDefault(String value, String defaultValue) {
+        return value == null || value.trim().isEmpty()
+                ? defaultValue : value.trim();
+    }
+}
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/domain/recovery/ControlStatus.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/domain/recovery/ControlStatus.java
@@ -0,0 +1,7 @@
+package com.ruoyi.wcs.domain.recovery;
+
+public enum ControlStatus {
+    PAUSED,
+    PARTIALLY_ENABLED,
+    ENABLED
+}
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/domain/recovery/RecoveryCase.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/domain/recovery/RecoveryCase.java
@@ -0,0 +1,127 @@
+package com.ruoyi.wcs.domain.recovery;
+
+import com.ruoyi.wcs.domain.shared.DomainRuleViolation;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public final class RecoveryCase {
+    private final String caseId;
+    private final String deviceId;
+    private final String commandKey;
+    private final String reason;
+    private final List<String> evidenceIds;
+    private final String expectedTaskNumber;
+    private final String expectedSite;
+    private final String expectedLoadState;
+    private final Instant createdAt;
+    private RecoveryCaseState state;
+    private String resolvedBy;
+    private Instant resolvedAt;
+    private RecoveryEvidence resolutionEvidence;
+    private String resolutionReason;
+
+    private RecoveryCase(String caseId, String deviceId, String commandKey,
+                         String reason, List<String> evidenceIds,
+                         String expectedTaskNumber, String expectedSite,
+                         String expectedLoadState, Instant createdAt) {
+        this.caseId = requireText(caseId, "caseId");
+        this.deviceId = requireText(deviceId, "deviceId");
+        this.commandKey = requireText(commandKey, "commandKey");
+        this.reason = requireText(reason, "reason");
+        this.evidenceIds = immutable(evidenceIds);
+        this.expectedTaskNumber = optional(expectedTaskNumber);
+        this.expectedSite = optional(expectedSite);
+        this.expectedLoadState = optional(expectedLoadState);
+        this.createdAt = java.util.Objects.requireNonNull(createdAt, "createdAt");
+        this.state = RecoveryCaseState.OPEN;
+    }
+
+    public static RecoveryCase open(String caseId, String deviceId,
+                                    String commandKey, String reason,
+                                    List<String> evidenceIds,
+                                    String expectedTaskNumber,
+                                    String expectedSite,
+                                    String expectedLoadState,
+                                    Instant createdAt) {
+        return new RecoveryCase(caseId, deviceId, commandKey, reason,
+                evidenceIds, expectedTaskNumber, expectedSite,
+                expectedLoadState, createdAt);
+    }
+
+    public void confirm(String operatorId, RecoveryEvidence evidence) {
+        requireOpen();
+        String operator = requireText(operatorId, "operatorId");
+        RecoveryEvidence facts = java.util.Objects.requireNonNull(evidence,
+                "evidence");
+        if (!matches(expectedTaskNumber, facts.getTaskNumber())
+                || !matches(expectedSite, facts.getObservedSite())
+                || !matches(expectedLoadState, facts.getLoadState())) {
+            throw new DomainRuleViolation(
+                    "operator recovery evidence does not match expected facts");
+        }
+        state = RecoveryCaseState.CONFIRMED;
+        resolvedBy = operator;
+        resolvedAt = facts.getEvidenceTime();
+        resolutionEvidence = facts;
+        resolutionReason = facts.getOperatorReason();
+    }
+
+    public void abort(String operatorId, String abortReason, Instant at) {
+        requireOpen();
+        resolvedBy = requireText(operatorId, "operatorId");
+        resolutionReason = requireText(abortReason, "reason");
+        resolvedAt = java.util.Objects.requireNonNull(at, "at");
+        state = RecoveryCaseState.ABORTED;
+    }
+
+    private void requireOpen() {
+        if (state != RecoveryCaseState.OPEN) {
+            throw new DomainRuleViolation("recovery case is already resolved");
+        }
+    }
+
+    public String getCaseId() { return caseId; }
+    public String getDeviceId() { return deviceId; }
+    public String getCommandKey() { return commandKey; }
+    public String getReason() { return reason; }
+    public List<String> getEvidenceIds() { return evidenceIds; }
+    public String getExpectedTaskNumber() { return expectedTaskNumber; }
+    public String getExpectedSite() { return expectedSite; }
+    public String getExpectedLoadState() { return expectedLoadState; }
+    public RecoveryCaseState getState() { return state; }
+    public String getResolvedBy() { return resolvedBy; }
+    public Instant getResolvedAt() { return resolvedAt; }
+    public RecoveryEvidence getResolutionEvidence() { return resolutionEvidence; }
+    public String getResolutionReason() { return resolutionReason; }
+    public Instant getCreatedAt() { return createdAt; }
+    public boolean isOpen() { return state == RecoveryCaseState.OPEN; }
+
+    private static boolean matches(String expected, String actual) {
+        return expected != null && expected.equals(actual);
+    }
+
+    private static List<String> immutable(List<String> values) {
+        if (values == null || values.isEmpty()) {
+            throw new IllegalArgumentException("evidenceIds must not be empty");
+        }
+        List<String> copy = new ArrayList<String>();
+        for (String value : values) {
+            copy.add(requireText(value, "evidenceId"));
+        }
+        return Collections.unmodifiableList(copy);
+    }
+
+    private static String optional(String value) {
+        return value == null || value.trim().isEmpty() ? null : value.trim();
+    }
+
+    private static String requireText(String value, String field) {
+        if (value == null || value.trim().isEmpty()) {
+            throw new IllegalArgumentException(field + " must not be blank");
+        }
+        return value.trim();
+    }
+}
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/domain/recovery/RecoveryCaseState.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/domain/recovery/RecoveryCaseState.java
@@ -0,0 +1,7 @@
+package com.ruoyi.wcs.domain.recovery;
+
+public enum RecoveryCaseState {
+    OPEN,
+    CONFIRMED,
+    ABORTED
+}
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/domain/recovery/RecoveryEvidence.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/domain/recovery/RecoveryEvidence.java
@@ -0,0 +1,42 @@
+package com.ruoyi.wcs.domain.recovery;
+
+import java.time.Instant;
+import java.util.Objects;
+
+public final class RecoveryEvidence {
+    private final String observedSite;
+    private final String loadState;
+    private final String taskNumber;
+    private final String deviceMode;
+    private final Instant evidenceTime;
+    private final String source;
+    private final String operatorReason;
+
+    public RecoveryEvidence(String observedSite, String loadState,
+                            String taskNumber, String deviceMode,
+                            Instant evidenceTime, String source,
+                            String operatorReason) {
+        this.observedSite = requireText(observedSite, "observedSite");
+        this.loadState = requireText(loadState, "loadState");
+        this.taskNumber = requireText(taskNumber, "taskNumber");
+        this.deviceMode = requireText(deviceMode, "deviceMode");
+        this.evidenceTime = Objects.requireNonNull(evidenceTime, "evidenceTime");
+        this.source = requireText(source, "source");
+        this.operatorReason = requireText(operatorReason, "operatorReason");
+    }
+
+    public String getObservedSite() { return observedSite; }
+    public String getLoadState() { return loadState; }
+    public String getTaskNumber() { return taskNumber; }
+    public String getDeviceMode() { return deviceMode; }
+    public Instant getEvidenceTime() { return evidenceTime; }
+    public String getSource() { return source; }
+    public String getOperatorReason() { return operatorReason; }
+
+    private static String requireText(String value, String field) {
+        if (value == null || value.trim().isEmpty()) {
+            throw new IllegalArgumentException(field + " must not be blank");
+        }
+        return value.trim();
+    }
+}
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/domain/recovery/StartupRecoveryCandidate.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/domain/recovery/StartupRecoveryCandidate.java
@@ -0,0 +1,69 @@
+package com.ruoyi.wcs.domain.recovery;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public final class StartupRecoveryCandidate {
+    private final String orderId;
+    private final String deviceId;
+    private final String commandKey;
+    private final List<String> resourceKeys;
+    private final String expectedTaskNumber;
+    private final String expectedSite;
+    private final String expectedLoadState;
+
+    public StartupRecoveryCandidate(String orderId, String deviceId,
+                                    String commandKey, List<String> resourceKeys) {
+        this(orderId, deviceId, commandKey, resourceKeys, null, null, null);
+    }
+
+    public StartupRecoveryCandidate(String orderId, String deviceId,
+                                    String commandKey, List<String> resourceKeys,
+                                    String expectedTaskNumber,
+                                    String expectedSite,
+                                    String expectedLoadState) {
+        this.orderId = requireText(orderId, "orderId");
+        this.deviceId = requireText(deviceId, "deviceId");
+        this.commandKey = requireText(commandKey, "commandKey");
+        if (resourceKeys == null) {
+            throw new IllegalArgumentException("resourceKeys must not be null");
+        }
+        this.resourceKeys = Collections.unmodifiableList(
+                new ArrayList<String>(resourceKeys));
+        boolean hasAnyExpectedFact = expectedTaskNumber != null
+                || expectedSite != null || expectedLoadState != null;
+        boolean hasAllExpectedFacts = hasText(expectedTaskNumber)
+                && hasText(expectedSite) && hasText(expectedLoadState);
+        if (hasAnyExpectedFact && !hasAllExpectedFacts) {
+            throw new IllegalArgumentException(
+                    "expected recovery facts must be complete");
+        }
+        this.expectedTaskNumber = optional(expectedTaskNumber);
+        this.expectedSite = optional(expectedSite);
+        this.expectedLoadState = optional(expectedLoadState);
+    }
+
+    public String getOrderId() { return orderId; }
+    public String getDeviceId() { return deviceId; }
+    public String getCommandKey() { return commandKey; }
+    public List<String> getResourceKeys() { return resourceKeys; }
+    public String getExpectedTaskNumber() { return expectedTaskNumber; }
+    public String getExpectedSite() { return expectedSite; }
+    public String getExpectedLoadState() { return expectedLoadState; }
+
+    private static boolean hasText(String value) {
+        return value != null && !value.trim().isEmpty();
+    }
+
+    private static String optional(String value) {
+        return value == null ? null : value.trim();
+    }
+
+    private static String requireText(String value, String field) {
+        if (value == null || value.trim().isEmpty()) {
+            throw new IllegalArgumentException(field + " must not be blank");
+        }
+        return value.trim();
+    }
+}
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/domain/recovery/StartupRecoveryResult.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/domain/recovery/StartupRecoveryResult.java
@@ -0,0 +1,45 @@
+package com.ruoyi.wcs.domain.recovery;
+
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+public final class StartupRecoveryResult {
+    private final ControlStatus controlStatus;
+    private final long fencingToken;
+    private final Set<String> isolatedDeviceIds;
+    private final Set<String> isolatedResourceKeys;
+    private final Set<String> resumedOrderIds;
+    private final Set<String> unresolvedRecoveryCaseIds;
+
+    public StartupRecoveryResult(ControlStatus controlStatus, long fencingToken,
+                                 Set<String> isolatedDeviceIds,
+                                 Set<String> isolatedResourceKeys,
+                                 Set<String> resumedOrderIds,
+                                 Set<String> unresolvedRecoveryCaseIds) {
+        this.controlStatus = java.util.Objects.requireNonNull(controlStatus,
+                "controlStatus");
+        if (fencingToken <= 0L) {
+            throw new IllegalArgumentException("fencingToken must be positive");
+        }
+        this.fencingToken = fencingToken;
+        this.isolatedDeviceIds = immutable(isolatedDeviceIds);
+        this.isolatedResourceKeys = immutable(isolatedResourceKeys);
+        this.resumedOrderIds = immutable(resumedOrderIds);
+        this.unresolvedRecoveryCaseIds = immutable(unresolvedRecoveryCaseIds);
+    }
+
+    public ControlStatus getControlStatus() { return controlStatus; }
+    public long getFencingToken() { return fencingToken; }
+    public Set<String> getIsolatedDeviceIds() { return isolatedDeviceIds; }
+    public Set<String> getIsolatedResourceKeys() { return isolatedResourceKeys; }
+    public Set<String> getResumedOrderIds() { return resumedOrderIds; }
+    public Set<String> getUnresolvedRecoveryCaseIds() {
+        return unresolvedRecoveryCaseIds;
+    }
+
+    private static Set<String> immutable(Set<String> values) {
+        return Collections.unmodifiableSet(new LinkedHashSet<String>(
+                java.util.Objects.requireNonNull(values, "values")));
+    }
+}
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/port/in/DeviceSnapshotUseCase.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/port/in/DeviceSnapshotUseCase.java
@@ -0,0 +1,8 @@
+package com.ruoyi.wcs.port.in;
+
+import com.ruoyi.wcs.application.device.ReconcileResult;
+import com.ruoyi.wcs.domain.execution.DeviceSnapshot;
+
+public interface DeviceSnapshotUseCase {
+    ReconcileResult reconcile(DeviceSnapshot snapshot);
+}
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/port/in/RecoveryUseCase.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/port/in/RecoveryUseCase.java
@@ -0,0 +1,10 @@
+package com.ruoyi.wcs.port.in;
+
+import com.ruoyi.wcs.domain.recovery.RecoveryEvidence;
+
+public interface RecoveryUseCase {
+    void confirmFacts(String caseId, String operatorId,
+                      RecoveryEvidence evidence);
+
+    void abort(String caseId, String operatorId, String reason);
+}
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/port/out/ControlLeaseRepository.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/port/out/ControlLeaseRepository.java
@@ -0,0 +1,5 @@
+package com.ruoyi.wcs.port.out;
+
+public interface ControlLeaseRepository {
+    long acquireNewFencingToken();
+}
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/port/out/RecoveryCaseOpener.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/port/out/RecoveryCaseOpener.java
@@ -5,4 +5,10 @@ import java.util.List;
 public interface RecoveryCaseOpener {
    void open(String deviceId, String commandKey, String reason,
              List<String> evidenceIds);
+
+    default void open(String deviceId, String commandKey, String reason,
+                      List<String> evidenceIds, String expectedTaskNumber,
+                      String expectedSite, String expectedLoadState) {
+        open(deviceId, commandKey, reason, evidenceIds);
+    }
 }
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/port/out/RecoveryCaseRepository.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/port/out/RecoveryCaseRepository.java
@@ -0,0 +1,17 @@
+package com.ruoyi.wcs.port.out;
+
+import com.ruoyi.wcs.domain.recovery.RecoveryCase;
+
+import java.util.List;
+
+public interface RecoveryCaseRepository {
+    RecoveryCase findById(String caseId);
+
+    RecoveryCase findOpen(String deviceId, String commandKey);
+
+    List<RecoveryCase> findOpenCases();
+
+    RecoveryCase saveIfAbsentOpen(RecoveryCase recoveryCase);
+
+    void save(RecoveryCase recoveryCase);
+}
--- a/ruoyi-wcs/src/main/java/com/ruoyi/wcs/port/out/StartupRecoveryDataProvider.java
+++ b/ruoyi-wcs/src/main/java/com/ruoyi/wcs/port/out/StartupRecoveryDataProvider.java
@@ -0,0 +1,12 @@
+package com.ruoyi.wcs.port.out;
+
+import com.ruoyi.wcs.domain.recovery.StartupRecoveryCandidate;
+
+import java.util.List;
+import java.util.Set;
+
+public interface StartupRecoveryDataProvider {
+    List<StartupRecoveryCandidate> loadActiveCandidates();
+
+    Set<String> findUnexplainedGlobalResourceKeys();
+}
--- a/ruoyi-wcs/src/test/java/com/ruoyi/wcs/application/recovery/StartupRecoveryServiceTest.java
+++ b/ruoyi-wcs/src/test/java/com/ruoyi/wcs/application/recovery/StartupRecoveryServiceTest.java
@@ -0,0 +1,420 @@
+package com.ruoyi.wcs.application.recovery;
+
+import com.ruoyi.wcs.application.device.ReconcileResult;
+import com.ruoyi.wcs.application.device.ReconcileStatus;
+import com.ruoyi.wcs.domain.recovery.ControlStatus;
+import com.ruoyi.wcs.domain.recovery.RecoveryCase;
+import com.ruoyi.wcs.domain.recovery.RecoveryEvidence;
+import com.ruoyi.wcs.domain.recovery.StartupRecoveryCandidate;
+import com.ruoyi.wcs.domain.recovery.StartupRecoveryResult;
+import com.ruoyi.wcs.domain.shared.DomainRuleViolation;
+import com.ruoyi.wcs.domain.shared.IdGenerator;
+import com.ruoyi.wcs.domain.execution.DeviceSnapshot;
+import com.ruoyi.wcs.port.in.DeviceSnapshotUseCase;
+import com.ruoyi.wcs.port.out.ControlLeaseRepository;
+import com.ruoyi.wcs.port.out.DeviceSnapshotGateway;
+import com.ruoyi.wcs.port.out.RecoveryCaseRepository;
+import com.ruoyi.wcs.port.out.StartupRecoveryDataProvider;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class StartupRecoveryServiceTest {
+
+    @Test
+    void startupKeepsDispatchPausedUntilAllActiveDevicesAreReconciled() {
+        FakeRecoveryCases cases = new FakeRecoveryCases();
+        RecoveryApplicationService recovery = recoveryService(cases);
+        StartupRecoveryDataProvider data = new StartupRecoveryDataProvider() {
+            @Override
+            public List<StartupRecoveryCandidate> loadActiveCandidates() {
+                return Arrays.asList(
+                        candidate("ORDER-1", "VEH-1", "CMD-1", "NODE:A"),
+                        candidate("ORDER-2", "VEH-2", "CMD-2", "LIFT:L1"));
+            }
+
+            @Override
+            public Set<String> findUnexplainedGlobalResourceKeys() {
+                return Collections.singleton("LIFT:L1");
+            }
+        };
+        DeviceSnapshotUseCase reconciler = new DeviceSnapshotUseCase() {
+            @Override
+            public ReconcileResult reconcile(DeviceSnapshot snapshot) {
+                return new ReconcileResult("VEH-1".equals(snapshot.getDeviceId())
+                        ? ReconcileStatus.CONSISTENT
+                        : ReconcileStatus.RECOVERY_REQUIRED,
+                        snapshot.getCommandKey(), Collections.<String>emptyList(),
+                        "VEH-1".equals(snapshot.getDeviceId())
+                                ? null : "TASK_NUMBER_MISMATCH");
+            }
+        };
+        StartupRecoveryService service = new StartupRecoveryService(
+                new FakeLease(), data, new FakeSnapshots(), reconciler,
+                recovery, cases);
+
+        StartupRecoveryResult result = service.recover();
+
+        assertEquals(ControlStatus.PAUSED, result.getControlStatus());
+        assertEquals(Collections.singleton("VEH-2"),
+                result.getIsolatedDeviceIds());
+        assertEquals(Collections.singleton("ORDER-1"), result.getResumedOrderIds());
+        assertEquals(Collections.singleton("LIFT:L1"),
+                result.getIsolatedResourceKeys());
+        assertEquals(1, result.getUnresolvedRecoveryCaseIds().size());
+    }
+
+    @Test
+    void startupPartiallyEnablesWhenOnlyDeviceScopedWorkIsIsolated() {
+        FakeRecoveryCases cases = new FakeRecoveryCases();
+        RecoveryApplicationService recovery = recoveryService(cases);
+        StartupRecoveryDataProvider data = new StartupRecoveryDataProvider() {
+            @Override
+            public List<StartupRecoveryCandidate> loadActiveCandidates() {
+                return Collections.singletonList(candidate(
+                        "ORDER-1", "VEH-1", "CMD-1", "NODE:A"));
+            }
+
+            @Override
+            public Set<String> findUnexplainedGlobalResourceKeys() {
+                return Collections.emptySet();
+            }
+        };
+        DeviceSnapshotUseCase reconciler = new DeviceSnapshotUseCase() {
+            @Override
+            public ReconcileResult reconcile(DeviceSnapshot snapshot) {
+                return new ReconcileResult(ReconcileStatus.RECOVERY_REQUIRED,
+                        snapshot.getCommandKey(), Collections.<String>emptyList(),
+                        "TASK_NUMBER_MISMATCH");
+            }
+        };
+
+        StartupRecoveryResult result = new StartupRecoveryService(
+                new FakeLease(), data, new FakeSnapshots(), reconciler,
+                recovery, cases).recover();
+
+        assertEquals(ControlStatus.PARTIALLY_ENABLED, result.getControlStatus());
+        assertEquals(Collections.singleton("VEH-1"),
+                result.getIsolatedDeviceIds());
+        assertEquals(Collections.singleton("NODE:A"),
+                result.getIsolatedResourceKeys());
+    }
+
+    @Test
+    void startupEnablesOnlyAfterReadingAndReconcilingEveryActiveDevice() {
+        FakeRecoveryCases cases = new FakeRecoveryCases();
+        final Set<String> readDevices = new LinkedHashSet<String>();
+        final Set<String> reconciledDevices = new LinkedHashSet<String>();
+        StartupRecoveryDataProvider data = new StartupRecoveryDataProvider() {
+            @Override
+            public List<StartupRecoveryCandidate> loadActiveCandidates() {
+                return Arrays.asList(
+                        candidate("ORDER-1", "VEH-1", "CMD-1", "NODE:A"),
+                        candidate("ORDER-2", "VEH-2", "CMD-2", "NODE:B"));
+            }
+
+            @Override
+            public Set<String> findUnexplainedGlobalResourceKeys() {
+                return Collections.emptySet();
+            }
+        };
+        DeviceSnapshotGateway snapshots = new DeviceSnapshotGateway() {
+            @Override
+            public DeviceSnapshot readSnapshot(String deviceId) {
+                readDevices.add(deviceId);
+                return snapshot(deviceId);
+            }
+        };
+        DeviceSnapshotUseCase reconciler = new DeviceSnapshotUseCase() {
+            @Override
+            public ReconcileResult reconcile(DeviceSnapshot snapshot) {
+                reconciledDevices.add(snapshot.getDeviceId());
+                return new ReconcileResult(ReconcileStatus.CONSISTENT,
+                        snapshot.getCommandKey(), Collections.<String>emptyList(), null);
+            }
+        };
+
+        StartupRecoveryResult result = new StartupRecoveryService(
+                new FakeLease(), data, snapshots, reconciler,
+                recoveryService(cases), cases).recover();
+
+        assertEquals(ControlStatus.ENABLED, result.getControlStatus());
+        assertEquals(99L, result.getFencingToken());
+        assertEquals(new LinkedHashSet<String>(Arrays.asList("VEH-1", "VEH-2")),
+                readDevices);
+        assertEquals(readDevices, reconciledDevices);
+        assertEquals(new LinkedHashSet<String>(Arrays.asList("ORDER-1", "ORDER-2")),
+                result.getResumedOrderIds());
+    }
+
+    @Test
+    void staleStartupSnapshotIsIsolatedInsteadOfResumingTheOrder() {
+        FakeRecoveryCases cases = new FakeRecoveryCases();
+        StartupRecoveryDataProvider data = dataFor(Collections.singletonList(
+                candidate("ORDER-1", "VEH-1", "CMD-1", "NODE:A")));
+        DeviceSnapshotUseCase reconciler = new DeviceSnapshotUseCase() {
+            @Override
+            public ReconcileResult reconcile(DeviceSnapshot snapshot) {
+                return new ReconcileResult(ReconcileStatus.IGNORED_STALE,
+                        snapshot.getCommandKey(), Collections.<String>emptyList(),
+                        null);
+            }
+        };
+
+        StartupRecoveryResult result = new StartupRecoveryService(
+                new FakeLease(), data, new FakeSnapshots(), reconciler,
+                recoveryService(cases), cases).recover();
+
+        assertEquals(ControlStatus.PARTIALLY_ENABLED, result.getControlStatus());
+        assertEquals(Collections.emptySet(), result.getResumedOrderIds());
+        assertEquals(Collections.singleton("VEH-1"),
+                result.getIsolatedDeviceIds());
+    }
+
+    @Test
+    void snapshotForAnotherDeviceCannotResumeTheCandidateOrder() {
+        FakeRecoveryCases cases = new FakeRecoveryCases();
+        final int[] reconcileCalls = {0};
+        DeviceSnapshotGateway snapshots = new DeviceSnapshotGateway() {
+            @Override
+            public DeviceSnapshot readSnapshot(String deviceId) {
+                return snapshot("VEH-2");
+            }
+        };
+        DeviceSnapshotUseCase reconciler = new DeviceSnapshotUseCase() {
+            @Override
+            public ReconcileResult reconcile(DeviceSnapshot snapshot) {
+                reconcileCalls[0]++;
+                return new ReconcileResult(ReconcileStatus.CONSISTENT,
+                        snapshot.getCommandKey(), Collections.<String>emptyList(), null);
+            }
+        };
+
+        StartupRecoveryResult result = new StartupRecoveryService(
+                new FakeLease(), dataFor(Collections.singletonList(candidate(
+                        "ORDER-1", "VEH-1", "CMD-1", "NODE:A"))),
+                snapshots, reconciler, recoveryService(cases), cases).recover();
+
+        assertEquals(0, reconcileCalls[0]);
+        assertEquals(Collections.emptySet(), result.getResumedOrderIds());
+        assertEquals(Collections.singleton("VEH-1"),
+                result.getIsolatedDeviceIds());
+    }
+
+    @Test
+    void automaticCaseWithExpectedFactsCanBeOperatorConfirmed() {
+        FakeRecoveryCases cases = new FakeRecoveryCases();
+        RecoveryApplicationService recovery = recoveryService(cases);
+        StartupRecoveryCandidate candidate = new StartupRecoveryCandidate(
+                "ORDER-1", "VEH-1", "CMD-1",
+                Collections.singletonList("NODE:A"),
+                "TASK-1", "SITE-B", "LOADED");
+        DeviceSnapshotUseCase reconciler = new DeviceSnapshotUseCase() {
+            @Override
+            public ReconcileResult reconcile(DeviceSnapshot snapshot) {
+                return new ReconcileResult(ReconcileStatus.RECOVERY_REQUIRED,
+                        snapshot.getCommandKey(), Collections.<String>emptyList(),
+                        "SITE_MISMATCH");
+            }
+        };
+        new StartupRecoveryService(new FakeLease(),
+                dataFor(Collections.singletonList(candidate)),
+                new FakeSnapshots(), reconciler, recovery, cases).recover();
+        RecoveryCase recoveryCase = cases.findOpen("VEH-1", "CMD-1");
+
+        recovery.confirmFacts(recoveryCase.getCaseId(), "OP-1",
+                new RecoveryEvidence("SITE-B", "LOADED", "TASK-1", "AUTO",
+                        Instant.EPOCH.plusSeconds(1), "PLC_AND_VISUAL",
+                        "matched expected startup facts"));
+
+        assertEquals("CONFIRMED", recoveryCase.getState().name());
+    }
+
+    @Test
+    void operatorCannotResolveCaseWithoutMatchingEvidence() {
+        FakeRecoveryCases cases = new FakeRecoveryCases();
+        RecoveryCase recoveryCase = RecoveryCase.open(
+                "RC-1", "VEH-1", "CMD-1", "TASK_NUMBER_MISMATCH",
+                Collections.singletonList("SNAP-1"), "TASK-1", "SITE-B",
+                "LOADED", Instant.EPOCH);
+        cases.save(recoveryCase);
+        RecoveryApplicationService service = recoveryService(cases);
+
+        assertThrows(DomainRuleViolation.class, () -> service.confirmFacts(
+                "RC-1", "OP-1", new RecoveryEvidence("SITE-A", "EMPTY",
+                        "WRONG-TASK", "AUTO", Instant.EPOCH.plusSeconds(1),
+                        "PLC_SNAPSHOT", "checked on site")));
+    }
+
+    @Test
+    void operatorCannotConfirmCaseWhoseExpectedFactsWereNotPersisted() {
+        FakeRecoveryCases cases = new FakeRecoveryCases();
+        RecoveryApplicationService service = recoveryService(cases);
+        service.open("VEH-1", "CMD-1", "AMBIGUOUS_SEND",
+                Collections.singletonList("SEND-1"));
+        RecoveryCase recoveryCase = cases.findOpen("VEH-1", "CMD-1");
+
+        assertThrows(DomainRuleViolation.class, () -> service.confirmFacts(
+                recoveryCase.getCaseId(), "OP-1", new RecoveryEvidence(
+                        "SITE-A", "EMPTY", "TASK-1", "AUTO",
+                        Instant.EPOCH.plusSeconds(1), "PLC_SNAPSHOT",
+                        "facts were guessed")));
+    }
+
+    @Test
+    void matchingOperatorEvidenceClosesTheCaseWithAnAuditTrail() {
+        FakeRecoveryCases cases = new FakeRecoveryCases();
+        cases.save(RecoveryCase.open("RC-1", "VEH-1", "CMD-1",
+                "TASK_NUMBER_MISMATCH", Collections.singletonList("SNAP-1"),
+                "TASK-1", "SITE-B", "LOADED", Instant.EPOCH));
+        RecoveryApplicationService service = recoveryService(cases);
+
+        service.confirmFacts("RC-1", "OP-1", new RecoveryEvidence(
+                "SITE-B", "LOADED", "TASK-1", "AUTO",
+                Instant.EPOCH.plusSeconds(2), "PLC_AND_VISUAL",
+                "task and load matched"));
+
+        assertEquals("OP-1", cases.findById("RC-1").getResolvedBy());
+        assertEquals("CONFIRMED", cases.findById("RC-1").getState().name());
+    }
+
+    @Test
+    void operatorAbortClosesTheCaseWithReasonAndOperatorAudit() {
+        FakeRecoveryCases cases = new FakeRecoveryCases();
+        cases.save(RecoveryCase.open("RC-1", "VEH-1", "CMD-1",
+                "TASK_NUMBER_MISMATCH", Collections.singletonList("SNAP-1"),
+                "TASK-1", "SITE-B", "LOADED", Instant.EPOCH));
+        RecoveryApplicationService service = recoveryService(cases);
+
+        service.abort("RC-1", "OP-2", "physical task was cancelled");
+
+        RecoveryCase aborted = cases.findById("RC-1");
+        assertEquals("ABORTED", aborted.getState().name());
+        assertEquals("OP-2", aborted.getResolvedBy());
+        assertEquals("physical task was cancelled", aborted.getResolutionReason());
+        assertEquals(Instant.EPOCH, aborted.getResolvedAt());
+    }
+
+    private static RecoveryApplicationService recoveryService(
+            RecoveryCaseRepository repository) {
+        return new RecoveryApplicationService(repository, new IdGenerator() {
+            private int sequence;
+
+            @Override
+            public String nextId() {
+                return "RC-" + (++sequence + 1);
+            }
+        }, new java.util.function.Supplier<Instant>() {
+            @Override
+            public Instant get() {
+                return Instant.EPOCH;
+            }
+        });
+    }
+
+    private static StartupRecoveryCandidate candidate(String orderId,
+                                                       String deviceId,
+                                                       String commandKey,
+                                                       String resourceKey) {
+        return new StartupRecoveryCandidate(orderId, deviceId, commandKey,
+                Collections.singletonList(resourceKey));
+    }
+
+    private static StartupRecoveryDataProvider dataFor(
+            final List<StartupRecoveryCandidate> candidates) {
+        return new StartupRecoveryDataProvider() {
+            @Override
+            public List<StartupRecoveryCandidate> loadActiveCandidates() {
+                return candidates;
+            }
+
+            @Override
+            public Set<String> findUnexplainedGlobalResourceKeys() {
+                return Collections.emptySet();
+            }
+        };
+    }
+
+    private static DeviceSnapshot snapshot(String deviceId) {
+        String suffix = deviceId.substring(deviceId.length() - 1);
+        return new DeviceSnapshot("SNAP-" + deviceId, deviceId,
+                "CMD-" + suffix, "TASK-1", 1L, Instant.EPOCH, true,
+                "BOOT-1", 1L, 1L, "SITE-A",
+                Collections.singletonList("SITE-A"), false);
+    }
+
+    private static final class FakeLease implements ControlLeaseRepository {
+        @Override
+        public long acquireNewFencingToken() {
+            return 99L;
+        }
+    }
+
+    private static final class FakeSnapshots implements DeviceSnapshotGateway {
+        @Override
+        public DeviceSnapshot readSnapshot(String deviceId) {
+            return snapshot(deviceId);
+        }
+    }
+
+    private static final class FakeRecoveryCases
+            implements RecoveryCaseRepository {
+        private final Map<String, RecoveryCase> values =
+                new LinkedHashMap<String, RecoveryCase>();
+
+        @Override
+        public RecoveryCase findById(String caseId) {
+            return values.get(caseId);
+        }
+
+        @Override
+        public RecoveryCase findOpen(String deviceId, String commandKey) {
+            for (RecoveryCase value : values.values()) {
+                if (value.isOpen() && value.getDeviceId().equals(deviceId)
+                        && value.getCommandKey().equals(commandKey)) {
+                    return value;
+                }
+            }
+            return null;
+        }
+
+        @Override
+        public List<RecoveryCase> findOpenCases() {
+            List<RecoveryCase> result = new ArrayList<RecoveryCase>();
+            for (RecoveryCase value : values.values()) {
+                if (value.isOpen()) {
+                    result.add(value);
+                }
+            }
+            return result;
+        }
+
+        @Override
+        public void save(RecoveryCase recoveryCase) {
+            values.put(recoveryCase.getCaseId(), recoveryCase);
+        }
+
+        @Override
+        public synchronized RecoveryCase saveIfAbsentOpen(
+                RecoveryCase recoveryCase) {
+            RecoveryCase existing = findOpen(recoveryCase.getDeviceId(),
+                    recoveryCase.getCommandKey());
+            if (existing != null) {
+                return existing;
+            }
+            save(recoveryCase);
+            return recoveryCase;
+        }
+    }
+}