Skip to content

Commit 82b3039

Browse files
K2GO-386 refactor(app-backstop): extract pure escalation rule; guard reap; fix debug/report
Second code-review pass on L3b. Five fixes. 1. Deep-op guard. DiskGuard.deepOpActive(ctx) reads EnvironmentLock.currentHolder. If a STOPPED-class op (clone, backup, restore, install) holds the box, the tick logs and returns without reaping. A reap mid-extract would corrupt the rootfs. 2. Debug path no longer escalates. checkWithFloor runs in FORCED mode. FORCED always CONTAINs. It never advances the trip state. Before, three debug triggers could stop the box. 3. Consecutive-trip semantics. A non-critical tick ends the spell and resets the count. Escalation now means the disk stayed critical across consecutive restarts, so the restart is not fixing it. Before, the count reset only on a 30-min gap or app restart. 4. Testable escalation. The decision moves to the pure DiskGuardEscalation.next(...). No android.* imports. DiskGuardEscalationTest covers five cases on a plain JVM. 5. Report cadence. A CONTAIN reports only on the first trip of a spell. An ESCALATE always reports. A thrash no longer enqueues one analytics event per trip. See controller/docs/ADR-386-unattended-disk-containment.md.
1 parent 7a2d145 commit 82b3039

3 files changed

Lines changed: 186 additions & 35 deletions

File tree

‎controller/app/src/main/java/org/appdevforall/k2go/diskguard/DiskGuard.java‎

Lines changed: 67 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,17 @@
88
*
99
* One tick reads free space (StorageProbe) and asks the pure rule
1010
* (DiskGuardPolicy). On CRITICAL it CONFIRMS with a fresh re-read
11-
* (it never acts on one reading), then reaps the box and reclaims
12-
* the runaway log. The default action KEEPS THE SYSTEM ALIVE: it
13-
* does not force the server down. A fresh service under a fresh
14-
* proot does not busy-loop, so the ADFA-5343 reconciler relaunches
15-
* a clean box. Only when the fill KEEPS RETURNING after several
16-
* restarts in a window does the guard stop and stay down as a last
17-
* resort, and tell the user. Every trip is reported to developers
18-
* through the delivery backbone.
11+
* (it never acts on one reading). It does NOT reap while a deep op
12+
* (clone/backup/restore/install) holds the box, because a reap
13+
* mid-operation would corrupt it. Otherwise it reaps the box and
14+
* reclaims the runaway log.
15+
*
16+
* The default action KEEPS THE SYSTEM ALIVE: it does not force the
17+
* server down. A fresh service under a fresh proot does not
18+
* busy-loop, so the ADFA-5343 reconciler relaunches a clean box.
19+
* Only when the disk stays critical for several trips in a row
20+
* (DiskGuardEscalation) does the guard stop and stay down as a last
21+
* resort and tell the user. Trips are reported to developers.
1922
*
2023
* Why Android-side: the fill happens when the box proot dies and a
2124
* service is orphaned off proot. An in-box kill does not reach the
@@ -38,10 +41,13 @@
3841

3942
import org.appdevforall.k2go.R;
4043
import org.appdevforall.k2go.delivery.DeliveryManager;
44+
import org.appdevforall.k2go.diskguard.domain.DiskGuardEscalation;
4145
import org.appdevforall.k2go.diskguard.domain.DiskGuardPolicy;
46+
import org.appdevforall.k2go.env.EnvironmentLock;
4247
import org.appdevforall.k2go.env.EnvironmentProcess;
4348
import org.appdevforall.k2go.env.ServerLifecycleReconciler;
4449
import org.appdevforall.k2go.storage.StorageProbe;
50+
import org.appdevforall.k2go.system.domain.Operation;
4551

4652
import org.json.JSONObject;
4753

@@ -60,16 +66,16 @@ public final class DiskGuard {
6066
// a momentary spike does not (ADR-386, "confirm before acting").
6167
private static final long CONFIRM_DELAY_MS = 1000L;
6268

63-
// Restart-to-keep-alive is the default. If the fill returns this many times inside TRIP_WINDOW_MS,
64-
// the restart is not fixing it, so the guard escalates to stop-and-stay-down as a last resort.
69+
// Restart-to-keep-alive is the default. If the disk stays critical this many trips in a row, the
70+
// restart is not fixing it, so the guard escalates to stop-and-stay-down as a last resort.
6571
private static final int ESCALATE_AFTER_TRIPS = 3;
6672
private static final long TRIP_WINDOW_MS = 30L * 60L * 1000L;
6773

6874
private static final String CHANNEL_ID = "disk_guard_channel";
6975
private static final int NOTIF_ID = 7386;
7076

7177
// Recent-trip state, in memory on purpose: it resets when the app process restarts, so a stale count
72-
// never carries across a restart. Guarded by the class monitor.
78+
// never carries across a restart. Read and written only under advanceTripState (class monitor).
7379
private static long lastTripElapsedMs = -1L;
7480
private static int tripCount = 0;
7581

@@ -80,36 +86,57 @@ private DiskGuard() {}
8086
* repeatedly from a poller. A null or UNKNOWN read is a no-op.
8187
*/
8288
public static boolean check(Context ctx) {
83-
return checkWithFloor(ctx, DiskGuardPolicy.CRITICAL_FLOOR_BYTES);
89+
return run(ctx, DiskGuardPolicy.CRITICAL_FLOOR_BYTES, false);
8490
}
8591

8692
/**
87-
* As {@link #check} but with an explicit floor. The debug device-verify hook passes a huge floor so
88-
* any real free-space read is CRITICAL. This forces the action without filling the disk for real.
93+
* The debug device-verify hook. It passes a huge floor so any real free-space read is CRITICAL, and
94+
* runs in FORCED mode: it exercises the reap/reclaim/restart path once but does NOT advance the real
95+
* escalation count, so triggering it repeatedly cannot stop the box.
8996
*/
9097
public static boolean checkWithFloor(Context ctx, long floorBytes) {
98+
return run(ctx, floorBytes, true);
99+
}
100+
101+
private static boolean run(Context ctx, long floorBytes, boolean forced) {
91102
if (ctx == null) return false;
92-
if (!confirmCritical(ctx, floorBytes)) return false;
103+
boolean critical = confirmCritical(ctx, floorBytes);
104+
105+
DiskGuardEscalation.Verdict v;
106+
if (forced) {
107+
// Forced (debug): CONTAIN if critical, and never touch the shared trip state or escalate.
108+
v = new DiskGuardEscalation.Verdict(
109+
critical ? DiskGuardEscalation.Action.CONTAIN : DiskGuardEscalation.Action.NONE,
110+
0, 0L, true);
111+
} else {
112+
v = advanceTripState(critical);
113+
}
114+
if (v.action == DiskGuardEscalation.Action.NONE) return false;
115+
116+
// Never reap while a deep op owns the box (clone/backup/restore/install). A reap mid-operation
117+
// would corrupt it. EnvironmentLock is the one owner of "is a stop-class op running".
118+
if (deepOpActive(ctx)) {
119+
Log.w(TAG, "K2GO-386: disk critical but a deep op holds the box; not reaping this tick");
120+
return false;
121+
}
93122

94-
int trip = recordTrip();
95123
boolean reaped = EnvironmentProcess.reapBox(ctx);
96124
long reclaimed = reclaimRunawayLog(ctx);
97125

98-
if (trip < ESCALATE_AFTER_TRIPS) {
99-
// Default: keep the system alive. Do NOT force desired=DOWN. The reconciler (ADFA-5343) owns
100-
// "should the box be up"; it is left UP, so it relaunches a fresh box. Ask it to act now
101-
// instead of on its next tick.
102-
ServerLifecycleReconciler.get().requestReconcileNow();
103-
report(ctx, "contained", floorBytes, reaped, reclaimed, trip);
104-
Log.w(TAG, "K2GO-386: contained disk pressure (trip " + trip + "): reaped=" + reaped
105-
+ ", reclaimed=" + reclaimed + " B, box restarting");
106-
} else {
126+
if (v.action == DiskGuardEscalation.Action.ESCALATE) {
107127
// Last resort: the fill keeps returning after restarts. Stop and stay down through the one
108128
// persisted lever, and tell the user. The user re-enables the server after freeing space.
109129
ServerLifecycleReconciler.get().setUserWantsOn(ctx, false);
110130
notifyUser(ctx);
111-
report(ctx, "escalated_stopped", floorBytes, reaped, reclaimed, trip);
112-
Log.w(TAG, "K2GO-386: recurring disk pressure (trip " + trip + "): stopped and staying down");
131+
report(ctx, "escalated_stopped", floorBytes, reaped, reclaimed, v.tripCount);
132+
Log.w(TAG, "K2GO-386: recurring disk pressure (trip " + v.tripCount + "): stopped and staying down");
133+
} else {
134+
// Default: keep the system alive. Leave desired=UP and ask the reconciler to relaunch a fresh
135+
// box now. Report only the first trip of a spell so a thrash does not spam telemetry.
136+
ServerLifecycleReconciler.get().requestReconcileNow();
137+
if (v.firstOfSpell) report(ctx, "contained", floorBytes, reaped, reclaimed, v.tripCount);
138+
Log.w(TAG, "K2GO-386: contained disk pressure (trip " + v.tripCount + "): reaped=" + reaped
139+
+ ", reclaimed=" + reclaimed + " B, box restarting");
113140
}
114141
return true;
115142
}
@@ -135,14 +162,19 @@ private static boolean confirmCritical(Context ctx, long floorBytes) {
135162
return stillCritical;
136163
}
137164

138-
/** Count trips inside a sliding window. Returns the trip number in the current window (1-based). */
139-
private static synchronized int recordTrip() {
140-
long now = SystemClock.elapsedRealtime();
141-
if (lastTripElapsedMs < 0L || now - lastTripElapsedMs > TRIP_WINDOW_MS) {
142-
tripCount = 0;
143-
}
144-
lastTripElapsedMs = now;
145-
return ++tripCount;
165+
/** Advance the shared trip state with the pure rule and return the verdict. */
166+
private static synchronized DiskGuardEscalation.Verdict advanceTripState(boolean critical) {
167+
DiskGuardEscalation.Verdict v = DiskGuardEscalation.next(
168+
critical, SystemClock.elapsedRealtime(), lastTripElapsedMs, tripCount,
169+
TRIP_WINDOW_MS, ESCALATE_AFTER_TRIPS);
170+
tripCount = v.tripCount;
171+
lastTripElapsedMs = v.lastElapsedMs;
172+
return v;
173+
}
174+
175+
/** True when a stop-class operation (clone/backup/restore/install) currently holds the box. */
176+
private static boolean deepOpActive(Context ctx) {
177+
return EnvironmentLock.currentHolder(ctx).executionClass == Operation.ExecutionClass.STOPPED;
146178
}
147179

148180
/** Report the event to developers through the delivery backbone (unattended; not user-facing). */
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
* ============================================================================
3+
* Name : DiskGuardEscalation.java
4+
* Author : AppDevForAll
5+
* Copyright : Copyright (c) 2026 AppDevForAll
6+
* Description : K2GO-386 (Layer 3). The pure escalation rule for the app-side
7+
* disk-fill backstop. It decides, on one guard tick, what to do
8+
* from whether the disk is critical now plus the recent-trip state.
9+
* No android.*, so it is unit-tested on a plain JVM. See ADR-386.
10+
* ============================================================================
11+
*/
12+
package org.appdevforall.k2go.diskguard.domain;
13+
14+
/**
15+
* A trip is a confirmed-critical tick. Trips count only while they stay CONSECUTIVE: a non-critical tick
16+
* ends the spell and resets the count, and a long gap (over the window) resets it too. The default action
17+
* is CONTAIN (reap, then let the box restart). If the disk stays critical for {@code escalateAfter} trips
18+
* in a row, the restart is not fixing it, so the action becomes ESCALATE (stop and stay down).
19+
*/
20+
public final class DiskGuardEscalation {
21+
22+
public enum Action { NONE, CONTAIN, ESCALATE }
23+
24+
/** The new trip state plus the action to take. Immutable. */
25+
public static final class Verdict {
26+
public final Action action;
27+
public final int tripCount; // new consecutive-trip count; 0 when not critical
28+
public final long lastElapsedMs; // new last-trip time
29+
public final boolean firstOfSpell; // tripCount == 1; used to report once per spell
30+
31+
public Verdict(Action action, int tripCount, long lastElapsedMs, boolean firstOfSpell) {
32+
this.action = action;
33+
this.tripCount = tripCount;
34+
this.lastElapsedMs = lastElapsedMs;
35+
this.firstOfSpell = firstOfSpell;
36+
}
37+
}
38+
39+
private DiskGuardEscalation() {}
40+
41+
/**
42+
* @param critical whether the disk is critical on this tick (already confirmed).
43+
* @param nowMs a monotonic clock reading (SystemClock.elapsedRealtime).
44+
* @param lastElapsedMs the previous trip's clock reading, or negative if none.
45+
* @param prevCount the previous consecutive-trip count.
46+
* @param windowMs a gap longer than this starts a new spell.
47+
* @param escalateAfter the trip number at which to escalate.
48+
*/
49+
public static Verdict next(boolean critical, long nowMs, long lastElapsedMs, int prevCount,
50+
long windowMs, int escalateAfter) {
51+
if (!critical) {
52+
// The disk recovered. End the spell so a later, separate fill starts fresh.
53+
return new Verdict(Action.NONE, 0, lastElapsedMs, false);
54+
}
55+
int count;
56+
if (prevCount <= 0 || lastElapsedMs < 0L || nowMs - lastElapsedMs > windowMs) {
57+
count = 1; // first trip of a new spell
58+
} else {
59+
count = prevCount + 1;
60+
}
61+
Action action = count >= escalateAfter ? Action.ESCALATE : Action.CONTAIN;
62+
return new Verdict(action, count, nowMs, count == 1);
63+
}
64+
}
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package org.appdevforall.k2go.diskguard.domain;
2+
3+
import static org.junit.Assert.assertEquals;
4+
import static org.junit.Assert.assertFalse;
5+
import static org.junit.Assert.assertTrue;
6+
7+
import org.appdevforall.k2go.diskguard.domain.DiskGuardEscalation.Action;
8+
import org.appdevforall.k2go.diskguard.domain.DiskGuardEscalation.Verdict;
9+
10+
import org.junit.Test;
11+
12+
public class DiskGuardEscalationTest {
13+
14+
private static final long WINDOW = 30L * 60L * 1000L;
15+
private static final int ESCALATE = 3;
16+
17+
@Test
18+
public void notCritical_isNoneAndResetsCount() {
19+
Verdict v = DiskGuardEscalation.next(false, 1000L, 500L, 2, WINDOW, ESCALATE);
20+
assertEquals(Action.NONE, v.action);
21+
assertEquals(0, v.tripCount);
22+
}
23+
24+
@Test
25+
public void firstCritical_isContainTripOne() {
26+
Verdict v = DiskGuardEscalation.next(true, 1000L, -1L, 0, WINDOW, ESCALATE);
27+
assertEquals(Action.CONTAIN, v.action);
28+
assertEquals(1, v.tripCount);
29+
assertTrue(v.firstOfSpell);
30+
}
31+
32+
@Test
33+
public void consecutiveCritical_incrementsWithinWindow() {
34+
Verdict v = DiskGuardEscalation.next(true, 2000L, 1000L, 1, WINDOW, ESCALATE);
35+
assertEquals(Action.CONTAIN, v.action);
36+
assertEquals(2, v.tripCount);
37+
assertFalse(v.firstOfSpell);
38+
}
39+
40+
@Test
41+
public void escalatesAtThreshold() {
42+
Verdict v = DiskGuardEscalation.next(true, 3000L, 2000L, 2, WINDOW, ESCALATE);
43+
assertEquals(Action.ESCALATE, v.action);
44+
assertEquals(3, v.tripCount);
45+
}
46+
47+
@Test
48+
public void gapBeyondWindow_startsNewSpell() {
49+
// prevCount is 2, but the gap exceeds the window, so the spell resets to trip 1.
50+
Verdict v = DiskGuardEscalation.next(true, 100_000_000L, 1000L, 2, WINDOW, ESCALATE);
51+
assertEquals(Action.CONTAIN, v.action);
52+
assertEquals(1, v.tripCount);
53+
assertTrue(v.firstOfSpell);
54+
}
55+
}

0 commit comments

Comments
 (0)