feat: defer ecc2 handoffs on saturated teams

This commit is contained in:
Affaan Mustafa
2026-04-08 03:06:19 -07:00
parent a3f600e25f
commit 91e145338f
3 changed files with 264 additions and 38 deletions

View File

@@ -279,16 +279,25 @@ async fn main() -> Result<()> {
use_worktree, use_worktree,
) )
.await?; .await?;
println!( if session::manager::assignment_action_routes_work(outcome.action) {
"Assignment routed: {} -> {} ({})", println!(
short_session(&lead_id), "Assignment routed: {} -> {} ({})",
short_session(&outcome.session_id), short_session(&lead_id),
match outcome.action { short_session(&outcome.session_id),
session::manager::AssignmentAction::Spawned => "spawned", match outcome.action {
session::manager::AssignmentAction::ReusedIdle => "reused-idle", session::manager::AssignmentAction::Spawned => "spawned",
session::manager::AssignmentAction::ReusedActive => "reused-active", session::manager::AssignmentAction::ReusedIdle => "reused-idle",
} session::manager::AssignmentAction::ReusedActive => "reused-active",
); session::manager::AssignmentAction::DeferredSaturated => unreachable!(),
}
);
} else {
println!(
"Assignment deferred: {} is saturated; task stayed in {} inbox",
short_session(&lead_id),
short_session(&lead_id),
);
}
} }
Some(Commands::DrainInbox { Some(Commands::DrainInbox {
session_id, session_id,
@@ -309,10 +318,18 @@ async fn main() -> Result<()> {
if outcomes.is_empty() { if outcomes.is_empty() {
println!("No unread task handoffs for {}", short_session(&lead_id)); println!("No unread task handoffs for {}", short_session(&lead_id));
} else { } else {
let routed_count = outcomes
.iter()
.filter(|outcome| session::manager::assignment_action_routes_work(outcome.action))
.count();
let deferred_count = outcomes.len().saturating_sub(routed_count);
println!( println!(
"Routed {} inbox task handoff(s) from {}", "Processed {} inbox task handoff(s) from {} ({} routed, {} deferred)",
outcomes.len(), outcomes.len(),
short_session(&lead_id) short_session(&lead_id)
,
routed_count,
deferred_count
); );
for outcome in outcomes { for outcome in outcomes {
println!( println!(
@@ -323,6 +340,9 @@ async fn main() -> Result<()> {
session::manager::AssignmentAction::Spawned => "spawned", session::manager::AssignmentAction::Spawned => "spawned",
session::manager::AssignmentAction::ReusedIdle => "reused-idle", session::manager::AssignmentAction::ReusedIdle => "reused-idle",
session::manager::AssignmentAction::ReusedActive => "reused-active", session::manager::AssignmentAction::ReusedActive => "reused-active",
session::manager::AssignmentAction::DeferredSaturated => {
"deferred-saturated"
}
}, },
outcome.task outcome.task
); );
@@ -345,18 +365,38 @@ async fn main() -> Result<()> {
if outcomes.is_empty() { if outcomes.is_empty() {
println!("No unread task handoff backlog found"); println!("No unread task handoff backlog found");
} else { } else {
let total_routed: usize = outcomes.iter().map(|outcome| outcome.routed.len()).sum(); let total_processed: usize = outcomes.iter().map(|outcome| outcome.routed.len()).sum();
let total_routed: usize = outcomes
.iter()
.map(|outcome| {
outcome
.routed
.iter()
.filter(|item| session::manager::assignment_action_routes_work(item.action))
.count()
})
.sum();
let total_deferred = total_processed.saturating_sub(total_routed);
println!( println!(
"Auto-dispatched {} task handoff(s) across {} lead session(s)", "Auto-dispatch processed {} task handoff(s) across {} lead session(s) ({} routed, {} deferred)",
total_processed,
outcomes.len(),
total_routed, total_routed,
outcomes.len() total_deferred
); );
for outcome in outcomes { for outcome in outcomes {
let routed = outcome
.routed
.iter()
.filter(|item| session::manager::assignment_action_routes_work(item.action))
.count();
let deferred = outcome.routed.len().saturating_sub(routed);
println!( println!(
"- {} | unread {} | routed {}", "- {} | unread {} | routed {} | deferred {}",
short_session(&outcome.lead_session_id), short_session(&outcome.lead_session_id),
outcome.unread_count, outcome.unread_count,
outcome.routed.len() routed,
deferred
); );
} }
} }
@@ -374,11 +414,23 @@ async fn main() -> Result<()> {
lead_limit, lead_limit,
) )
.await?; .await?;
let total_routed: usize = outcome let total_processed: usize = outcome
.dispatched .dispatched
.iter() .iter()
.map(|dispatch| dispatch.routed.len()) .map(|dispatch| dispatch.routed.len())
.sum(); .sum();
let total_routed: usize = outcome
.dispatched
.iter()
.map(|dispatch| {
dispatch
.routed
.iter()
.filter(|item| session::manager::assignment_action_routes_work(item.action))
.count()
})
.sum();
let total_deferred = total_processed.saturating_sub(total_routed);
let total_rerouted: usize = outcome let total_rerouted: usize = outcome
.rebalanced .rebalanced
.iter() .iter()
@@ -392,9 +444,11 @@ async fn main() -> Result<()> {
println!("Backlog already clear"); println!("Backlog already clear");
} else { } else {
println!( println!(
"Coordinated backlog: dispatched {} handoff(s) across {} lead(s); rebalanced {} handoff(s) across {} lead(s); remaining {} handoff(s) across {} session(s) [{} absorbable, {} saturated]", "Coordinated backlog: processed {} handoff(s) across {} lead(s) ({} routed, {} deferred); rebalanced {} handoff(s) across {} lead(s); remaining {} handoff(s) across {} session(s) [{} absorbable, {} saturated]",
total_routed, total_processed,
outcome.dispatched.len(), outcome.dispatched.len(),
total_routed,
total_deferred,
total_rerouted, total_rerouted,
outcome.rebalanced.len(), outcome.rebalanced.len(),
outcome.remaining_backlog_messages, outcome.remaining_backlog_messages,
@@ -470,6 +524,9 @@ async fn main() -> Result<()> {
session::manager::AssignmentAction::Spawned => "spawned", session::manager::AssignmentAction::Spawned => "spawned",
session::manager::AssignmentAction::ReusedIdle => "reused-idle", session::manager::AssignmentAction::ReusedIdle => "reused-idle",
session::manager::AssignmentAction::ReusedActive => "reused-active", session::manager::AssignmentAction::ReusedActive => "reused-active",
session::manager::AssignmentAction::DeferredSaturated => {
"deferred-saturated"
}
}, },
outcome.task outcome.task
); );

View File

@@ -121,7 +121,9 @@ pub async fn drain_inbox(
) )
.await?; .await?;
let _ = db.mark_message_read(message.id)?; if assignment_action_routes_work(outcome.action) {
let _ = db.mark_message_read(message.id)?;
}
outcomes.push(InboxDrainOutcome { outcomes.push(InboxDrainOutcome {
message_id: message.id, message_id: message.id,
task, task,
@@ -461,7 +463,7 @@ async fn assign_session_in_dir_with_runner_program(
}); });
} }
if let Some(idle_delegate) = delegates if let Some(_idle_delegate) = delegates
.iter() .iter()
.filter(|session| session.state == SessionState::Idle) .filter(|session| session.state == SessionState::Idle)
.min_by_key(|session| { .min_by_key(|session| {
@@ -471,16 +473,9 @@ async fn assign_session_in_dir_with_runner_program(
) )
}) })
{ {
send_task_handoff(
db,
&lead,
&idle_delegate.id,
task,
"reused idle delegate with existing inbox backlog",
)?;
return Ok(AssignmentOutcome { return Ok(AssignmentOutcome {
session_id: idle_delegate.id.clone(), session_id: lead.id.clone(),
action: AssignmentAction::ReusedIdle, action: AssignmentAction::DeferredSaturated,
}); });
} }
@@ -494,6 +489,13 @@ async fn assign_session_in_dir_with_runner_program(
) )
}) })
{ {
if unread_counts.get(&active_delegate.id).copied().unwrap_or(0) > 0 {
return Ok(AssignmentOutcome {
session_id: lead.id.clone(),
action: AssignmentAction::DeferredSaturated,
});
}
send_task_handoff( send_task_handoff(
db, db,
&lead, &lead,
@@ -1074,6 +1076,11 @@ pub enum AssignmentAction {
Spawned, Spawned,
ReusedIdle, ReusedIdle,
ReusedActive, ReusedActive,
DeferredSaturated,
}
pub fn assignment_action_routes_work(action: AssignmentAction) -> bool {
!matches!(action, AssignmentAction::DeferredSaturated)
} }
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
@@ -1862,6 +1869,73 @@ mod tests {
Ok(()) Ok(())
} }
#[tokio::test(flavor = "current_thread")]
async fn assign_session_defers_when_team_is_saturated() -> Result<()> {
let tempdir = TestDir::new("manager-assign-defer-saturated")?;
let repo_root = tempdir.path().join("repo");
init_git_repo(&repo_root)?;
let mut cfg = build_config(tempdir.path());
cfg.max_parallel_sessions = 1;
let db = StateStore::open(&cfg.db_path)?;
let now = Utc::now();
db.insert_session(&Session {
id: "lead".to_string(),
task: "lead task".to_string(),
agent_type: "claude".to_string(),
working_dir: repo_root.clone(),
state: SessionState::Running,
pid: Some(42),
worktree: None,
created_at: now - Duration::minutes(3),
updated_at: now - Duration::minutes(3),
metrics: SessionMetrics::default(),
})?;
db.insert_session(&Session {
id: "busy-worker".to_string(),
task: "existing work".to_string(),
agent_type: "claude".to_string(),
working_dir: repo_root.clone(),
state: SessionState::Running,
pid: Some(55),
worktree: None,
created_at: now - Duration::minutes(2),
updated_at: now - Duration::minutes(2),
metrics: SessionMetrics::default(),
})?;
db.send_message(
"lead",
"busy-worker",
"{\"task\":\"existing work\",\"context\":\"Delegated from lead\"}",
"task_handoff",
)?;
let (fake_runner, _) = write_fake_claude(tempdir.path())?;
let outcome = assign_session_in_dir_with_runner_program(
&db,
&cfg,
"lead",
"New delegated task",
"claude",
true,
&repo_root,
&fake_runner,
)
.await?;
assert_eq!(outcome.action, AssignmentAction::DeferredSaturated);
assert_eq!(outcome.session_id, "lead");
let busy_messages = db.list_messages_for_session("busy-worker", 10)?;
assert!(!busy_messages.iter().any(|message| {
message.msg_type == "task_handoff"
&& message.content.contains("New delegated task")
}));
Ok(())
}
#[tokio::test(flavor = "current_thread")] #[tokio::test(flavor = "current_thread")]
async fn drain_inbox_routes_unread_task_handoffs_and_marks_them_read() -> Result<()> { async fn drain_inbox_routes_unread_task_handoffs_and_marks_them_read() -> Result<()> {
let tempdir = TestDir::new("manager-drain-inbox")?; let tempdir = TestDir::new("manager-drain-inbox")?;
@@ -1909,6 +1983,73 @@ mod tests {
Ok(()) Ok(())
} }
#[tokio::test(flavor = "current_thread")]
async fn drain_inbox_leaves_saturated_handoffs_unread() -> Result<()> {
let tempdir = TestDir::new("manager-drain-inbox-defer")?;
let repo_root = tempdir.path().join("repo");
init_git_repo(&repo_root)?;
let mut cfg = build_config(tempdir.path());
cfg.max_parallel_sessions = 1;
let db = StateStore::open(&cfg.db_path)?;
let now = Utc::now();
db.insert_session(&Session {
id: "lead".to_string(),
task: "lead task".to_string(),
agent_type: "claude".to_string(),
working_dir: repo_root.clone(),
state: SessionState::Running,
pid: Some(42),
worktree: None,
created_at: now - Duration::minutes(3),
updated_at: now - Duration::minutes(3),
metrics: SessionMetrics::default(),
})?;
db.insert_session(&Session {
id: "busy-worker".to_string(),
task: "existing work".to_string(),
agent_type: "claude".to_string(),
working_dir: repo_root.clone(),
state: SessionState::Running,
pid: Some(55),
worktree: None,
created_at: now - Duration::minutes(2),
updated_at: now - Duration::minutes(2),
metrics: SessionMetrics::default(),
})?;
db.send_message(
"lead",
"busy-worker",
"{\"task\":\"existing work\",\"context\":\"Delegated from lead\"}",
"task_handoff",
)?;
db.send_message(
"planner",
"lead",
"{\"task\":\"Review auth changes\",\"context\":\"Inbound request\"}",
"task_handoff",
)?;
let outcomes = drain_inbox(&db, &cfg, "lead", "claude", true, 5).await?;
assert_eq!(outcomes.len(), 1);
assert_eq!(outcomes[0].task, "Review auth changes");
assert_eq!(outcomes[0].action, AssignmentAction::DeferredSaturated);
assert_eq!(outcomes[0].session_id, "lead");
let unread = db.unread_message_counts()?;
assert_eq!(unread.get("lead"), Some(&1));
assert_eq!(unread.get("busy-worker"), Some(&1));
let messages = db.list_messages_for_session("busy-worker", 10)?;
assert!(!messages.iter().any(|message| {
message.msg_type == "task_handoff"
&& message.content.contains("Review auth changes")
}));
Ok(())
}
#[tokio::test(flavor = "current_thread")] #[tokio::test(flavor = "current_thread")]
async fn auto_dispatch_backlog_routes_multiple_lead_inboxes() -> Result<()> { async fn auto_dispatch_backlog_routes_multiple_lead_inboxes() -> Result<()> {
let tempdir = TestDir::new("manager-auto-dispatch")?; let tempdir = TestDir::new("manager-auto-dispatch")?;

View File

@@ -817,7 +817,18 @@ impl Dashboard {
} }
}; };
let total_routed: usize = outcomes.iter().map(|outcome| outcome.routed.len()).sum(); let total_processed: usize = outcomes.iter().map(|outcome| outcome.routed.len()).sum();
let total_routed: usize = outcomes
.iter()
.map(|outcome| {
outcome
.routed
.iter()
.filter(|item| manager::assignment_action_routes_work(item.action))
.count()
})
.sum();
let total_deferred = total_processed.saturating_sub(total_routed);
let selected_session_id = self let selected_session_id = self
.sessions .sessions
.get(self.selected_session) .get(self.selected_session)
@@ -831,13 +842,15 @@ impl Dashboard {
self.sync_selected_lineage(); self.sync_selected_lineage();
self.refresh_logs(); self.refresh_logs();
if total_routed == 0 { if total_processed == 0 {
self.set_operator_note("no unread handoff backlog found".to_string()); self.set_operator_note("no unread handoff backlog found".to_string());
} else { } else {
self.set_operator_note(format!( self.set_operator_note(format!(
"auto-dispatched {} handoff(s) across {} lead session(s)", "auto-dispatch processed {} handoff(s) across {} lead session(s) ({} routed, {} deferred)",
total_processed,
outcomes.len(),
total_routed, total_routed,
outcomes.len() total_deferred
)); ));
} }
} }
@@ -908,11 +921,23 @@ impl Dashboard {
return; return;
} }
}; };
let total_routed: usize = outcome let total_processed: usize = outcome
.dispatched .dispatched
.iter() .iter()
.map(|dispatch| dispatch.routed.len()) .map(|dispatch| dispatch.routed.len())
.sum(); .sum();
let total_routed: usize = outcome
.dispatched
.iter()
.map(|dispatch| {
dispatch
.routed
.iter()
.filter(|item| manager::assignment_action_routes_work(item.action))
.count()
})
.sum();
let total_deferred = total_processed.saturating_sub(total_routed);
let total_rerouted: usize = outcome let total_rerouted: usize = outcome
.rebalanced .rebalanced
.iter() .iter()
@@ -932,13 +957,15 @@ impl Dashboard {
self.sync_selected_lineage(); self.sync_selected_lineage();
self.refresh_logs(); self.refresh_logs();
if total_routed == 0 && total_rerouted == 0 && outcome.remaining_backlog_sessions == 0 { if total_processed == 0 && total_rerouted == 0 && outcome.remaining_backlog_sessions == 0 {
self.set_operator_note("backlog already clear".to_string()); self.set_operator_note("backlog already clear".to_string());
} else { } else {
self.set_operator_note(format!( self.set_operator_note(format!(
"coordinated backlog: dispatched {} across {} lead(s), rebalanced {} across {} lead(s), remaining {} across {} session(s) [{} absorbable, {} saturated]", "coordinated backlog: processed {} across {} lead(s) ({} routed, {} deferred), rebalanced {} across {} lead(s), remaining {} across {} session(s) [{} absorbable, {} saturated]",
total_routed, total_processed,
outcome.dispatched.len(), outcome.dispatched.len(),
total_routed,
total_deferred,
total_rerouted, total_rerouted,
outcome.rebalanced.len(), outcome.rebalanced.len(),
outcome.remaining_backlog_messages, outcome.remaining_backlog_messages,
@@ -1940,6 +1967,7 @@ fn assignment_action_label(action: manager::AssignmentAction) -> &'static str {
manager::AssignmentAction::Spawned => "spawned", manager::AssignmentAction::Spawned => "spawned",
manager::AssignmentAction::ReusedIdle => "reused idle", manager::AssignmentAction::ReusedIdle => "reused idle",
manager::AssignmentAction::ReusedActive => "reused active", manager::AssignmentAction::ReusedActive => "reused active",
manager::AssignmentAction::DeferredSaturated => "deferred saturated",
} }
} }