feat: add ecc2 remote dispatch intake

This commit is contained in:
Affaan Mustafa
2026-04-10 09:21:30 -07:00
parent bbed46d3eb
commit 7809518612
6 changed files with 1098 additions and 6 deletions

View File

@@ -31,6 +31,10 @@ pub async fn run(db: StateStore, cfg: Config) -> Result<()> {
tracing::error!("Scheduled task dispatch pass failed: {e}");
}
if let Err(e) = maybe_run_remote_dispatch(&db, &cfg).await {
tracing::error!("Remote dispatch pass failed: {e}");
}
if let Err(e) = coordinate_backlog_cycle(&db, &cfg).await {
tracing::error!("Backlog coordination pass failed: {e}");
}
@@ -101,6 +105,25 @@ async fn maybe_run_due_schedules(db: &StateStore, cfg: &Config) -> Result<usize>
Ok(outcomes.len())
}
async fn maybe_run_remote_dispatch(db: &StateStore, cfg: &Config) -> Result<usize> {
let outcomes =
manager::run_remote_dispatch_requests(db, cfg, cfg.max_parallel_sessions).await?;
let routed = outcomes
.iter()
.filter(|outcome| {
matches!(
outcome.action,
manager::RemoteDispatchAction::SpawnedTopLevel
| manager::RemoteDispatchAction::Assigned(_)
)
})
.count();
if routed > 0 {
tracing::info!("Dispatched {} remote request(s)", routed);
}
Ok(routed)
}
async fn maybe_auto_dispatch(db: &StateStore, cfg: &Config) -> Result<usize> {
let summary = maybe_auto_dispatch_with_recorder(
cfg,

View File

@@ -17,7 +17,7 @@ use super::{
ScheduledTask, Session, SessionAgentProfile, SessionGrouping, SessionHarnessInfo,
SessionMetrics, SessionState,
};
use crate::comms::{self, MessageType};
use crate::comms::{self, MessageType, TaskPriority};
use crate::config::Config;
use crate::observability::{log_tool_call, ToolCallEvent, ToolLogEntry, ToolLogPage, ToolLogger};
use crate::worktree;
@@ -252,6 +252,64 @@ pub fn delete_scheduled_task(db: &StateStore, schedule_id: i64) -> Result<bool>
Ok(db.delete_scheduled_task(schedule_id)? > 0)
}
#[allow(clippy::too_many_arguments)]
pub fn create_remote_dispatch_request(
db: &StateStore,
cfg: &Config,
task: &str,
target_session_id: Option<&str>,
priority: TaskPriority,
agent_type: &str,
profile_name: Option<&str>,
use_worktree: bool,
grouping: SessionGrouping,
source: &str,
requester: Option<&str>,
) -> Result<super::RemoteDispatchRequest> {
let working_dir =
std::env::current_dir().context("Failed to resolve current working directory")?;
let project = grouping
.project
.as_deref()
.and_then(normalize_group_label)
.unwrap_or_else(|| default_project_label(&working_dir));
let task_group = grouping
.task_group
.as_deref()
.and_then(normalize_group_label)
.unwrap_or_else(|| default_task_group_label(task));
let agent_type = HarnessKind::canonical_agent_type(agent_type);
if let Some(profile_name) = profile_name {
cfg.resolve_agent_profile(profile_name)?;
}
if let Some(target_session_id) = target_session_id {
let _ = resolve_session(db, target_session_id)?;
}
db.insert_remote_dispatch_request(
target_session_id,
task,
priority,
&agent_type,
profile_name,
&working_dir,
&project,
&task_group,
use_worktree,
source,
requester,
)
}
pub fn list_remote_dispatch_requests(
db: &StateStore,
include_processed: bool,
limit: usize,
) -> Result<Vec<super::RemoteDispatchRequest>> {
db.list_remote_dispatch_requests(include_processed, limit)
}
pub async fn run_due_schedules(
db: &StateStore,
cfg: &Config,
@@ -262,6 +320,133 @@ pub async fn run_due_schedules(
run_due_schedules_with_runner_program(db, cfg, limit, &runner_program).await
}
pub async fn run_remote_dispatch_requests(
db: &StateStore,
cfg: &Config,
limit: usize,
) -> Result<Vec<RemoteDispatchOutcome>> {
let requests = db.list_pending_remote_dispatch_requests(limit)?;
let runner_program =
std::env::current_exe().context("Failed to resolve ECC executable path")?;
run_remote_dispatch_requests_with_runner_program(db, cfg, requests, &runner_program).await
}
async fn run_remote_dispatch_requests_with_runner_program(
db: &StateStore,
cfg: &Config,
requests: Vec<super::RemoteDispatchRequest>,
runner_program: &Path,
) -> Result<Vec<RemoteDispatchOutcome>> {
let mut outcomes = Vec::new();
for request in requests {
let grouping = SessionGrouping {
project: normalize_group_label(&request.project),
task_group: normalize_group_label(&request.task_group),
};
let outcome = if let Some(target_session_id) = request.target_session_id.as_deref() {
match assign_session_in_dir_with_runner_program(
db,
cfg,
target_session_id,
&request.task,
&request.agent_type,
request.use_worktree,
&request.working_dir,
&runner_program,
request.profile_name.as_deref(),
grouping,
)
.await
{
Ok(assignment) if assignment.action == AssignmentAction::DeferredSaturated => {
RemoteDispatchOutcome {
request_id: request.id,
task: request.task.clone(),
priority: request.priority,
target_session_id: request.target_session_id.clone(),
session_id: None,
action: RemoteDispatchAction::DeferredSaturated,
}
}
Ok(assignment) => {
db.record_remote_dispatch_success(
request.id,
Some(&assignment.session_id),
Some(assignment.action.label()),
)?;
RemoteDispatchOutcome {
request_id: request.id,
task: request.task.clone(),
priority: request.priority,
target_session_id: request.target_session_id.clone(),
session_id: Some(assignment.session_id),
action: RemoteDispatchAction::Assigned(assignment.action),
}
}
Err(error) => {
db.record_remote_dispatch_failure(request.id, &error.to_string())?;
RemoteDispatchOutcome {
request_id: request.id,
task: request.task.clone(),
priority: request.priority,
target_session_id: request.target_session_id.clone(),
session_id: None,
action: RemoteDispatchAction::Failed(error.to_string()),
}
}
}
} else {
match queue_session_in_dir_with_runner_program(
db,
cfg,
&request.task,
&request.agent_type,
request.use_worktree,
&request.working_dir,
&runner_program,
request.profile_name.as_deref(),
None,
grouping,
)
.await
{
Ok(session_id) => {
db.record_remote_dispatch_success(
request.id,
Some(&session_id),
Some("spawned_top_level"),
)?;
RemoteDispatchOutcome {
request_id: request.id,
task: request.task.clone(),
priority: request.priority,
target_session_id: None,
session_id: Some(session_id),
action: RemoteDispatchAction::SpawnedTopLevel,
}
}
Err(error) => {
db.record_remote_dispatch_failure(request.id, &error.to_string())?;
RemoteDispatchOutcome {
request_id: request.id,
task: request.task.clone(),
priority: request.priority,
target_session_id: None,
session_id: None,
action: RemoteDispatchAction::Failed(error.to_string()),
}
}
}
};
outcomes.push(outcome);
}
Ok(outcomes)
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TemplateLaunchStepOutcome {
pub step_name: String,
@@ -3076,6 +3261,25 @@ pub struct ScheduledRunOutcome {
pub next_run_at: chrono::DateTime<chrono::Utc>,
}
#[derive(Debug, Clone, Serialize)]
pub struct RemoteDispatchOutcome {
pub request_id: i64,
pub task: String,
pub priority: TaskPriority,
pub target_session_id: Option<String>,
pub session_id: Option<String>,
pub action: RemoteDispatchAction,
}
#[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "snake_case", tag = "type", content = "details")]
pub enum RemoteDispatchAction {
SpawnedTopLevel,
Assigned(AssignmentAction),
DeferredSaturated,
Failed(String),
}
pub struct RebalanceOutcome {
pub from_session_id: String,
pub message_id: i64,
@@ -3130,7 +3334,8 @@ pub enum CoordinationHealth {
EscalationRequired,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum AssignmentAction {
Spawned,
ReusedIdle,
@@ -3138,6 +3343,17 @@ pub enum AssignmentAction {
DeferredSaturated,
}
impl AssignmentAction {
fn label(self) -> &'static str {
match self {
Self::Spawned => "spawned",
Self::ReusedIdle => "reused_idle",
Self::ReusedActive => "reused_active",
Self::DeferredSaturated => "deferred_saturated",
}
}
}
pub fn preview_assignment_for_task(
db: &StateStore,
cfg: &Config,
@@ -4341,6 +4557,152 @@ mod tests {
Ok(())
}
#[tokio::test(flavor = "current_thread")]
async fn run_remote_dispatch_requests_prioritizes_critical_targeted_work() -> Result<()> {
let tempdir = TestDir::new("manager-run-remote-dispatch-priority")?;
let repo_root = tempdir.path().join("repo");
init_git_repo(&repo_root)?;
let cfg = build_config(tempdir.path());
let db = StateStore::open(&cfg.db_path)?;
let (fake_runner, _log_path) = write_fake_claude(tempdir.path())?;
let now = Utc::now();
db.insert_session(&Session {
id: "lead".to_string(),
task: "Lead orchestration".to_string(),
project: "repo".to_string(),
task_group: "Lead orchestration".to_string(),
agent_type: "claude".to_string(),
working_dir: repo_root.clone(),
state: SessionState::Running,
pid: None,
worktree: None,
created_at: now,
updated_at: now,
last_heartbeat_at: now,
metrics: SessionMetrics::default(),
})?;
let low = create_remote_dispatch_request(
&db,
&cfg,
"Low priority cleanup",
Some("lead"),
TaskPriority::Low,
"claude",
None,
true,
SessionGrouping::default(),
"cli",
None,
)?;
let critical = create_remote_dispatch_request(
&db,
&cfg,
"Critical production incident",
Some("lead"),
TaskPriority::Critical,
"claude",
None,
true,
SessionGrouping::default(),
"cli",
None,
)?;
let outcomes = run_remote_dispatch_requests_with_runner_program(
&db,
&cfg,
db.list_pending_remote_dispatch_requests(1)?,
&fake_runner,
)
.await?;
assert_eq!(outcomes.len(), 1);
assert_eq!(outcomes[0].request_id, critical.id);
assert!(matches!(
outcomes[0].action,
RemoteDispatchAction::Assigned(AssignmentAction::Spawned)
));
let low_request = db
.get_remote_dispatch_request(low.id)?
.context("low priority request should still exist")?;
assert_eq!(
low_request.status,
crate::session::RemoteDispatchStatus::Pending
);
let critical_request = db
.get_remote_dispatch_request(critical.id)?
.context("critical request should still exist")?;
assert_eq!(
critical_request.status,
crate::session::RemoteDispatchStatus::Dispatched
);
assert!(critical_request.result_session_id.is_some());
Ok(())
}
#[tokio::test(flavor = "current_thread")]
async fn run_remote_dispatch_requests_spawns_top_level_session_when_untargeted() -> Result<()> {
let tempdir = TestDir::new("manager-run-remote-dispatch-top-level")?;
let repo_root = tempdir.path().join("repo");
init_git_repo(&repo_root)?;
let cfg = build_config(tempdir.path());
let db = StateStore::open(&cfg.db_path)?;
let (fake_runner, _log_path) = write_fake_claude(tempdir.path())?;
let request = db.insert_remote_dispatch_request(
None,
"Remote phone triage",
TaskPriority::High,
"claude",
None,
&repo_root,
"ecc-core",
"phone dispatch",
true,
"http",
Some("127.0.0.1"),
)?;
let outcomes = run_remote_dispatch_requests_with_runner_program(
&db,
&cfg,
db.list_pending_remote_dispatch_requests(10)?,
&fake_runner,
)
.await?;
assert_eq!(outcomes.len(), 1);
assert_eq!(outcomes[0].request_id, request.id);
assert!(matches!(
outcomes[0].action,
RemoteDispatchAction::SpawnedTopLevel
));
let request = db
.get_remote_dispatch_request(request.id)?
.context("remote request should still exist")?;
assert_eq!(
request.status,
crate::session::RemoteDispatchStatus::Dispatched
);
let session_id = request
.result_session_id
.clone()
.context("spawned top-level request should record a session id")?;
let session = db
.get_session(&session_id)?
.context("spawned session should exist")?;
assert_eq!(session.project, "ecc-core");
assert_eq!(session.task_group, "phone dispatch");
Ok(())
}
#[tokio::test(flavor = "current_thread")]
async fn stop_session_kills_process_and_optionally_cleans_worktree() -> Result<()> {
let tempdir = TestDir::new("manager-stop-session")?;

View File

@@ -395,6 +395,57 @@ pub struct ScheduledTask {
pub updated_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RemoteDispatchRequest {
pub id: i64,
pub target_session_id: Option<String>,
pub task: String,
pub priority: crate::comms::TaskPriority,
pub agent_type: String,
pub profile_name: Option<String>,
pub working_dir: PathBuf,
pub project: String,
pub task_group: String,
pub use_worktree: bool,
pub source: String,
pub requester: Option<String>,
pub status: RemoteDispatchStatus,
pub result_session_id: Option<String>,
pub result_action: Option<String>,
pub error: Option<String>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub dispatched_at: Option<DateTime<Utc>>,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum RemoteDispatchStatus {
Pending,
Dispatched,
Failed,
}
impl fmt::Display for RemoteDispatchStatus {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Pending => write!(f, "pending"),
Self::Dispatched => write!(f, "dispatched"),
Self::Failed => write!(f, "failed"),
}
}
}
impl RemoteDispatchStatus {
pub fn from_db_value(value: &str) -> Self {
match value {
"dispatched" => Self::Dispatched,
"failed" => Self::Failed,
_ => Self::Pending,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct FileActivityEntry {
pub session_id: String,

View File

@@ -18,8 +18,9 @@ use super::{
ContextGraphCompactionStats, ContextGraphEntity, ContextGraphEntityDetail,
ContextGraphObservation, ContextGraphRecallEntry, ContextGraphRelation, ContextGraphSyncStats,
ContextObservationPriority, DecisionLogEntry, FileActivityAction, FileActivityEntry,
HarnessKind, ScheduledTask, Session, SessionAgentProfile, SessionHarnessInfo, SessionMessage,
SessionMetrics, SessionState, WorktreeInfo,
HarnessKind, RemoteDispatchRequest, RemoteDispatchStatus, ScheduledTask, Session,
SessionAgentProfile, SessionHarnessInfo, SessionMessage, SessionMetrics, SessionState,
WorktreeInfo,
};
pub struct StateStore {
@@ -315,6 +316,28 @@ impl StateStore {
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS remote_dispatch_requests (
id INTEGER PRIMARY KEY AUTOINCREMENT,
target_session_id TEXT REFERENCES sessions(id) ON DELETE SET NULL,
task TEXT NOT NULL,
priority INTEGER NOT NULL DEFAULT 1,
agent_type TEXT NOT NULL,
profile_name TEXT,
working_dir TEXT NOT NULL,
project TEXT NOT NULL DEFAULT '',
task_group TEXT NOT NULL DEFAULT '',
use_worktree INTEGER NOT NULL DEFAULT 1,
source TEXT NOT NULL DEFAULT '',
requester TEXT,
status TEXT NOT NULL DEFAULT 'pending',
result_session_id TEXT,
result_action TEXT,
error TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
dispatched_at TEXT
);
CREATE TABLE IF NOT EXISTS conflict_incidents (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conflict_key TEXT NOT NULL UNIQUE,
@@ -377,6 +400,8 @@ impl StateStore {
ON conflict_incidents(first_session_id, second_session_id, resolved_at, updated_at);
CREATE INDEX IF NOT EXISTS idx_pending_worktree_queue_requested_at
ON pending_worktree_queue(requested_at, session_id);
CREATE INDEX IF NOT EXISTS idx_remote_dispatch_requests_status_priority
ON remote_dispatch_requests(status, priority DESC, created_at, id);
INSERT OR IGNORE INTO daemon_activity (id) VALUES (1);
",
@@ -1164,6 +1189,153 @@ impl StateStore {
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub fn insert_remote_dispatch_request(
&self,
target_session_id: Option<&str>,
task: &str,
priority: crate::comms::TaskPriority,
agent_type: &str,
profile_name: Option<&str>,
working_dir: &Path,
project: &str,
task_group: &str,
use_worktree: bool,
source: &str,
requester: Option<&str>,
) -> Result<RemoteDispatchRequest> {
let now = chrono::Utc::now();
self.conn.execute(
"INSERT INTO remote_dispatch_requests (
target_session_id,
task,
priority,
agent_type,
profile_name,
working_dir,
project,
task_group,
use_worktree,
source,
requester,
status,
created_at,
updated_at
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, 'pending', ?12, ?13)",
rusqlite::params![
target_session_id,
task,
task_priority_db_value(priority),
agent_type,
profile_name,
working_dir.display().to_string(),
project,
task_group,
if use_worktree { 1_i64 } else { 0_i64 },
source,
requester,
now.to_rfc3339(),
now.to_rfc3339(),
],
)?;
let id = self.conn.last_insert_rowid();
self.get_remote_dispatch_request(id)?.ok_or_else(|| {
anyhow::anyhow!("Remote dispatch request {id} was not found after insert")
})
}
pub fn list_remote_dispatch_requests(
&self,
include_processed: bool,
limit: usize,
) -> Result<Vec<RemoteDispatchRequest>> {
let sql = if include_processed {
"SELECT id, target_session_id, task, priority, agent_type, profile_name, working_dir,
project, task_group, use_worktree, source, requester, status,
result_session_id, result_action, error, created_at, updated_at, dispatched_at
FROM remote_dispatch_requests
ORDER BY CASE status WHEN 'pending' THEN 0 WHEN 'failed' THEN 1 ELSE 2 END ASC,
priority DESC, created_at ASC, id ASC
LIMIT ?1"
} else {
"SELECT id, target_session_id, task, priority, agent_type, profile_name, working_dir,
project, task_group, use_worktree, source, requester, status,
result_session_id, result_action, error, created_at, updated_at, dispatched_at
FROM remote_dispatch_requests
WHERE status = 'pending'
ORDER BY priority DESC, created_at ASC, id ASC
LIMIT ?1"
};
let mut stmt = self.conn.prepare(sql)?;
let rows = stmt.query_map([limit as i64], map_remote_dispatch_request)?;
rows.collect::<Result<Vec<_>, _>>().map_err(Into::into)
}
pub fn list_pending_remote_dispatch_requests(
&self,
limit: usize,
) -> Result<Vec<RemoteDispatchRequest>> {
self.list_remote_dispatch_requests(false, limit)
}
pub fn get_remote_dispatch_request(
&self,
request_id: i64,
) -> Result<Option<RemoteDispatchRequest>> {
self.conn
.query_row(
"SELECT id, target_session_id, task, priority, agent_type, profile_name, working_dir,
project, task_group, use_worktree, source, requester, status,
result_session_id, result_action, error, created_at, updated_at, dispatched_at
FROM remote_dispatch_requests
WHERE id = ?1",
[request_id],
map_remote_dispatch_request,
)
.optional()
.map_err(Into::into)
}
pub fn record_remote_dispatch_success(
&self,
request_id: i64,
result_session_id: Option<&str>,
result_action: Option<&str>,
) -> Result<()> {
let now = chrono::Utc::now();
self.conn.execute(
"UPDATE remote_dispatch_requests
SET status = 'dispatched',
result_session_id = ?2,
result_action = ?3,
error = NULL,
dispatched_at = ?4,
updated_at = ?4
WHERE id = ?1",
rusqlite::params![
request_id,
result_session_id,
result_action,
now.to_rfc3339()
],
)?;
Ok(())
}
pub fn record_remote_dispatch_failure(&self, request_id: i64, error: &str) -> Result<()> {
let now = chrono::Utc::now();
self.conn.execute(
"UPDATE remote_dispatch_requests
SET status = 'failed',
error = ?2,
updated_at = ?3
WHERE id = ?1",
rusqlite::params![request_id, error, now.to_rfc3339()],
)?;
Ok(())
}
pub fn update_metrics(&self, session_id: &str, metrics: &SessionMetrics) -> Result<()> {
self.conn.execute(
"UPDATE sessions
@@ -3727,6 +3899,36 @@ fn map_scheduled_task(row: &rusqlite::Row<'_>) -> rusqlite::Result<ScheduledTask
})
}
fn map_remote_dispatch_request(row: &rusqlite::Row<'_>) -> rusqlite::Result<RemoteDispatchRequest> {
let created_at = parse_store_timestamp(row.get::<_, String>(16)?, 16)?;
let updated_at = parse_store_timestamp(row.get::<_, String>(17)?, 17)?;
let dispatched_at = row
.get::<_, Option<String>>(18)?
.map(|value| parse_store_timestamp(value, 18))
.transpose()?;
Ok(RemoteDispatchRequest {
id: row.get(0)?,
target_session_id: normalize_optional_string(row.get(1)?),
task: row.get(2)?,
priority: task_priority_from_db_value(row.get::<_, i64>(3)?),
agent_type: row.get(4)?,
profile_name: normalize_optional_string(row.get(5)?),
working_dir: PathBuf::from(row.get::<_, String>(6)?),
project: row.get(7)?,
task_group: row.get(8)?,
use_worktree: row.get::<_, i64>(9)? != 0,
source: row.get(10)?,
requester: normalize_optional_string(row.get(11)?),
status: RemoteDispatchStatus::from_db_value(&row.get::<_, String>(12)?),
result_session_id: normalize_optional_string(row.get(13)?),
result_action: normalize_optional_string(row.get(14)?),
error: normalize_optional_string(row.get(15)?),
created_at,
updated_at,
dispatched_at,
})
}
fn parse_timestamp_column(
value: String,
index: usize,
@@ -3769,6 +3971,24 @@ fn default_input_params_json() -> String {
"{}".to_string()
}
fn task_priority_db_value(priority: crate::comms::TaskPriority) -> i64 {
match priority {
crate::comms::TaskPriority::Low => 0,
crate::comms::TaskPriority::Normal => 1,
crate::comms::TaskPriority::High => 2,
crate::comms::TaskPriority::Critical => 3,
}
}
fn task_priority_from_db_value(value: i64) -> crate::comms::TaskPriority {
match value {
0 => crate::comms::TaskPriority::Low,
2 => crate::comms::TaskPriority::High,
3 => crate::comms::TaskPriority::Critical,
_ => crate::comms::TaskPriority::Normal,
}
}
fn infer_file_activity_action(tool_name: &str) -> FileActivityAction {
let tool_name = tool_name.trim().to_ascii_lowercase();
if tool_name.contains("read") {