Add dashboard health restart action (#38)
This commit is contained in:
@@ -1019,6 +1019,162 @@ describe('web dashboard server handler', () => {
|
||||
expect(noQueue.status).toBe(503);
|
||||
});
|
||||
|
||||
it('restarts the service stack through the health action endpoint', async () => {
|
||||
let restartCalls = 0;
|
||||
const handler = createWebDashboardHandler({
|
||||
readStatusSnapshots: () => [],
|
||||
getTasks: () => [],
|
||||
getPairedTasks: () => [],
|
||||
restartServiceStack: () => {
|
||||
restartCalls += 1;
|
||||
return ['ejclaw'];
|
||||
},
|
||||
now: () => '2026-04-26T05:30:00.000Z',
|
||||
});
|
||||
const restart = () =>
|
||||
handler(
|
||||
new Request('http://localhost/api/services/stack/actions', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
action: 'restart',
|
||||
requestId: 'stack-restart-1',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await restart();
|
||||
expect(response.status).toBe(200);
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
ok: true,
|
||||
restart: {
|
||||
id: 'web-restart-stack-restart-1',
|
||||
target: 'stack',
|
||||
requestedAt: '2026-04-26T05:30:00.000Z',
|
||||
completedAt: '2026-04-26T05:30:00.000Z',
|
||||
status: 'success',
|
||||
services: ['ejclaw'],
|
||||
},
|
||||
});
|
||||
expect(restartCalls).toBe(1);
|
||||
|
||||
const duplicate = await restart();
|
||||
expect(duplicate.status).toBe(200);
|
||||
await expect(duplicate.json()).resolves.toMatchObject({
|
||||
ok: true,
|
||||
duplicate: true,
|
||||
restart: {
|
||||
id: 'web-restart-stack-restart-1',
|
||||
status: 'success',
|
||||
},
|
||||
});
|
||||
expect(restartCalls).toBe(1);
|
||||
|
||||
const overview = await handler(
|
||||
new Request('http://localhost/api/overview'),
|
||||
);
|
||||
expect(overview.status).toBe(200);
|
||||
await expect(overview.json()).resolves.toMatchObject({
|
||||
operations: {
|
||||
serviceRestarts: [
|
||||
{
|
||||
id: 'web-restart-stack-restart-1',
|
||||
status: 'success',
|
||||
services: ['ejclaw'],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('records failed service stack restarts', async () => {
|
||||
const handler = createWebDashboardHandler({
|
||||
readStatusSnapshots: () => [],
|
||||
getTasks: () => [],
|
||||
getPairedTasks: () => [],
|
||||
restartServiceStack: () => {
|
||||
throw new Error('systemctl failed');
|
||||
},
|
||||
now: () => '2026-04-26T05:35:00.000Z',
|
||||
});
|
||||
|
||||
const response = await handler(
|
||||
new Request('http://localhost/api/services/stack/actions', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
action: 'restart',
|
||||
requestId: 'stack-restart-fail',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(500);
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
error: 'systemctl failed',
|
||||
restart: {
|
||||
id: 'web-restart-stack-restart-fail',
|
||||
target: 'stack',
|
||||
status: 'failed',
|
||||
error: 'systemctl failed',
|
||||
},
|
||||
});
|
||||
|
||||
const overview = await handler(
|
||||
new Request('http://localhost/api/overview'),
|
||||
);
|
||||
await expect(overview.json()).resolves.toMatchObject({
|
||||
operations: {
|
||||
serviceRestarts: [
|
||||
{
|
||||
id: 'web-restart-stack-restart-fail',
|
||||
status: 'failed',
|
||||
error: 'systemctl failed',
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects invalid service restart requests', async () => {
|
||||
let restartCalls = 0;
|
||||
const handler = createWebDashboardHandler({
|
||||
readStatusSnapshots: () => [],
|
||||
getTasks: () => [],
|
||||
getPairedTasks: () => [],
|
||||
restartServiceStack: () => {
|
||||
restartCalls += 1;
|
||||
return ['ejclaw'];
|
||||
},
|
||||
});
|
||||
|
||||
const invalidAction = await handler(
|
||||
new Request('http://localhost/api/services/stack/actions', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ action: 'stop' }),
|
||||
}),
|
||||
);
|
||||
expect(invalidAction.status).toBe(400);
|
||||
|
||||
const invalidTarget = await handler(
|
||||
new Request('http://localhost/api/services/ejclaw/actions', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ action: 'restart' }),
|
||||
}),
|
||||
);
|
||||
expect(invalidTarget.status).toBe(400);
|
||||
|
||||
const wrongMethod = await handler(
|
||||
new Request('http://localhost/api/services/stack/actions', {
|
||||
method: 'GET',
|
||||
}),
|
||||
);
|
||||
expect(wrongMethod.status).toBe(405);
|
||||
expect(restartCalls).toBe(0);
|
||||
});
|
||||
|
||||
it('injects room messages and queues room work from the web dashboard', async () => {
|
||||
const messages: NewMessage[] = [];
|
||||
const metadata: Array<{
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { execFileSync } from 'child_process';
|
||||
|
||||
import { CronExpressionParser } from 'cron-parser';
|
||||
|
||||
@@ -39,7 +40,13 @@ import {
|
||||
|
||||
const DEFAULT_STATUS_MAX_AGE_MS = 10 * 60 * 1000;
|
||||
const ROOM_MESSAGE_ID_CACHE_LIMIT = 500;
|
||||
const SERVICE_RESTART_LOG_LIMIT = 20;
|
||||
const STACK_RESTART_UNIT_NAME = 'ejclaw-stack-restart.service';
|
||||
const WEB_TASK_PROMPT_MAX_LENGTH = 8000;
|
||||
const MANAGED_SERVICE_CALLER_FALLBACK_MESSAGE =
|
||||
'Stack restart unit is not installed yet. Run setup service from an external shell before retrying from a managed EJClaw service.';
|
||||
const UNIT_NOT_FOUND_PATTERN =
|
||||
/(Unit .* not found|Could not find the requested service|not-found)/i;
|
||||
|
||||
type PairedFollowUpTask = Pick<
|
||||
PairedTask,
|
||||
@@ -54,6 +61,16 @@ type WebPairedFollowUpScheduler = (args: {
|
||||
enqueue: () => void;
|
||||
}) => boolean;
|
||||
|
||||
export interface ServiceRestartRecord {
|
||||
id: string;
|
||||
target: 'stack';
|
||||
requestedAt: string;
|
||||
completedAt: string | null;
|
||||
status: 'running' | 'success' | 'failed';
|
||||
services: string[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface WebDashboardHandlerOptions {
|
||||
staticDir?: string;
|
||||
statusMaxAgeMs?: number;
|
||||
@@ -113,6 +130,7 @@ export interface WebDashboardHandlerOptions {
|
||||
hasMessage?: (chatJid: string, id: string) => boolean;
|
||||
enqueueMessageCheck?: (chatJid: string, groupFolder: string) => void;
|
||||
nudgeScheduler?: () => void;
|
||||
restartServiceStack?: () => string[];
|
||||
now?: () => string;
|
||||
}
|
||||
|
||||
@@ -194,6 +212,7 @@ function serveStaticFile(staticDir: string, pathname: string): Response {
|
||||
|
||||
type TaskAction = 'pause' | 'resume' | 'cancel';
|
||||
type InboxAction = 'run' | 'decline' | 'dismiss';
|
||||
type ServiceAction = 'restart';
|
||||
|
||||
interface InboxActionRequest {
|
||||
action: InboxAction;
|
||||
@@ -201,6 +220,11 @@ interface InboxActionRequest {
|
||||
lastOccurredAt: string | null;
|
||||
}
|
||||
|
||||
interface ServiceActionRequest {
|
||||
action: ServiceAction;
|
||||
requestId: string | null;
|
||||
}
|
||||
|
||||
function parseTaskPath(pathname: string): string | null {
|
||||
const match = pathname.match(/^\/api\/tasks\/([^/]+)$/);
|
||||
if (!match) return null;
|
||||
@@ -241,6 +265,16 @@ function parseRoomMessagePath(pathname: string): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
function parseServiceActionPath(pathname: string): string | null {
|
||||
const match = pathname.match(/^\/api\/services\/([^/]+)\/actions$/);
|
||||
if (!match) return null;
|
||||
try {
|
||||
return decodeURIComponent(match[1]);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function parsePairedInboxTarget(
|
||||
inboxId: string,
|
||||
): { taskId: string; status: PairedTask['status'] } | null {
|
||||
@@ -278,6 +312,10 @@ function isInboxAction(value: unknown): value is InboxAction {
|
||||
return value === 'run' || value === 'decline' || value === 'dismiss';
|
||||
}
|
||||
|
||||
function isServiceAction(value: unknown): value is ServiceAction {
|
||||
return value === 'restart';
|
||||
}
|
||||
|
||||
function isPairedTaskStatus(value: unknown): value is PairedTask['status'] {
|
||||
return (
|
||||
value === 'active' ||
|
||||
@@ -322,6 +360,24 @@ async function readInboxAction(
|
||||
}
|
||||
}
|
||||
|
||||
async function readServiceAction(
|
||||
request: Request,
|
||||
): Promise<ServiceActionRequest | null> {
|
||||
try {
|
||||
const body = (await request.json()) as {
|
||||
action?: unknown;
|
||||
requestId?: unknown;
|
||||
};
|
||||
if (!isServiceAction(body.action)) return null;
|
||||
return {
|
||||
action: body.action,
|
||||
requestId: sanitizeServiceActionRequestId(body.requestId),
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
interface ScheduledTaskMutationBody {
|
||||
roomJid?: unknown;
|
||||
groupFolder?: unknown;
|
||||
@@ -464,6 +520,20 @@ function makeWebRunId(prefix: string): string {
|
||||
return `web-${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
}
|
||||
|
||||
function sanitizeServiceActionRequestId(value: unknown): string | null {
|
||||
if (typeof value !== 'string') return null;
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) return null;
|
||||
const safe = trimmed.replace(/[^A-Za-z0-9._:-]/g, '-').slice(0, 120);
|
||||
return safe || null;
|
||||
}
|
||||
|
||||
function makeServiceRestartId(requestId: string | null): string {
|
||||
return requestId
|
||||
? `web-restart-${requestId}`
|
||||
: `web-restart-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
}
|
||||
|
||||
function sanitizeInboxActionRequestId(value: unknown): string | null {
|
||||
if (typeof value !== 'string') return null;
|
||||
const trimmed = value.trim();
|
||||
@@ -513,6 +583,58 @@ function pairedFollowUpIntentForStatus(
|
||||
return null;
|
||||
}
|
||||
|
||||
function isRoot(): boolean {
|
||||
return typeof process.getuid === 'function' && process.getuid() === 0;
|
||||
}
|
||||
|
||||
function restartEjclawStackServices(): string[] {
|
||||
if (process.platform !== 'linux') {
|
||||
throw new Error('Service restart only supports Linux systemd services');
|
||||
}
|
||||
|
||||
const services = ['ejclaw'];
|
||||
const systemctlArgs = isRoot() ? [] : ['--user'];
|
||||
|
||||
try {
|
||||
execFileSync(
|
||||
'systemctl',
|
||||
[...systemctlArgs, 'start', '--wait', STACK_RESTART_UNIT_NAME],
|
||||
{ stdio: 'ignore' },
|
||||
);
|
||||
} catch (error) {
|
||||
const stderr =
|
||||
error && typeof error === 'object' && 'stderr' in error
|
||||
? String((error as { stderr?: unknown }).stderr || '')
|
||||
: '';
|
||||
const stdout =
|
||||
error && typeof error === 'object' && 'stdout' in error
|
||||
? String((error as { stdout?: unknown }).stdout || '')
|
||||
: '';
|
||||
const message =
|
||||
error instanceof Error ? error.message : `${stdout}\n${stderr}`.trim();
|
||||
const combined = `${message}\n${stdout}\n${stderr}`;
|
||||
if (!UNIT_NOT_FOUND_PATTERN.test(combined)) {
|
||||
throw error;
|
||||
}
|
||||
if (process.env.SERVICE_ID) {
|
||||
throw new Error(MANAGED_SERVICE_CALLER_FALLBACK_MESSAGE);
|
||||
}
|
||||
|
||||
execFileSync('systemctl', [...systemctlArgs, 'restart', ...services], {
|
||||
stdio: 'ignore',
|
||||
});
|
||||
for (const service of services) {
|
||||
execFileSync(
|
||||
'systemctl',
|
||||
[...systemctlArgs, 'is-active', '--quiet', service],
|
||||
{ stdio: 'ignore' },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
export function createWebDashboardHandler(
|
||||
opts: WebDashboardHandlerOptions = {},
|
||||
): (request: Request) => Response | Promise<Response> {
|
||||
@@ -534,10 +656,14 @@ export function createWebDashboardHandler(
|
||||
const messageExists = opts.hasMessage ?? hasMessage;
|
||||
const enqueueMessageCheck = opts.enqueueMessageCheck;
|
||||
const nudgeScheduler = opts.nudgeScheduler;
|
||||
const restartServiceStack =
|
||||
opts.restartServiceStack ?? restartEjclawStackServices;
|
||||
const statusMaxAgeMs = opts.statusMaxAgeMs ?? DEFAULT_STATUS_MAX_AGE_MS;
|
||||
const seenRoomMessageIds: string[] = [];
|
||||
const seenRoomMessageIdSet = new Set<string>();
|
||||
const dismissedInboxKeys = new Set<string>();
|
||||
const recentServiceRestarts: ServiceRestartRecord[] = [];
|
||||
const activeServiceRestartTargets = new Set<string>();
|
||||
|
||||
function rememberRoomMessageId(id: string): boolean {
|
||||
if (seenRoomMessageIdSet.has(id)) return false;
|
||||
@@ -560,12 +686,20 @@ export function createWebDashboardHandler(
|
||||
);
|
||||
}
|
||||
|
||||
function rememberServiceRestart(record: ServiceRestartRecord): void {
|
||||
recentServiceRestarts.unshift(record);
|
||||
if (recentServiceRestarts.length > SERVICE_RESTART_LOG_LIMIT) {
|
||||
recentServiceRestarts.length = SERVICE_RESTART_LOG_LIMIT;
|
||||
}
|
||||
}
|
||||
|
||||
return async (request: Request): Promise<Response> => {
|
||||
const url = new URL(request.url);
|
||||
const actionTaskId = parseTaskActionPath(url.pathname);
|
||||
const taskId = parseTaskPath(url.pathname);
|
||||
const actionInboxId = parseInboxActionPath(url.pathname);
|
||||
const messageRoomJid = parseRoomMessagePath(url.pathname);
|
||||
const actionServiceId = parseServiceActionPath(url.pathname);
|
||||
|
||||
if (actionTaskId) {
|
||||
if (request.method !== 'POST') {
|
||||
@@ -815,6 +949,83 @@ export function createWebDashboardHandler(
|
||||
return jsonResponse({ ok: true, id, queued: true });
|
||||
}
|
||||
|
||||
if (actionServiceId) {
|
||||
if (request.method !== 'POST') {
|
||||
return jsonResponse({ error: 'Method not allowed' }, { status: 405 });
|
||||
}
|
||||
if (actionServiceId !== 'stack') {
|
||||
return jsonResponse(
|
||||
{ error: 'Unsupported service restart target' },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const serviceRequest = await readServiceAction(request);
|
||||
if (!serviceRequest) {
|
||||
return jsonResponse(
|
||||
{ error: 'Invalid service action' },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const id = makeServiceRestartId(serviceRequest.requestId);
|
||||
const previous = recentServiceRestarts.find((record) => record.id === id);
|
||||
if (serviceRequest.requestId && previous) {
|
||||
if (previous.status === 'failed') {
|
||||
return jsonResponse(
|
||||
{
|
||||
error: previous.error ?? 'Service restart failed',
|
||||
duplicate: true,
|
||||
restart: previous,
|
||||
},
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
return jsonResponse({
|
||||
ok: true,
|
||||
duplicate: true,
|
||||
restart: previous,
|
||||
});
|
||||
}
|
||||
|
||||
if (activeServiceRestartTargets.has(actionServiceId)) {
|
||||
return jsonResponse(
|
||||
{ error: 'Service restart is already running' },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const requestedAt = opts.now?.() ?? new Date().toISOString();
|
||||
const record: ServiceRestartRecord = {
|
||||
id,
|
||||
target: 'stack',
|
||||
requestedAt,
|
||||
completedAt: null,
|
||||
status: 'running',
|
||||
services: [],
|
||||
};
|
||||
rememberServiceRestart(record);
|
||||
activeServiceRestartTargets.add(actionServiceId);
|
||||
|
||||
try {
|
||||
const services = restartServiceStack();
|
||||
record.completedAt = opts.now?.() ?? new Date().toISOString();
|
||||
record.status = 'success';
|
||||
record.services = services;
|
||||
return jsonResponse({ ok: true, restart: record });
|
||||
} catch (error) {
|
||||
record.completedAt = opts.now?.() ?? new Date().toISOString();
|
||||
record.status = 'failed';
|
||||
record.error = error instanceof Error ? error.message : String(error);
|
||||
return jsonResponse(
|
||||
{ error: record.error, restart: record },
|
||||
{ status: 500 },
|
||||
);
|
||||
} finally {
|
||||
activeServiceRestartTargets.delete(actionServiceId);
|
||||
}
|
||||
}
|
||||
|
||||
if (url.pathname === '/api/tasks' && request.method === 'POST') {
|
||||
if (!loadRoomBindings) {
|
||||
return jsonResponse(
|
||||
@@ -994,6 +1205,9 @@ export function createWebDashboardHandler(
|
||||
});
|
||||
return jsonResponse({
|
||||
...overview,
|
||||
operations: {
|
||||
serviceRestarts: recentServiceRestarts,
|
||||
},
|
||||
inbox: overview.inbox.filter((item) => !isInboxItemDismissed(item)),
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user