feat: implement auto-sync scheduler for LoTW and DCL
Add automatic synchronization scheduler that allows users to configure periodic sync intervals for LoTW and DCL via the settings page. Features: - Users can enable/disable auto-sync per service (LoTW/DCL) - Configurable sync intervals (1-720 hours) - Settings page UI for managing auto-sync preferences - Dashboard shows upcoming scheduled auto-sync jobs - Scheduler runs every minute, triggers syncs when due - Survives server restarts via database persistence - Graceful shutdown support (SIGINT/SIGTERM) Backend: - New autoSyncSettings table with user preferences - auto-sync.service.js for CRUD operations and scheduling logic - scheduler.service.js for periodic tick processing - API endpoints: GET/PUT /auto-sync/settings, GET /auto-sync/scheduler/status Frontend: - Auto-sync settings section in settings page - Upcoming auto-sync section on dashboard with scheduled job cards - Purple-themed UI for scheduled jobs with countdown animation Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
340
src/backend/services/auto-sync.service.js
Normal file
340
src/backend/services/auto-sync.service.js
Normal file
@@ -0,0 +1,340 @@
|
||||
import { db, logger } from '../config.js';
|
||||
import { autoSyncSettings, users } from '../db/schema/index.js';
|
||||
import { eq, and, lte, or } from 'drizzle-orm';
|
||||
|
||||
/**
|
||||
* Auto-Sync Settings Service
|
||||
* Manages user preferences for automatic DCL and LoTW synchronization
|
||||
*/
|
||||
|
||||
// Validation constants
|
||||
export const MIN_INTERVAL_HOURS = 1;
|
||||
export const MAX_INTERVAL_HOURS = 720; // 30 days
|
||||
export const DEFAULT_INTERVAL_HOURS = 24;
|
||||
|
||||
/**
|
||||
* Get auto-sync settings for a user
|
||||
* Creates default settings if they don't exist
|
||||
*
|
||||
* @param {number} userId - User ID
|
||||
* @returns {Promise<Object>} Auto-sync settings
|
||||
*/
|
||||
export async function getAutoSyncSettings(userId) {
|
||||
try {
|
||||
let [settings] = await db
|
||||
.select()
|
||||
.from(autoSyncSettings)
|
||||
.where(eq(autoSyncSettings.userId, userId));
|
||||
|
||||
// Create default settings if they don't exist
|
||||
if (!settings) {
|
||||
logger.debug('Creating default auto-sync settings for user', { userId });
|
||||
[settings] = await db
|
||||
.insert(autoSyncSettings)
|
||||
.values({
|
||||
userId,
|
||||
lotwEnabled: false,
|
||||
lotwIntervalHours: DEFAULT_INTERVAL_HOURS,
|
||||
dclEnabled: false,
|
||||
dclIntervalHours: DEFAULT_INTERVAL_HOURS,
|
||||
})
|
||||
.returning();
|
||||
}
|
||||
|
||||
return {
|
||||
lotwEnabled: settings.lotwEnabled,
|
||||
lotwIntervalHours: settings.lotwIntervalHours,
|
||||
lotwLastSyncAt: settings.lotwLastSyncAt,
|
||||
lotwNextSyncAt: settings.lotwNextSyncAt,
|
||||
dclEnabled: settings.dclEnabled,
|
||||
dclIntervalHours: settings.dclIntervalHours,
|
||||
dclLastSyncAt: settings.dclLastSyncAt,
|
||||
dclNextSyncAt: settings.dclNextSyncAt,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get auto-sync settings', { error: error.message, userId });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate interval hours
|
||||
* @param {number} hours - Interval hours to validate
|
||||
* @returns {Object} Validation result
|
||||
*/
|
||||
function validateIntervalHours(hours) {
|
||||
if (typeof hours !== 'number' || isNaN(hours)) {
|
||||
return { valid: false, error: 'Interval must be a number' };
|
||||
}
|
||||
if (!Number.isInteger(hours)) {
|
||||
return { valid: false, error: 'Interval must be a whole number of hours' };
|
||||
}
|
||||
if (hours < MIN_INTERVAL_HOURS) {
|
||||
return { valid: false, error: `Interval must be at least ${MIN_INTERVAL_HOURS} hour` };
|
||||
}
|
||||
if (hours > MAX_INTERVAL_HOURS) {
|
||||
return { valid: false, error: `Interval must be at most ${MAX_INTERVAL_HOURS} hours (30 days)` };
|
||||
}
|
||||
return { valid: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate next sync time based on interval
|
||||
* @param {number} intervalHours - Interval in hours
|
||||
* @returns {Date} Next sync time
|
||||
*/
|
||||
function calculateNextSyncTime(intervalHours) {
|
||||
const nextSync = new Date();
|
||||
nextSync.setHours(nextSync.getHours() + intervalHours);
|
||||
return nextSync;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update auto-sync settings for a user
|
||||
*
|
||||
* @param {number} userId - User ID
|
||||
* @param {Object} settings - Settings to update
|
||||
* @returns {Promise<Object>} Updated settings
|
||||
*/
|
||||
export async function updateAutoSyncSettings(userId, settings) {
|
||||
try {
|
||||
// Get current settings
|
||||
let [currentSettings] = await db
|
||||
.select()
|
||||
.from(autoSyncSettings)
|
||||
.where(eq(autoSyncSettings.userId, userId));
|
||||
|
||||
// Create default settings if they don't exist
|
||||
if (!currentSettings) {
|
||||
[currentSettings] = await db
|
||||
.insert(autoSyncSettings)
|
||||
.values({
|
||||
userId,
|
||||
lotwEnabled: false,
|
||||
lotwIntervalHours: DEFAULT_INTERVAL_HOURS,
|
||||
dclEnabled: false,
|
||||
dclIntervalHours: DEFAULT_INTERVAL_HOURS,
|
||||
})
|
||||
.returning();
|
||||
}
|
||||
|
||||
// Prepare update data
|
||||
const updateData = {
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
|
||||
// Validate and update LoTW settings
|
||||
if (settings.lotwEnabled !== undefined) {
|
||||
if (typeof settings.lotwEnabled !== 'boolean') {
|
||||
throw new Error('lotwEnabled must be a boolean');
|
||||
}
|
||||
updateData.lotwEnabled = settings.lotwEnabled;
|
||||
|
||||
// If enabling for the first time or interval changed, set next sync time
|
||||
if (settings.lotwEnabled && (!currentSettings.lotwEnabled || settings.lotwIntervalHours)) {
|
||||
const intervalHours = settings.lotwIntervalHours || currentSettings.lotwIntervalHours;
|
||||
const validation = validateIntervalHours(intervalHours);
|
||||
if (!validation.valid) {
|
||||
throw new Error(`LoTW interval: ${validation.error}`);
|
||||
}
|
||||
updateData.lotwNextSyncAt = calculateNextSyncTime(intervalHours);
|
||||
} else if (!settings.lotwEnabled) {
|
||||
// Clear next sync when disabling
|
||||
updateData.lotwNextSyncAt = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (settings.lotwIntervalHours !== undefined) {
|
||||
const validation = validateIntervalHours(settings.lotwIntervalHours);
|
||||
if (!validation.valid) {
|
||||
throw new Error(`LoTW interval: ${validation.error}`);
|
||||
}
|
||||
updateData.lotwIntervalHours = settings.lotwIntervalHours;
|
||||
|
||||
// Update next sync time if LoTW is enabled
|
||||
if (currentSettings.lotwEnabled || settings.lotwEnabled) {
|
||||
updateData.lotwNextSyncAt = calculateNextSyncTime(settings.lotwIntervalHours);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate and update DCL settings
|
||||
if (settings.dclEnabled !== undefined) {
|
||||
if (typeof settings.dclEnabled !== 'boolean') {
|
||||
throw new Error('dclEnabled must be a boolean');
|
||||
}
|
||||
updateData.dclEnabled = settings.dclEnabled;
|
||||
|
||||
// If enabling for the first time or interval changed, set next sync time
|
||||
if (settings.dclEnabled && (!currentSettings.dclEnabled || settings.dclIntervalHours)) {
|
||||
const intervalHours = settings.dclIntervalHours || currentSettings.dclIntervalHours;
|
||||
const validation = validateIntervalHours(intervalHours);
|
||||
if (!validation.valid) {
|
||||
throw new Error(`DCL interval: ${validation.error}`);
|
||||
}
|
||||
updateData.dclNextSyncAt = calculateNextSyncTime(intervalHours);
|
||||
} else if (!settings.dclEnabled) {
|
||||
// Clear next sync when disabling
|
||||
updateData.dclNextSyncAt = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (settings.dclIntervalHours !== undefined) {
|
||||
const validation = validateIntervalHours(settings.dclIntervalHours);
|
||||
if (!validation.valid) {
|
||||
throw new Error(`DCL interval: ${validation.error}`);
|
||||
}
|
||||
updateData.dclIntervalHours = settings.dclIntervalHours;
|
||||
|
||||
// Update next sync time if DCL is enabled
|
||||
if (currentSettings.dclEnabled || settings.dclEnabled) {
|
||||
updateData.dclNextSyncAt = calculateNextSyncTime(settings.dclIntervalHours);
|
||||
}
|
||||
}
|
||||
|
||||
// Update settings in database
|
||||
const [updated] = await db
|
||||
.update(autoSyncSettings)
|
||||
.set(updateData)
|
||||
.where(eq(autoSyncSettings.userId, userId))
|
||||
.returning();
|
||||
|
||||
logger.info('Updated auto-sync settings', {
|
||||
userId,
|
||||
lotwEnabled: updated.lotwEnabled,
|
||||
lotwIntervalHours: updated.lotwIntervalHours,
|
||||
dclEnabled: updated.dclEnabled,
|
||||
dclIntervalHours: updated.dclIntervalHours,
|
||||
});
|
||||
|
||||
return {
|
||||
lotwEnabled: updated.lotwEnabled,
|
||||
lotwIntervalHours: updated.lotwIntervalHours,
|
||||
lotwLastSyncAt: updated.lotwLastSyncAt,
|
||||
lotwNextSyncAt: updated.lotwNextSyncAt,
|
||||
dclEnabled: updated.dclEnabled,
|
||||
dclIntervalHours: updated.dclIntervalHours,
|
||||
dclLastSyncAt: updated.dclLastSyncAt,
|
||||
dclNextSyncAt: updated.dclNextSyncAt,
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to update auto-sync settings', { error: error.message, userId });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get users with pending syncs for a specific service
|
||||
*
|
||||
* @param {string} service - 'lotw' or 'dcl'
|
||||
* @returns {Promise<Array>} List of users with pending syncs
|
||||
*/
|
||||
export async function getPendingSyncUsers(service) {
|
||||
try {
|
||||
if (service !== 'lotw' && service !== 'dcl') {
|
||||
throw new Error('Service must be "lotw" or "dcl"');
|
||||
}
|
||||
|
||||
const enabledField = service === 'lotw' ? autoSyncSettings.lotwEnabled : autoSyncSettings.dclEnabled;
|
||||
const nextSyncField = service === 'lotw' ? autoSyncSettings.lotwNextSyncAt : autoSyncSettings.dclNextSyncAt;
|
||||
const credentialField = service === 'lotw' ? users.lotwUsername : users.dclApiKey;
|
||||
|
||||
const now = new Date();
|
||||
|
||||
// Get users with auto-sync enabled and next sync time in the past
|
||||
const results = await db
|
||||
.select({
|
||||
userId: autoSyncSettings.userId,
|
||||
lotwEnabled: autoSyncSettings.lotwEnabled,
|
||||
lotwIntervalHours: autoSyncSettings.lotwIntervalHours,
|
||||
lotwNextSyncAt: autoSyncSettings.lotwNextSyncAt,
|
||||
dclEnabled: autoSyncSettings.dclEnabled,
|
||||
dclIntervalHours: autoSyncSettings.dclIntervalHours,
|
||||
dclNextSyncAt: autoSyncSettings.dclNextSyncAt,
|
||||
hasCredentials: credentialField, // Just check if field exists (not null/empty)
|
||||
})
|
||||
.from(autoSyncSettings)
|
||||
.innerJoin(users, eq(autoSyncSettings.userId, users.id))
|
||||
.where(
|
||||
and(
|
||||
eq(enabledField, true),
|
||||
lte(nextSyncField, now)
|
||||
)
|
||||
);
|
||||
|
||||
// Filter out users without credentials
|
||||
const withCredentials = results.filter(r => r.hasCredentials);
|
||||
|
||||
logger.debug('Found pending sync users', {
|
||||
service,
|
||||
total: results.length,
|
||||
withCredentials: withCredentials.length,
|
||||
});
|
||||
|
||||
return withCredentials;
|
||||
} catch (error) {
|
||||
logger.error('Failed to get pending sync users', { error: error.message, service });
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update sync timestamps after a successful sync
|
||||
*
|
||||
* @param {number} userId - User ID
|
||||
* @param {string} service - 'lotw' or 'dcl'
|
||||
* @param {Date} lastSyncDate - Date of last sync
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export async function updateSyncTimestamps(userId, service, lastSyncDate) {
|
||||
try {
|
||||
if (service !== 'lotw' && service !== 'dcl') {
|
||||
throw new Error('Service must be "lotw" or "dcl"');
|
||||
}
|
||||
|
||||
// Get current settings to find the interval
|
||||
const [currentSettings] = await db
|
||||
.select()
|
||||
.from(autoSyncSettings)
|
||||
.where(eq(autoSyncSettings.userId, userId));
|
||||
|
||||
if (!currentSettings) {
|
||||
logger.warn('No auto-sync settings found for user', { userId, service });
|
||||
return;
|
||||
}
|
||||
|
||||
const intervalHours = service === 'lotw'
|
||||
? currentSettings.lotwIntervalHours
|
||||
: currentSettings.dclIntervalHours;
|
||||
|
||||
// Calculate next sync time
|
||||
const nextSyncAt = calculateNextSyncTime(intervalHours);
|
||||
|
||||
// Update timestamps
|
||||
const updateData = {
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
|
||||
if (service === 'lotw') {
|
||||
updateData.lotwLastSyncAt = lastSyncDate;
|
||||
updateData.lotwNextSyncAt = nextSyncAt;
|
||||
} else {
|
||||
updateData.dclLastSyncAt = lastSyncDate;
|
||||
updateData.dclNextSyncAt = nextSyncAt;
|
||||
}
|
||||
|
||||
await db
|
||||
.update(autoSyncSettings)
|
||||
.set(updateData)
|
||||
.where(eq(autoSyncSettings.userId, userId));
|
||||
|
||||
logger.debug('Updated sync timestamps', {
|
||||
userId,
|
||||
service,
|
||||
lastSyncAt: lastSyncDate.toISOString(),
|
||||
nextSyncAt: nextSyncAt.toISOString(),
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to update sync timestamps', { error: error.message, userId, service });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
234
src/backend/services/scheduler.service.js
Normal file
234
src/backend/services/scheduler.service.js
Normal file
@@ -0,0 +1,234 @@
|
||||
import { logger } from '../config.js';
|
||||
import {
|
||||
getPendingSyncUsers,
|
||||
updateSyncTimestamps,
|
||||
} from './auto-sync.service.js';
|
||||
import {
|
||||
enqueueJob,
|
||||
getUserActiveJob,
|
||||
} from './job-queue.service.js';
|
||||
import { getUserById } from './auth.service.js';
|
||||
|
||||
/**
|
||||
* Auto-Sync Scheduler Service
|
||||
* Manages automatic synchronization of DCL and LoTW data
|
||||
* Runs every minute to check for due syncs and enqueues jobs
|
||||
*/
|
||||
|
||||
// Scheduler state
|
||||
let schedulerInterval = null;
|
||||
let isRunning = false;
|
||||
let isShuttingDown = false;
|
||||
|
||||
// Scheduler configuration
|
||||
const SCHEDULER_TICK_INTERVAL_MS = 60 * 1000; // 1 minute
|
||||
const INITIAL_DELAY_MS = 5000; // 5 seconds after server start
|
||||
|
||||
// Allow faster tick interval for testing (set via environment variable)
|
||||
const TEST_MODE = process.env.SCHEDULER_TEST_MODE === 'true';
|
||||
const TEST_TICK_INTERVAL_MS = 10 * 1000; // 10 seconds in test mode
|
||||
|
||||
/**
|
||||
* Get scheduler status
|
||||
* @returns {Object} Scheduler status
|
||||
*/
|
||||
export function getSchedulerStatus() {
|
||||
return {
|
||||
isRunning,
|
||||
isShuttingDown,
|
||||
tickIntervalMs: TEST_MODE ? TEST_TICK_INTERVAL_MS : SCHEDULER_TICK_INTERVAL_MS,
|
||||
activeInterval: !!schedulerInterval,
|
||||
testMode: TEST_MODE,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Process pending syncs for a specific service
|
||||
* @param {string} service - 'lotw' or 'dcl'
|
||||
*/
|
||||
async function processServiceSyncs(service) {
|
||||
try {
|
||||
const pendingUsers = await getPendingSyncUsers(service);
|
||||
|
||||
if (pendingUsers.length === 0) {
|
||||
logger.debug('No pending syncs', { service });
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info('Processing pending syncs', {
|
||||
service,
|
||||
count: pendingUsers.length,
|
||||
});
|
||||
|
||||
for (const user of pendingUsers) {
|
||||
if (isShuttingDown) {
|
||||
logger.info('Scheduler shutting down, skipping pending sync', {
|
||||
service,
|
||||
userId: user.userId,
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
// Check if there's already an active job for this user and service
|
||||
const activeJob = await getUserActiveJob(user.userId, `${service}_sync`);
|
||||
|
||||
if (activeJob) {
|
||||
logger.debug('User already has active job, skipping', {
|
||||
service,
|
||||
userId: user.userId,
|
||||
activeJobId: activeJob.id,
|
||||
});
|
||||
|
||||
// Update the next sync time to try again later
|
||||
// This prevents continuous checking while a job is running
|
||||
await updateSyncTimestamps(user.userId, service, new Date());
|
||||
continue;
|
||||
}
|
||||
|
||||
// Enqueue the sync job
|
||||
logger.info('Enqueuing auto-sync job', {
|
||||
service,
|
||||
userId: user.userId,
|
||||
});
|
||||
|
||||
const result = await enqueueJob(user.userId, `${service}_sync`);
|
||||
|
||||
if (result.success) {
|
||||
// Update timestamps immediately on successful enqueue
|
||||
await updateSyncTimestamps(user.userId, service, new Date());
|
||||
} else {
|
||||
logger.warn('Failed to enqueue auto-sync job', {
|
||||
service,
|
||||
userId: user.userId,
|
||||
reason: result.error,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error processing user sync', {
|
||||
service,
|
||||
userId: user.userId,
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error processing service syncs', {
|
||||
service,
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main scheduler tick function
|
||||
* Checks for pending LoTW and DCL syncs and processes them
|
||||
*/
|
||||
async function schedulerTick() {
|
||||
if (isShuttingDown) {
|
||||
logger.debug('Scheduler shutdown in progress, skipping tick');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
logger.debug('Scheduler tick started');
|
||||
|
||||
// Process LoTW syncs
|
||||
await processServiceSyncs('lotw');
|
||||
|
||||
// Process DCL syncs
|
||||
await processServiceSyncs('dcl');
|
||||
|
||||
logger.debug('Scheduler tick completed');
|
||||
} catch (error) {
|
||||
logger.error('Scheduler tick error', {
|
||||
error: error.message,
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the scheduler
|
||||
* Begins periodic checks for pending syncs
|
||||
*/
|
||||
export function startScheduler() {
|
||||
if (isRunning) {
|
||||
logger.warn('Scheduler already running');
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if scheduler is disabled via environment variable
|
||||
if (process.env.DISABLE_SCHEDULER === 'true') {
|
||||
logger.info('Scheduler disabled via DISABLE_SCHEDULER environment variable');
|
||||
return;
|
||||
}
|
||||
|
||||
isRunning = true;
|
||||
isShuttingDown = false;
|
||||
|
||||
const tickInterval = TEST_MODE ? TEST_TICK_INTERVAL_MS : SCHEDULER_TICK_INTERVAL_MS;
|
||||
|
||||
// Initial delay to allow server to fully start
|
||||
logger.info('Scheduler starting, initial tick in 5 seconds', {
|
||||
testMode: TEST_MODE,
|
||||
tickIntervalMs: tickInterval,
|
||||
});
|
||||
|
||||
// Schedule first tick
|
||||
setTimeout(() => {
|
||||
if (!isShuttingDown) {
|
||||
schedulerTick();
|
||||
|
||||
// Set up recurring interval
|
||||
schedulerInterval = setInterval(() => {
|
||||
if (!isShuttingDown) {
|
||||
schedulerTick();
|
||||
}
|
||||
}, tickInterval);
|
||||
|
||||
logger.info('Scheduler started', {
|
||||
tickIntervalMs: tickInterval,
|
||||
testMode: TEST_MODE,
|
||||
});
|
||||
}
|
||||
}, INITIAL_DELAY_MS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the scheduler gracefully
|
||||
* Waits for current tick to complete before stopping
|
||||
*/
|
||||
export async function stopScheduler() {
|
||||
if (!isRunning) {
|
||||
logger.debug('Scheduler not running');
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info('Stopping scheduler...');
|
||||
isShuttingDown = true;
|
||||
|
||||
// Clear the interval
|
||||
if (schedulerInterval) {
|
||||
clearInterval(schedulerInterval);
|
||||
schedulerInterval = null;
|
||||
}
|
||||
|
||||
// Wait a moment for any in-progress tick to complete
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
isRunning = false;
|
||||
logger.info('Scheduler stopped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger an immediate scheduler tick (for testing or manual sync)
|
||||
*/
|
||||
export async function triggerSchedulerTick() {
|
||||
if (!isRunning) {
|
||||
throw new Error('Scheduler is not running');
|
||||
}
|
||||
|
||||
logger.info('Manual scheduler tick triggered');
|
||||
await schedulerTick();
|
||||
}
|
||||
Reference in New Issue
Block a user