diff --git a/src/backend/index.js b/src/backend/index.js index ea86082..4b63d90 100644 --- a/src/backend/index.js +++ b/src/backend/index.js @@ -314,8 +314,9 @@ const app = new Elysia() * POST /api/lotw/sync * Queue a LoTW sync job (requires authentication) * Returns immediately with job ID + * Body: { syncType?: 'qsl_delta' | 'qsl_full' | 'qso_delta' | 'qso_full' } */ - .post('/api/lotw/sync', async ({ user, set }) => { + .post('/api/lotw/sync', async ({ user, body, set }) => { if (!user) { logger.warn('/api/lotw/sync: Unauthorized access attempt'); set.status = 401; @@ -323,7 +324,8 @@ const app = new Elysia() } try { - const result = await enqueueJob(user.id, 'lotw_sync'); + const { syncType = 'qsl_delta' } = body || {}; + const result = await enqueueJob(user.id, 'lotw_sync', { syncType }); if (!result.success && result.existingJob) { return { diff --git a/src/backend/services/job-queue.service.js b/src/backend/services/job-queue.service.js index a5e3ba1..fdbe7da 100644 --- a/src/backend/services/job-queue.service.js +++ b/src/backend/services/job-queue.service.js @@ -22,10 +22,13 @@ const activeJobs = new Map(); * Enqueue a new sync job * @param {number} userId - User ID * @param {string} jobType - Type of job ('lotw_sync' or 'dcl_sync') + * @param {Object} options - Optional job parameters + * @param {string} options.syncType - LoTW sync type: 'qsl_delta' (default), 'qsl_full', 'qso_delta', 'qso_full' * @returns {Promise} Job object with ID */ -export async function enqueueJob(userId, jobType = 'lotw_sync') { - logger.debug('Enqueueing sync job', { userId, jobType }); +export async function enqueueJob(userId, jobType = 'lotw_sync', options = {}) { + const { syncType = 'qsl_delta' } = options; + logger.debug('Enqueueing sync job', { userId, jobType, syncType }); // Check for existing active job of the same type const existingJob = await getUserActiveJob(userId, jobType); @@ -49,10 +52,10 @@ export async function enqueueJob(userId, jobType = 'lotw_sync') { }) .returning(); - logger.info('Job created', { jobId: job.id, userId, jobType }); + logger.info('Job created', { jobId: job.id, userId, jobType, syncType }); // Start processing asynchronously (don't await) - processJobAsync(job.id, userId, jobType).catch((error) => { + processJobAsync(job.id, userId, jobType, syncType).catch((error) => { logger.error(`Job processing error`, { jobId: job.id, error: error.message }); }); @@ -73,8 +76,9 @@ export async function enqueueJob(userId, jobType = 'lotw_sync') { * @param {number} jobId - Job ID * @param {number} userId - User ID * @param {string} jobType - Type of job ('lotw_sync' or 'dcl_sync') + * @param {string} syncType - LoTW sync type: 'qsl_delta', 'qsl_full', 'qso_delta', 'qso_full' */ -async function processJobAsync(jobId, userId, jobType) { +async function processJobAsync(jobId, userId, jobType, syncType = 'qsl_delta') { const jobPromise = (async () => { try { const { getUserById } = await import('./auth.service.js'); @@ -130,15 +134,28 @@ async function processJobAsync(jobId, userId, jobType) { return null; } - // Get last QSL date for incremental sync - const { getLastLoTWQSLDate, syncQSOs } = await import('./lotw.service.js'); - const lastQSLDate = await getLastLoTWQSLDate(userId); - const sinceDate = lastQSLDate || new Date('2000-01-01'); + // Get the appropriate date based on sync type + const { getLastLoTWQSODate, getLastLoTWQSLDate, syncQSOs } = await import('./lotw.service.js'); - if (lastQSLDate) { - logger.info(`Job ${jobId}: LoTW incremental sync`, { since: sinceDate.toISOString().split('T')[0] }); + let sinceDate = null; + let dateSource = ''; + + if (syncType.includes('delta')) { + // Delta sync: use date filter + if (syncType === 'qso_delta') { + const lastQSODate = await getLastLoTWQSODate(userId); + sinceDate = lastQSODate || new Date('2000-01-01'); + dateSource = lastQSODate ? 'QSO' : 'full'; + } else { + // qsl_delta + const lastQSLDate = await getLastLoTWQSLDate(userId); + sinceDate = lastQSLDate || new Date('2000-01-01'); + dateSource = lastQSLDate ? 'QSL' : 'full'; + } + logger.info(`Job ${jobId}: LoTW ${syncType} sync`, { since: sinceDate.toISOString().split('T')[0], source: dateSource }); } else { - logger.info(`Job ${jobId}: LoTW full sync`); + // Full sync: no date filter + logger.info(`Job ${jobId}: LoTW ${syncType} full sync`); } // Update job progress @@ -147,8 +164,8 @@ async function processJobAsync(jobId, userId, jobType) { step: 'fetch', }); - // Execute the sync - result = await syncQSOs(userId, user.lotwUsername, user.lotwPassword, sinceDate, jobId); + // Execute the sync with syncType + result = await syncQSOs(userId, user.lotwUsername, user.lotwPassword, sinceDate, jobId, syncType); } // Update job as completed diff --git a/src/backend/services/lotw.service.js b/src/backend/services/lotw.service.js index 652b346..8d32018 100644 --- a/src/backend/services/lotw.service.js +++ b/src/backend/services/lotw.service.js @@ -49,15 +49,24 @@ const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); /** * Fetch QSOs from LoTW with retry support + * @param {string} lotwUsername - LoTW username + * @param {string} lotwPassword - LoTW password + * @param {Date|null} sinceDate - Optional date for incremental sync + * @param {string} syncType - Type of sync: 'qsl_delta' (default), 'qsl_full', 'qso_delta', 'qso_full' */ -async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) { +async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null, syncType = 'qsl_delta') { const url = 'https://lotw.arrl.org/lotwuser/lotwreport.adi'; + // Determine qso_qsl parameter based on sync type + // qsl_* = only QSLs (confirmed QSOs) + // qso_* = all QSOs (confirmed + unconfirmed) + const qsoQslValue = syncType.startsWith('qsl') ? 'yes' : 'no'; + const params = new URLSearchParams({ login: lotwUsername, password: lotwPassword, qso_query: '1', - qso_qsl: 'yes', + qso_qsl: qsoQslValue, qso_qsldetail: 'yes', qso_mydetail: 'yes', qso_withown: 'yes', @@ -66,9 +75,9 @@ async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) { if (sinceDate) { const dateStr = sinceDate.toISOString().split('T')[0]; params.append('qso_qslsince', dateStr); - logger.debug('Incremental sync since', { date: dateStr }); + logger.debug('Incremental sync', { syncType, since: dateStr }); } else { - logger.debug('Full sync - fetching all QSOs'); + logger.debug('Full sync', { syncType }); } const fullUrl = `${url}?${params.toString()}`; @@ -187,8 +196,9 @@ function convertQSODatabaseFormat(adifQSO, userId) { * @param {string} lotwPassword - LoTW password * @param {Date|null} sinceDate - Optional date for incremental sync * @param {number|null} jobId - Optional job ID for progress tracking + * @param {string} syncType - Type of sync: 'qsl_delta' (default), 'qsl_full', 'qso_delta', 'qso_full' */ -export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null, jobId = null) { +export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null, jobId = null, syncType = 'qsl_delta') { if (jobId) { await updateJobProgress(jobId, { message: 'Fetching QSOs from LoTW...', @@ -196,7 +206,7 @@ export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = n }); } - const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate); + const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate, syncType); // Check for error response from LoTW fetch if (!adifQSOs) { @@ -499,6 +509,27 @@ export async function getLastLoTWQSLDate(userId) { return new Date(`${year}-${month}-${day}`); } +/** + * Get the date of the last LoTW QSO for a user (for incremental sync of all QSOs) + */ +export async function getLastLoTWQSODate(userId) { + const [result] = await db + .select({ maxDate: max(qsos.qsoDate) }) + .from(qsos) + .where(eq(qsos.userId, userId)); + + if (!result || !result.maxDate) return null; + + const dateStr = result.maxDate; + if (!dateStr || dateStr === '') return null; + + const year = dateStr.substring(0, 4); + const month = dateStr.substring(4, 6); + const day = dateStr.substring(6, 8); + + return new Date(`${year}-${month}-${day}`); +} + /** * Delete all QSOs for a user */ diff --git a/src/frontend/src/lib/api.js b/src/frontend/src/lib/api.js index f4ccfef..a0f8d3b 100644 --- a/src/frontend/src/lib/api.js +++ b/src/frontend/src/lib/api.js @@ -72,7 +72,10 @@ export const qsosAPI = { getStats: () => apiRequest('/qsos/stats'), - syncFromLoTW: () => apiRequest('/lotw/sync', { method: 'POST' }), + syncFromLoTW: (syncType = 'qsl_delta') => apiRequest('/lotw/sync', { + method: 'POST', + body: JSON.stringify({ syncType }), + }), syncFromDCL: () => apiRequest('/dcl/sync', { method: 'POST' }), diff --git a/src/frontend/src/routes/qsos/+page.svelte b/src/frontend/src/routes/qsos/+page.svelte index 3f0e738..71cc2cd 100644 --- a/src/frontend/src/routes/qsos/+page.svelte +++ b/src/frontend/src/routes/qsos/+page.svelte @@ -39,6 +39,9 @@ let selectedQSO = null; let showQSODetailModal = false; + // LoTW sync type selection + let lotwSyncType = 'qsl_delta'; // Options: 'qsl_delta', 'qsl_full', 'qso_delta', 'qso_full' + let filters = { band: '', mode: '', @@ -226,7 +229,7 @@ async function handleLoTWSync() { try { - const response = await qsosAPI.syncFromLoTW(); + const response = await qsosAPI.syncFromLoTW(lotwSyncType); if (response.jobId) { startLoTWPolling(response.jobId); @@ -387,6 +390,18 @@ {/if} + + +