feat: add LoTW sync type support (QSL/QSO delta/full)

- Add syncType parameter to LoTW sync: 'qsl_delta', 'qsl_full', 'qso_delta', 'qso_full'
- qsl_* = only confirmed QSOs (qso_qsl=yes)
- qso_* = all QSOs confirmed+unconfirmed (qso_qsl=no)
- delta = incremental sync with date filter
- full = sync all records without date filter
- Add getLastLoTWQSODate() for QSO-based incremental sync
- Add sync type dropdown selector on QSO page
- Update job queue service to handle sync types
- Update API endpoint to accept syncType in request body

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
2026-01-20 12:09:28 +01:00
parent a50b4ae724
commit 5b7893536e
5 changed files with 111 additions and 24 deletions

View File

@@ -314,8 +314,9 @@ const app = new Elysia()
* POST /api/lotw/sync * POST /api/lotw/sync
* Queue a LoTW sync job (requires authentication) * Queue a LoTW sync job (requires authentication)
* Returns immediately with job ID * Returns immediately with job ID
* Body: { syncType?: 'qsl_delta' | 'qsl_full' | 'qso_delta' | 'qso_full' }
*/ */
.post('/api/lotw/sync', async ({ user, set }) => { .post('/api/lotw/sync', async ({ user, body, set }) => {
if (!user) { if (!user) {
logger.warn('/api/lotw/sync: Unauthorized access attempt'); logger.warn('/api/lotw/sync: Unauthorized access attempt');
set.status = 401; set.status = 401;
@@ -323,7 +324,8 @@ const app = new Elysia()
} }
try { try {
const result = await enqueueJob(user.id, 'lotw_sync'); const { syncType = 'qsl_delta' } = body || {};
const result = await enqueueJob(user.id, 'lotw_sync', { syncType });
if (!result.success && result.existingJob) { if (!result.success && result.existingJob) {
return { return {

View File

@@ -22,10 +22,13 @@ const activeJobs = new Map();
* Enqueue a new sync job * Enqueue a new sync job
* @param {number} userId - User ID * @param {number} userId - User ID
* @param {string} jobType - Type of job ('lotw_sync' or 'dcl_sync') * @param {string} jobType - Type of job ('lotw_sync' or 'dcl_sync')
* @param {Object} options - Optional job parameters
* @param {string} options.syncType - LoTW sync type: 'qsl_delta' (default), 'qsl_full', 'qso_delta', 'qso_full'
* @returns {Promise<Object>} Job object with ID * @returns {Promise<Object>} Job object with ID
*/ */
export async function enqueueJob(userId, jobType = 'lotw_sync') { export async function enqueueJob(userId, jobType = 'lotw_sync', options = {}) {
logger.debug('Enqueueing sync job', { userId, jobType }); const { syncType = 'qsl_delta' } = options;
logger.debug('Enqueueing sync job', { userId, jobType, syncType });
// Check for existing active job of the same type // Check for existing active job of the same type
const existingJob = await getUserActiveJob(userId, jobType); const existingJob = await getUserActiveJob(userId, jobType);
@@ -49,10 +52,10 @@ export async function enqueueJob(userId, jobType = 'lotw_sync') {
}) })
.returning(); .returning();
logger.info('Job created', { jobId: job.id, userId, jobType }); logger.info('Job created', { jobId: job.id, userId, jobType, syncType });
// Start processing asynchronously (don't await) // Start processing asynchronously (don't await)
processJobAsync(job.id, userId, jobType).catch((error) => { processJobAsync(job.id, userId, jobType, syncType).catch((error) => {
logger.error(`Job processing error`, { jobId: job.id, error: error.message }); logger.error(`Job processing error`, { jobId: job.id, error: error.message });
}); });
@@ -73,8 +76,9 @@ export async function enqueueJob(userId, jobType = 'lotw_sync') {
* @param {number} jobId - Job ID * @param {number} jobId - Job ID
* @param {number} userId - User ID * @param {number} userId - User ID
* @param {string} jobType - Type of job ('lotw_sync' or 'dcl_sync') * @param {string} jobType - Type of job ('lotw_sync' or 'dcl_sync')
* @param {string} syncType - LoTW sync type: 'qsl_delta', 'qsl_full', 'qso_delta', 'qso_full'
*/ */
async function processJobAsync(jobId, userId, jobType) { async function processJobAsync(jobId, userId, jobType, syncType = 'qsl_delta') {
const jobPromise = (async () => { const jobPromise = (async () => {
try { try {
const { getUserById } = await import('./auth.service.js'); const { getUserById } = await import('./auth.service.js');
@@ -130,15 +134,28 @@ async function processJobAsync(jobId, userId, jobType) {
return null; return null;
} }
// Get last QSL date for incremental sync // Get the appropriate date based on sync type
const { getLastLoTWQSLDate, syncQSOs } = await import('./lotw.service.js'); const { getLastLoTWQSODate, getLastLoTWQSLDate, syncQSOs } = await import('./lotw.service.js');
const lastQSLDate = await getLastLoTWQSLDate(userId);
const sinceDate = lastQSLDate || new Date('2000-01-01');
if (lastQSLDate) { let sinceDate = null;
logger.info(`Job ${jobId}: LoTW incremental sync`, { since: sinceDate.toISOString().split('T')[0] }); let dateSource = '';
if (syncType.includes('delta')) {
// Delta sync: use date filter
if (syncType === 'qso_delta') {
const lastQSODate = await getLastLoTWQSODate(userId);
sinceDate = lastQSODate || new Date('2000-01-01');
dateSource = lastQSODate ? 'QSO' : 'full';
} else {
// qsl_delta
const lastQSLDate = await getLastLoTWQSLDate(userId);
sinceDate = lastQSLDate || new Date('2000-01-01');
dateSource = lastQSLDate ? 'QSL' : 'full';
}
logger.info(`Job ${jobId}: LoTW ${syncType} sync`, { since: sinceDate.toISOString().split('T')[0], source: dateSource });
} else { } else {
logger.info(`Job ${jobId}: LoTW full sync`); // Full sync: no date filter
logger.info(`Job ${jobId}: LoTW ${syncType} full sync`);
} }
// Update job progress // Update job progress
@@ -147,8 +164,8 @@ async function processJobAsync(jobId, userId, jobType) {
step: 'fetch', step: 'fetch',
}); });
// Execute the sync // Execute the sync with syncType
result = await syncQSOs(userId, user.lotwUsername, user.lotwPassword, sinceDate, jobId); result = await syncQSOs(userId, user.lotwUsername, user.lotwPassword, sinceDate, jobId, syncType);
} }
// Update job as completed // Update job as completed

View File

@@ -49,15 +49,24 @@ const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
/** /**
* Fetch QSOs from LoTW with retry support * Fetch QSOs from LoTW with retry support
* @param {string} lotwUsername - LoTW username
* @param {string} lotwPassword - LoTW password
* @param {Date|null} sinceDate - Optional date for incremental sync
* @param {string} syncType - Type of sync: 'qsl_delta' (default), 'qsl_full', 'qso_delta', 'qso_full'
*/ */
async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) { async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null, syncType = 'qsl_delta') {
const url = 'https://lotw.arrl.org/lotwuser/lotwreport.adi'; const url = 'https://lotw.arrl.org/lotwuser/lotwreport.adi';
// Determine qso_qsl parameter based on sync type
// qsl_* = only QSLs (confirmed QSOs)
// qso_* = all QSOs (confirmed + unconfirmed)
const qsoQslValue = syncType.startsWith('qsl') ? 'yes' : 'no';
const params = new URLSearchParams({ const params = new URLSearchParams({
login: lotwUsername, login: lotwUsername,
password: lotwPassword, password: lotwPassword,
qso_query: '1', qso_query: '1',
qso_qsl: 'yes', qso_qsl: qsoQslValue,
qso_qsldetail: 'yes', qso_qsldetail: 'yes',
qso_mydetail: 'yes', qso_mydetail: 'yes',
qso_withown: 'yes', qso_withown: 'yes',
@@ -66,9 +75,9 @@ async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
if (sinceDate) { if (sinceDate) {
const dateStr = sinceDate.toISOString().split('T')[0]; const dateStr = sinceDate.toISOString().split('T')[0];
params.append('qso_qslsince', dateStr); params.append('qso_qslsince', dateStr);
logger.debug('Incremental sync since', { date: dateStr }); logger.debug('Incremental sync', { syncType, since: dateStr });
} else { } else {
logger.debug('Full sync - fetching all QSOs'); logger.debug('Full sync', { syncType });
} }
const fullUrl = `${url}?${params.toString()}`; const fullUrl = `${url}?${params.toString()}`;
@@ -187,8 +196,9 @@ function convertQSODatabaseFormat(adifQSO, userId) {
* @param {string} lotwPassword - LoTW password * @param {string} lotwPassword - LoTW password
* @param {Date|null} sinceDate - Optional date for incremental sync * @param {Date|null} sinceDate - Optional date for incremental sync
* @param {number|null} jobId - Optional job ID for progress tracking * @param {number|null} jobId - Optional job ID for progress tracking
* @param {string} syncType - Type of sync: 'qsl_delta' (default), 'qsl_full', 'qso_delta', 'qso_full'
*/ */
export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null, jobId = null) { export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null, jobId = null, syncType = 'qsl_delta') {
if (jobId) { if (jobId) {
await updateJobProgress(jobId, { await updateJobProgress(jobId, {
message: 'Fetching QSOs from LoTW...', message: 'Fetching QSOs from LoTW...',
@@ -196,7 +206,7 @@ export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = n
}); });
} }
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate); const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate, syncType);
// Check for error response from LoTW fetch // Check for error response from LoTW fetch
if (!adifQSOs) { if (!adifQSOs) {
@@ -499,6 +509,27 @@ export async function getLastLoTWQSLDate(userId) {
return new Date(`${year}-${month}-${day}`); return new Date(`${year}-${month}-${day}`);
} }
/**
* Get the date of the last LoTW QSO for a user (for incremental sync of all QSOs)
*/
export async function getLastLoTWQSODate(userId) {
const [result] = await db
.select({ maxDate: max(qsos.qsoDate) })
.from(qsos)
.where(eq(qsos.userId, userId));
if (!result || !result.maxDate) return null;
const dateStr = result.maxDate;
if (!dateStr || dateStr === '') return null;
const year = dateStr.substring(0, 4);
const month = dateStr.substring(4, 6);
const day = dateStr.substring(6, 8);
return new Date(`${year}-${month}-${day}`);
}
/** /**
* Delete all QSOs for a user * Delete all QSOs for a user
*/ */

View File

@@ -72,7 +72,10 @@ export const qsosAPI = {
getStats: () => apiRequest('/qsos/stats'), getStats: () => apiRequest('/qsos/stats'),
syncFromLoTW: () => apiRequest('/lotw/sync', { method: 'POST' }), syncFromLoTW: (syncType = 'qsl_delta') => apiRequest('/lotw/sync', {
method: 'POST',
body: JSON.stringify({ syncType }),
}),
syncFromDCL: () => apiRequest('/dcl/sync', { method: 'POST' }), syncFromDCL: () => apiRequest('/dcl/sync', { method: 'POST' }),

View File

@@ -39,6 +39,9 @@
let selectedQSO = null; let selectedQSO = null;
let showQSODetailModal = false; let showQSODetailModal = false;
// LoTW sync type selection
let lotwSyncType = 'qsl_delta'; // Options: 'qsl_delta', 'qsl_full', 'qso_delta', 'qso_full'
let filters = { let filters = {
band: '', band: '',
mode: '', mode: '',
@@ -226,7 +229,7 @@
async function handleLoTWSync() { async function handleLoTWSync() {
try { try {
const response = await qsosAPI.syncFromLoTW(); const response = await qsosAPI.syncFromLoTW(lotwSyncType);
if (response.jobId) { if (response.jobId) {
startLoTWPolling(response.jobId); startLoTWPolling(response.jobId);
@@ -387,6 +390,18 @@
</button> </button>
{/if} {/if}
<!-- LoTW sync type selector -->
<select
class="sync-type-select"
bind:value={lotwSyncType}
disabled={lotwSyncStatus === 'running' || lotwSyncStatus === 'pending' || deleting}
>
<option value="qsl_delta">LoTW: QSL Delta</option>
<option value="qsl_full">LoTW: QSL Full</option>
<option value="qso_delta">LoTW: QSO Delta</option>
<option value="qso_full">LoTW: QSO Full</option>
</select>
<SyncButton <SyncButton
service="lotw" service="lotw"
syncStatus={lotwSyncStatus} syncStatus={lotwSyncStatus}
@@ -922,6 +937,25 @@
flex-wrap: wrap; flex-wrap: wrap;
} }
.sync-type-select {
padding: 0.75rem 1rem;
border: 1px solid #ddd;
border-radius: 4px;
background-color: white;
font-size: 1rem;
cursor: pointer;
transition: border-color 0.2s;
}
.sync-type-select:hover:not(:disabled) {
border-color: #4a90e2;
}
.sync-type-select:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.filters { .filters {
background: #f8f9fa; background: #f8f9fa;
padding: 1rem; padding: 1rem;