feat: add LoTW sync type support (QSL/QSO delta/full)
- Add syncType parameter to LoTW sync: 'qsl_delta', 'qsl_full', 'qso_delta', 'qso_full' - qsl_* = only confirmed QSOs (qso_qsl=yes) - qso_* = all QSOs confirmed+unconfirmed (qso_qsl=no) - delta = incremental sync with date filter - full = sync all records without date filter - Add getLastLoTWQSODate() for QSO-based incremental sync - Add sync type dropdown selector on QSO page - Update job queue service to handle sync types - Update API endpoint to accept syncType in request body Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -22,10 +22,13 @@ const activeJobs = new Map();
|
||||
* Enqueue a new sync job
|
||||
* @param {number} userId - User ID
|
||||
* @param {string} jobType - Type of job ('lotw_sync' or 'dcl_sync')
|
||||
* @param {Object} options - Optional job parameters
|
||||
* @param {string} options.syncType - LoTW sync type: 'qsl_delta' (default), 'qsl_full', 'qso_delta', 'qso_full'
|
||||
* @returns {Promise<Object>} Job object with ID
|
||||
*/
|
||||
export async function enqueueJob(userId, jobType = 'lotw_sync') {
|
||||
logger.debug('Enqueueing sync job', { userId, jobType });
|
||||
export async function enqueueJob(userId, jobType = 'lotw_sync', options = {}) {
|
||||
const { syncType = 'qsl_delta' } = options;
|
||||
logger.debug('Enqueueing sync job', { userId, jobType, syncType });
|
||||
|
||||
// Check for existing active job of the same type
|
||||
const existingJob = await getUserActiveJob(userId, jobType);
|
||||
@@ -49,10 +52,10 @@ export async function enqueueJob(userId, jobType = 'lotw_sync') {
|
||||
})
|
||||
.returning();
|
||||
|
||||
logger.info('Job created', { jobId: job.id, userId, jobType });
|
||||
logger.info('Job created', { jobId: job.id, userId, jobType, syncType });
|
||||
|
||||
// Start processing asynchronously (don't await)
|
||||
processJobAsync(job.id, userId, jobType).catch((error) => {
|
||||
processJobAsync(job.id, userId, jobType, syncType).catch((error) => {
|
||||
logger.error(`Job processing error`, { jobId: job.id, error: error.message });
|
||||
});
|
||||
|
||||
@@ -73,8 +76,9 @@ export async function enqueueJob(userId, jobType = 'lotw_sync') {
|
||||
* @param {number} jobId - Job ID
|
||||
* @param {number} userId - User ID
|
||||
* @param {string} jobType - Type of job ('lotw_sync' or 'dcl_sync')
|
||||
* @param {string} syncType - LoTW sync type: 'qsl_delta', 'qsl_full', 'qso_delta', 'qso_full'
|
||||
*/
|
||||
async function processJobAsync(jobId, userId, jobType) {
|
||||
async function processJobAsync(jobId, userId, jobType, syncType = 'qsl_delta') {
|
||||
const jobPromise = (async () => {
|
||||
try {
|
||||
const { getUserById } = await import('./auth.service.js');
|
||||
@@ -130,15 +134,28 @@ async function processJobAsync(jobId, userId, jobType) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Get last QSL date for incremental sync
|
||||
const { getLastLoTWQSLDate, syncQSOs } = await import('./lotw.service.js');
|
||||
const lastQSLDate = await getLastLoTWQSLDate(userId);
|
||||
const sinceDate = lastQSLDate || new Date('2000-01-01');
|
||||
// Get the appropriate date based on sync type
|
||||
const { getLastLoTWQSODate, getLastLoTWQSLDate, syncQSOs } = await import('./lotw.service.js');
|
||||
|
||||
if (lastQSLDate) {
|
||||
logger.info(`Job ${jobId}: LoTW incremental sync`, { since: sinceDate.toISOString().split('T')[0] });
|
||||
let sinceDate = null;
|
||||
let dateSource = '';
|
||||
|
||||
if (syncType.includes('delta')) {
|
||||
// Delta sync: use date filter
|
||||
if (syncType === 'qso_delta') {
|
||||
const lastQSODate = await getLastLoTWQSODate(userId);
|
||||
sinceDate = lastQSODate || new Date('2000-01-01');
|
||||
dateSource = lastQSODate ? 'QSO' : 'full';
|
||||
} else {
|
||||
// qsl_delta
|
||||
const lastQSLDate = await getLastLoTWQSLDate(userId);
|
||||
sinceDate = lastQSLDate || new Date('2000-01-01');
|
||||
dateSource = lastQSLDate ? 'QSL' : 'full';
|
||||
}
|
||||
logger.info(`Job ${jobId}: LoTW ${syncType} sync`, { since: sinceDate.toISOString().split('T')[0], source: dateSource });
|
||||
} else {
|
||||
logger.info(`Job ${jobId}: LoTW full sync`);
|
||||
// Full sync: no date filter
|
||||
logger.info(`Job ${jobId}: LoTW ${syncType} full sync`);
|
||||
}
|
||||
|
||||
// Update job progress
|
||||
@@ -147,8 +164,8 @@ async function processJobAsync(jobId, userId, jobType) {
|
||||
step: 'fetch',
|
||||
});
|
||||
|
||||
// Execute the sync
|
||||
result = await syncQSOs(userId, user.lotwUsername, user.lotwPassword, sinceDate, jobId);
|
||||
// Execute the sync with syncType
|
||||
result = await syncQSOs(userId, user.lotwUsername, user.lotwPassword, sinceDate, jobId, syncType);
|
||||
}
|
||||
|
||||
// Update job as completed
|
||||
|
||||
@@ -49,15 +49,24 @@ const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
/**
|
||||
* Fetch QSOs from LoTW with retry support
|
||||
* @param {string} lotwUsername - LoTW username
|
||||
* @param {string} lotwPassword - LoTW password
|
||||
* @param {Date|null} sinceDate - Optional date for incremental sync
|
||||
* @param {string} syncType - Type of sync: 'qsl_delta' (default), 'qsl_full', 'qso_delta', 'qso_full'
|
||||
*/
|
||||
async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||
async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null, syncType = 'qsl_delta') {
|
||||
const url = 'https://lotw.arrl.org/lotwuser/lotwreport.adi';
|
||||
|
||||
// Determine qso_qsl parameter based on sync type
|
||||
// qsl_* = only QSLs (confirmed QSOs)
|
||||
// qso_* = all QSOs (confirmed + unconfirmed)
|
||||
const qsoQslValue = syncType.startsWith('qsl') ? 'yes' : 'no';
|
||||
|
||||
const params = new URLSearchParams({
|
||||
login: lotwUsername,
|
||||
password: lotwPassword,
|
||||
qso_query: '1',
|
||||
qso_qsl: 'yes',
|
||||
qso_qsl: qsoQslValue,
|
||||
qso_qsldetail: 'yes',
|
||||
qso_mydetail: 'yes',
|
||||
qso_withown: 'yes',
|
||||
@@ -66,9 +75,9 @@ async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||
if (sinceDate) {
|
||||
const dateStr = sinceDate.toISOString().split('T')[0];
|
||||
params.append('qso_qslsince', dateStr);
|
||||
logger.debug('Incremental sync since', { date: dateStr });
|
||||
logger.debug('Incremental sync', { syncType, since: dateStr });
|
||||
} else {
|
||||
logger.debug('Full sync - fetching all QSOs');
|
||||
logger.debug('Full sync', { syncType });
|
||||
}
|
||||
|
||||
const fullUrl = `${url}?${params.toString()}`;
|
||||
@@ -187,8 +196,9 @@ function convertQSODatabaseFormat(adifQSO, userId) {
|
||||
* @param {string} lotwPassword - LoTW password
|
||||
* @param {Date|null} sinceDate - Optional date for incremental sync
|
||||
* @param {number|null} jobId - Optional job ID for progress tracking
|
||||
* @param {string} syncType - Type of sync: 'qsl_delta' (default), 'qsl_full', 'qso_delta', 'qso_full'
|
||||
*/
|
||||
export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null, jobId = null) {
|
||||
export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null, jobId = null, syncType = 'qsl_delta') {
|
||||
if (jobId) {
|
||||
await updateJobProgress(jobId, {
|
||||
message: 'Fetching QSOs from LoTW...',
|
||||
@@ -196,7 +206,7 @@ export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = n
|
||||
});
|
||||
}
|
||||
|
||||
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
|
||||
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate, syncType);
|
||||
|
||||
// Check for error response from LoTW fetch
|
||||
if (!adifQSOs) {
|
||||
@@ -499,6 +509,27 @@ export async function getLastLoTWQSLDate(userId) {
|
||||
return new Date(`${year}-${month}-${day}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the date of the last LoTW QSO for a user (for incremental sync of all QSOs)
|
||||
*/
|
||||
export async function getLastLoTWQSODate(userId) {
|
||||
const [result] = await db
|
||||
.select({ maxDate: max(qsos.qsoDate) })
|
||||
.from(qsos)
|
||||
.where(eq(qsos.userId, userId));
|
||||
|
||||
if (!result || !result.maxDate) return null;
|
||||
|
||||
const dateStr = result.maxDate;
|
||||
if (!dateStr || dateStr === '') return null;
|
||||
|
||||
const year = dateStr.substring(0, 4);
|
||||
const month = dateStr.substring(4, 6);
|
||||
const day = dateStr.substring(6, 8);
|
||||
|
||||
return new Date(`${year}-${month}-${day}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete all QSOs for a user
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user