revert: remove LoTW sync type support (QSL/QSO delta/full)
This reverts commit 5b78935 which added:
- Sync type parameter (qsl_delta, qsl_full, qso_delta, qso_full)
- getLastLoTWQSODate() function
- Sync type dropdown on QSO page
- Job queue handling of sync types
Reason: LoTW doesn't provide DXCC entity data for unconfirmed QSOs,
which causes award calculation issues. Going back to QSL-only sync.
Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -314,9 +314,8 @@ const app = new Elysia()
|
|||||||
* POST /api/lotw/sync
|
* POST /api/lotw/sync
|
||||||
* Queue a LoTW sync job (requires authentication)
|
* Queue a LoTW sync job (requires authentication)
|
||||||
* Returns immediately with job ID
|
* Returns immediately with job ID
|
||||||
* Body: { syncType?: 'qsl_delta' | 'qsl_full' | 'qso_delta' | 'qso_full' }
|
|
||||||
*/
|
*/
|
||||||
.post('/api/lotw/sync', async ({ user, body, set }) => {
|
.post('/api/lotw/sync', async ({ user, set }) => {
|
||||||
if (!user) {
|
if (!user) {
|
||||||
logger.warn('/api/lotw/sync: Unauthorized access attempt');
|
logger.warn('/api/lotw/sync: Unauthorized access attempt');
|
||||||
set.status = 401;
|
set.status = 401;
|
||||||
@@ -324,8 +323,7 @@ const app = new Elysia()
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { syncType = 'qsl_delta' } = body || {};
|
const result = await enqueueJob(user.id, 'lotw_sync');
|
||||||
const result = await enqueueJob(user.id, 'lotw_sync', { syncType });
|
|
||||||
|
|
||||||
if (!result.success && result.existingJob) {
|
if (!result.success && result.existingJob) {
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -22,13 +22,10 @@ const activeJobs = new Map();
|
|||||||
* Enqueue a new sync job
|
* Enqueue a new sync job
|
||||||
* @param {number} userId - User ID
|
* @param {number} userId - User ID
|
||||||
* @param {string} jobType - Type of job ('lotw_sync' or 'dcl_sync')
|
* @param {string} jobType - Type of job ('lotw_sync' or 'dcl_sync')
|
||||||
* @param {Object} options - Optional job parameters
|
|
||||||
* @param {string} options.syncType - LoTW sync type: 'qsl_delta' (default), 'qsl_full', 'qso_delta', 'qso_full'
|
|
||||||
* @returns {Promise<Object>} Job object with ID
|
* @returns {Promise<Object>} Job object with ID
|
||||||
*/
|
*/
|
||||||
export async function enqueueJob(userId, jobType = 'lotw_sync', options = {}) {
|
export async function enqueueJob(userId, jobType = 'lotw_sync') {
|
||||||
const { syncType = 'qsl_delta' } = options;
|
logger.debug('Enqueueing sync job', { userId, jobType });
|
||||||
logger.debug('Enqueueing sync job', { userId, jobType, syncType });
|
|
||||||
|
|
||||||
// Check for existing active job of the same type
|
// Check for existing active job of the same type
|
||||||
const existingJob = await getUserActiveJob(userId, jobType);
|
const existingJob = await getUserActiveJob(userId, jobType);
|
||||||
@@ -52,10 +49,10 @@ export async function enqueueJob(userId, jobType = 'lotw_sync', options = {}) {
|
|||||||
})
|
})
|
||||||
.returning();
|
.returning();
|
||||||
|
|
||||||
logger.info('Job created', { jobId: job.id, userId, jobType, syncType });
|
logger.info('Job created', { jobId: job.id, userId, jobType });
|
||||||
|
|
||||||
// Start processing asynchronously (don't await)
|
// Start processing asynchronously (don't await)
|
||||||
processJobAsync(job.id, userId, jobType, syncType).catch((error) => {
|
processJobAsync(job.id, userId, jobType).catch((error) => {
|
||||||
logger.error(`Job processing error`, { jobId: job.id, error: error.message });
|
logger.error(`Job processing error`, { jobId: job.id, error: error.message });
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -76,9 +73,8 @@ export async function enqueueJob(userId, jobType = 'lotw_sync', options = {}) {
|
|||||||
* @param {number} jobId - Job ID
|
* @param {number} jobId - Job ID
|
||||||
* @param {number} userId - User ID
|
* @param {number} userId - User ID
|
||||||
* @param {string} jobType - Type of job ('lotw_sync' or 'dcl_sync')
|
* @param {string} jobType - Type of job ('lotw_sync' or 'dcl_sync')
|
||||||
* @param {string} syncType - LoTW sync type: 'qsl_delta', 'qsl_full', 'qso_delta', 'qso_full'
|
|
||||||
*/
|
*/
|
||||||
async function processJobAsync(jobId, userId, jobType, syncType = 'qsl_delta') {
|
async function processJobAsync(jobId, userId, jobType) {
|
||||||
const jobPromise = (async () => {
|
const jobPromise = (async () => {
|
||||||
try {
|
try {
|
||||||
const { getUserById } = await import('./auth.service.js');
|
const { getUserById } = await import('./auth.service.js');
|
||||||
@@ -134,28 +130,15 @@ async function processJobAsync(jobId, userId, jobType, syncType = 'qsl_delta') {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the appropriate date based on sync type
|
// Get last QSL date for incremental sync
|
||||||
const { getLastLoTWQSODate, getLastLoTWQSLDate, syncQSOs } = await import('./lotw.service.js');
|
const { getLastLoTWQSLDate, syncQSOs } = await import('./lotw.service.js');
|
||||||
|
const lastQSLDate = await getLastLoTWQSLDate(userId);
|
||||||
|
const sinceDate = lastQSLDate || new Date('2000-01-01');
|
||||||
|
|
||||||
let sinceDate = null;
|
if (lastQSLDate) {
|
||||||
let dateSource = '';
|
logger.info(`Job ${jobId}: LoTW incremental sync`, { since: sinceDate.toISOString().split('T')[0] });
|
||||||
|
|
||||||
if (syncType.includes('delta')) {
|
|
||||||
// Delta sync: use date filter
|
|
||||||
if (syncType === 'qso_delta') {
|
|
||||||
const lastQSODate = await getLastLoTWQSODate(userId);
|
|
||||||
sinceDate = lastQSODate || new Date('2000-01-01');
|
|
||||||
dateSource = lastQSODate ? 'QSO' : 'full';
|
|
||||||
} else {
|
|
||||||
// qsl_delta
|
|
||||||
const lastQSLDate = await getLastLoTWQSLDate(userId);
|
|
||||||
sinceDate = lastQSLDate || new Date('2000-01-01');
|
|
||||||
dateSource = lastQSLDate ? 'QSL' : 'full';
|
|
||||||
}
|
|
||||||
logger.info(`Job ${jobId}: LoTW ${syncType} sync`, { since: sinceDate.toISOString().split('T')[0], source: dateSource });
|
|
||||||
} else {
|
} else {
|
||||||
// Full sync: no date filter
|
logger.info(`Job ${jobId}: LoTW full sync`);
|
||||||
logger.info(`Job ${jobId}: LoTW ${syncType} full sync`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update job progress
|
// Update job progress
|
||||||
@@ -164,8 +147,8 @@ async function processJobAsync(jobId, userId, jobType, syncType = 'qsl_delta') {
|
|||||||
step: 'fetch',
|
step: 'fetch',
|
||||||
});
|
});
|
||||||
|
|
||||||
// Execute the sync with syncType
|
// Execute the sync
|
||||||
result = await syncQSOs(userId, user.lotwUsername, user.lotwPassword, sinceDate, jobId, syncType);
|
result = await syncQSOs(userId, user.lotwUsername, user.lotwPassword, sinceDate, jobId);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update job as completed
|
// Update job as completed
|
||||||
|
|||||||
@@ -49,24 +49,15 @@ const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Fetch QSOs from LoTW with retry support
|
* Fetch QSOs from LoTW with retry support
|
||||||
* @param {string} lotwUsername - LoTW username
|
|
||||||
* @param {string} lotwPassword - LoTW password
|
|
||||||
* @param {Date|null} sinceDate - Optional date for incremental sync
|
|
||||||
* @param {string} syncType - Type of sync: 'qsl_delta' (default), 'qsl_full', 'qso_delta', 'qso_full'
|
|
||||||
*/
|
*/
|
||||||
async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null, syncType = 'qsl_delta') {
|
async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||||
const url = 'https://lotw.arrl.org/lotwuser/lotwreport.adi';
|
const url = 'https://lotw.arrl.org/lotwuser/lotwreport.adi';
|
||||||
|
|
||||||
// Determine qso_qsl parameter based on sync type
|
|
||||||
// qsl_* = only QSLs (confirmed QSOs)
|
|
||||||
// qso_* = all QSOs (confirmed + unconfirmed)
|
|
||||||
const qsoQslValue = syncType.startsWith('qsl') ? 'yes' : 'no';
|
|
||||||
|
|
||||||
const params = new URLSearchParams({
|
const params = new URLSearchParams({
|
||||||
login: lotwUsername,
|
login: lotwUsername,
|
||||||
password: lotwPassword,
|
password: lotwPassword,
|
||||||
qso_query: '1',
|
qso_query: '1',
|
||||||
qso_qsl: qsoQslValue,
|
qso_qsl: 'yes',
|
||||||
qso_qsldetail: 'yes',
|
qso_qsldetail: 'yes',
|
||||||
qso_mydetail: 'yes',
|
qso_mydetail: 'yes',
|
||||||
qso_withown: 'yes',
|
qso_withown: 'yes',
|
||||||
@@ -75,9 +66,9 @@ async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null, s
|
|||||||
if (sinceDate) {
|
if (sinceDate) {
|
||||||
const dateStr = sinceDate.toISOString().split('T')[0];
|
const dateStr = sinceDate.toISOString().split('T')[0];
|
||||||
params.append('qso_qslsince', dateStr);
|
params.append('qso_qslsince', dateStr);
|
||||||
logger.debug('Incremental sync', { syncType, since: dateStr });
|
logger.debug('Incremental sync since', { date: dateStr });
|
||||||
} else {
|
} else {
|
||||||
logger.debug('Full sync', { syncType });
|
logger.debug('Full sync - fetching all QSOs');
|
||||||
}
|
}
|
||||||
|
|
||||||
const fullUrl = `${url}?${params.toString()}`;
|
const fullUrl = `${url}?${params.toString()}`;
|
||||||
@@ -196,9 +187,8 @@ function convertQSODatabaseFormat(adifQSO, userId) {
|
|||||||
* @param {string} lotwPassword - LoTW password
|
* @param {string} lotwPassword - LoTW password
|
||||||
* @param {Date|null} sinceDate - Optional date for incremental sync
|
* @param {Date|null} sinceDate - Optional date for incremental sync
|
||||||
* @param {number|null} jobId - Optional job ID for progress tracking
|
* @param {number|null} jobId - Optional job ID for progress tracking
|
||||||
* @param {string} syncType - Type of sync: 'qsl_delta' (default), 'qsl_full', 'qso_delta', 'qso_full'
|
|
||||||
*/
|
*/
|
||||||
export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null, jobId = null, syncType = 'qsl_delta') {
|
export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null, jobId = null) {
|
||||||
if (jobId) {
|
if (jobId) {
|
||||||
await updateJobProgress(jobId, {
|
await updateJobProgress(jobId, {
|
||||||
message: 'Fetching QSOs from LoTW...',
|
message: 'Fetching QSOs from LoTW...',
|
||||||
@@ -206,7 +196,7 @@ export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = n
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate, syncType);
|
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
|
||||||
|
|
||||||
// Check for error response from LoTW fetch
|
// Check for error response from LoTW fetch
|
||||||
if (!adifQSOs) {
|
if (!adifQSOs) {
|
||||||
@@ -514,27 +504,6 @@ export async function getLastLoTWQSLDate(userId) {
|
|||||||
return new Date(`${year}-${month}-${day}`);
|
return new Date(`${year}-${month}-${day}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the date of the last LoTW QSO for a user (for incremental sync of all QSOs)
|
|
||||||
*/
|
|
||||||
export async function getLastLoTWQSODate(userId) {
|
|
||||||
const [result] = await db
|
|
||||||
.select({ maxDate: max(qsos.qsoDate) })
|
|
||||||
.from(qsos)
|
|
||||||
.where(eq(qsos.userId, userId));
|
|
||||||
|
|
||||||
if (!result || !result.maxDate) return null;
|
|
||||||
|
|
||||||
const dateStr = result.maxDate;
|
|
||||||
if (!dateStr || dateStr === '') return null;
|
|
||||||
|
|
||||||
const year = dateStr.substring(0, 4);
|
|
||||||
const month = dateStr.substring(4, 6);
|
|
||||||
const day = dateStr.substring(6, 8);
|
|
||||||
|
|
||||||
return new Date(`${year}-${month}-${day}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete all QSOs for a user
|
* Delete all QSOs for a user
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -72,10 +72,7 @@ export const qsosAPI = {
|
|||||||
|
|
||||||
getStats: () => apiRequest('/qsos/stats'),
|
getStats: () => apiRequest('/qsos/stats'),
|
||||||
|
|
||||||
syncFromLoTW: (syncType = 'qsl_delta') => apiRequest('/lotw/sync', {
|
syncFromLoTW: () => apiRequest('/lotw/sync', { method: 'POST' }),
|
||||||
method: 'POST',
|
|
||||||
body: JSON.stringify({ syncType }),
|
|
||||||
}),
|
|
||||||
|
|
||||||
syncFromDCL: () => apiRequest('/dcl/sync', { method: 'POST' }),
|
syncFromDCL: () => apiRequest('/dcl/sync', { method: 'POST' }),
|
||||||
|
|
||||||
|
|||||||
@@ -39,9 +39,6 @@
|
|||||||
let selectedQSO = null;
|
let selectedQSO = null;
|
||||||
let showQSODetailModal = false;
|
let showQSODetailModal = false;
|
||||||
|
|
||||||
// LoTW sync type selection
|
|
||||||
let lotwSyncType = 'qsl_delta'; // Options: 'qsl_delta', 'qsl_full', 'qso_delta', 'qso_full'
|
|
||||||
|
|
||||||
let filters = {
|
let filters = {
|
||||||
band: '',
|
band: '',
|
||||||
mode: '',
|
mode: '',
|
||||||
@@ -229,7 +226,7 @@
|
|||||||
|
|
||||||
async function handleLoTWSync() {
|
async function handleLoTWSync() {
|
||||||
try {
|
try {
|
||||||
const response = await qsosAPI.syncFromLoTW(lotwSyncType);
|
const response = await qsosAPI.syncFromLoTW();
|
||||||
|
|
||||||
if (response.jobId) {
|
if (response.jobId) {
|
||||||
startLoTWPolling(response.jobId);
|
startLoTWPolling(response.jobId);
|
||||||
@@ -390,18 +387,6 @@
|
|||||||
</button>
|
</button>
|
||||||
{/if}
|
{/if}
|
||||||
|
|
||||||
<!-- LoTW sync type selector -->
|
|
||||||
<select
|
|
||||||
class="sync-type-select"
|
|
||||||
bind:value={lotwSyncType}
|
|
||||||
disabled={lotwSyncStatus === 'running' || lotwSyncStatus === 'pending' || deleting}
|
|
||||||
>
|
|
||||||
<option value="qsl_delta">LoTW: QSL Delta</option>
|
|
||||||
<option value="qsl_full">LoTW: QSL Full</option>
|
|
||||||
<option value="qso_delta">LoTW: QSO Delta</option>
|
|
||||||
<option value="qso_full">LoTW: QSO Full</option>
|
|
||||||
</select>
|
|
||||||
|
|
||||||
<SyncButton
|
<SyncButton
|
||||||
service="lotw"
|
service="lotw"
|
||||||
syncStatus={lotwSyncStatus}
|
syncStatus={lotwSyncStatus}
|
||||||
@@ -938,25 +923,6 @@
|
|||||||
flex-wrap: wrap;
|
flex-wrap: wrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.sync-type-select {
|
|
||||||
padding: 0.75rem 1rem;
|
|
||||||
border: 1px solid #ddd;
|
|
||||||
border-radius: 4px;
|
|
||||||
background-color: white;
|
|
||||||
font-size: 1rem;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: border-color 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.sync-type-select:hover:not(:disabled) {
|
|
||||||
border-color: #4a90e2;
|
|
||||||
}
|
|
||||||
|
|
||||||
.sync-type-select:disabled {
|
|
||||||
opacity: 0.5;
|
|
||||||
cursor: not-allowed;
|
|
||||||
}
|
|
||||||
|
|
||||||
.filters {
|
.filters {
|
||||||
background: #f8f9fa;
|
background: #f8f9fa;
|
||||||
padding: 1rem;
|
padding: 1rem;
|
||||||
|
|||||||
Reference in New Issue
Block a user