refactor: simplify codebase and replace external dependencies with Bun built-ins
Backend changes: - Merge duplicate award logic (calculatePointsAwardProgress + getPointsAwardEntityBreakdown) - Simplify LoTW service (merge syncQSOs functions, simplify polling) - Remove job queue abstraction (hardcode LoTW sync, remove processor registry) - Consolidate config files (database.js, logger.js, jwt.js → single config.js) - Replace bcrypt with Bun.password.hash/verify - Replace Pino logger with console-based logger - Fix: export syncQSOs and getLastLoTWQSLDate for job queue imports - Fix: correct database path resolution using new URL() Frontend changes: - Simplify auth store (remove localStorage wrappers, reduce from 222→109 lines) - Consolidate API layer (remove verbose JSDoc, 180→80 lines) - Add shared UI components (Loading, ErrorDisplay, BackButton) Dependencies: - Remove bcrypt (replaced with Bun.password) - Remove pino and pino-pretty (replaced with console logger) Total: ~445 lines removed (net), 3 dependencies removed Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,24 +1,17 @@
|
||||
import { db } from '../config/database.js';
|
||||
import { db, logger } from '../config.js';
|
||||
import { qsos } from '../db/schema/index.js';
|
||||
import { max, sql, eq, and, desc } from 'drizzle-orm';
|
||||
import { registerProcessor, updateJobProgress } from './job-queue.service.js';
|
||||
import logger from '../config/logger.js';
|
||||
import { updateJobProgress } from './job-queue.service.js';
|
||||
|
||||
/**
|
||||
* LoTW (Logbook of the World) Service
|
||||
* Fetches QSOs from ARRL's LoTW system
|
||||
*/
|
||||
|
||||
// Wavelog-compatible constants
|
||||
const LOTW_CONNECT_TIMEOUT = 30;
|
||||
|
||||
// Configuration for long-polling
|
||||
const POLLING_CONFIG = {
|
||||
maxRetries: 30,
|
||||
retryDelay: 10000,
|
||||
requestTimeout: 60000,
|
||||
maxTotalTime: 600000,
|
||||
};
|
||||
// Simplified polling configuration
|
||||
const MAX_RETRIES = 30;
|
||||
const RETRY_DELAY = 10000;
|
||||
const REQUEST_TIMEOUT = 60000;
|
||||
|
||||
/**
|
||||
* Check if LoTW response indicates the report is still being prepared
|
||||
@@ -53,7 +46,7 @@ function isReportPending(responseData) {
|
||||
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
/**
|
||||
* Fetch QSOs from LoTW with long-polling support
|
||||
* Fetch QSOs from LoTW with retry support
|
||||
*/
|
||||
async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||
const url = 'https://lotw.arrl.org/lotwuser/lotwreport.adi';
|
||||
@@ -79,23 +72,14 @@ async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||
const fullUrl = `${url}?${params.toString()}`;
|
||||
logger.debug('Fetching from LoTW', { url: fullUrl.replace(/password=[^&]+/, 'password=***') });
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
for (let attempt = 0; attempt < POLLING_CONFIG.maxRetries; attempt++) {
|
||||
const elapsed = Date.now() - startTime;
|
||||
if (elapsed > POLLING_CONFIG.maxTotalTime) {
|
||||
return {
|
||||
error: `LoTW sync timeout: exceeded maximum wait time of ${POLLING_CONFIG.maxTotalTime / 1000} seconds`
|
||||
};
|
||||
}
|
||||
|
||||
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
||||
if (attempt > 0) {
|
||||
logger.debug(`Retry attempt ${attempt + 1}/${POLLING_CONFIG.maxRetries}`, { elapsed: Math.round(elapsed / 1000) });
|
||||
logger.debug(`Retry attempt ${attempt + 1}/${MAX_RETRIES}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), POLLING_CONFIG.requestTimeout);
|
||||
const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT);
|
||||
|
||||
const response = await fetch(fullUrl, { signal: controller.signal });
|
||||
clearTimeout(timeoutId);
|
||||
@@ -103,7 +87,7 @@ async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||
if (!response.ok) {
|
||||
if (response.status === 503) {
|
||||
logger.warn('LoTW returned 503, retrying...');
|
||||
await sleep(POLLING_CONFIG.retryDelay);
|
||||
await sleep(RETRY_DELAY);
|
||||
continue;
|
||||
} else if (response.status === 401) {
|
||||
return { error: 'Invalid LoTW credentials. Please check your username and password in Settings.' };
|
||||
@@ -111,7 +95,7 @@ async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||
return { error: 'LoTW service not found (404). The LoTW API URL may have changed.' };
|
||||
} else {
|
||||
logger.warn(`LoTW returned ${response.status}, retrying...`);
|
||||
await sleep(POLLING_CONFIG.retryDelay);
|
||||
await sleep(RETRY_DELAY);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -126,10 +110,9 @@ async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||
if (!header.includes('arrl logbook of the world')) {
|
||||
if (isReportPending(adifData)) {
|
||||
logger.debug('LoTW report still being prepared, waiting...');
|
||||
await sleep(POLLING_CONFIG.retryDelay);
|
||||
await sleep(RETRY_DELAY);
|
||||
continue;
|
||||
}
|
||||
|
||||
return { error: 'Downloaded LoTW report is invalid. Check your credentials.' };
|
||||
}
|
||||
|
||||
@@ -143,7 +126,7 @@ async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||
} catch (error) {
|
||||
if (error.name === 'AbortError') {
|
||||
logger.debug('Request timeout, retrying...');
|
||||
await sleep(POLLING_CONFIG.retryDelay);
|
||||
await sleep(RETRY_DELAY);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -151,9 +134,9 @@ async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (attempt < POLLING_CONFIG.maxRetries - 1) {
|
||||
if (attempt < MAX_RETRIES - 1) {
|
||||
logger.warn(`Error on attempt ${attempt + 1}`, { error: error.message });
|
||||
await sleep(POLLING_CONFIG.retryDelay);
|
||||
await sleep(RETRY_DELAY);
|
||||
continue;
|
||||
} else {
|
||||
throw error;
|
||||
@@ -161,9 +144,9 @@ async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||
}
|
||||
}
|
||||
|
||||
const totalTime = Math.round((Date.now() - startTime) / 1000);
|
||||
const totalTime = Math.round((Date.now() - Date.now()) / 1000);
|
||||
return {
|
||||
error: `LoTW sync failed: Report not ready after ${POLLING_CONFIG.maxRetries} attempts (${totalTime}s). LoTW may be experiencing high load. Please try again later.`
|
||||
error: `LoTW sync failed: Report not ready after ${MAX_RETRIES} attempts (${totalTime}s). LoTW may be experiencing high load. Please try again later.`
|
||||
};
|
||||
}
|
||||
|
||||
@@ -259,30 +242,67 @@ function normalizeMode(mode) {
|
||||
|
||||
/**
|
||||
* Sync QSOs from LoTW to database
|
||||
* @param {number} userId - User ID
|
||||
* @param {string} lotwUsername - LoTW username
|
||||
* @param {string} lotwPassword - LoTW password
|
||||
* @param {Date|null} sinceDate - Optional date for incremental sync
|
||||
* @param {number|null} jobId - Optional job ID for progress tracking
|
||||
*/
|
||||
async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null) {
|
||||
export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null, jobId = null) {
|
||||
if (jobId) {
|
||||
await updateJobProgress(jobId, {
|
||||
message: 'Fetching QSOs from LoTW...',
|
||||
step: 'fetch',
|
||||
});
|
||||
}
|
||||
|
||||
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
|
||||
|
||||
if (!adifQSOs || adifQSOs.length === 0) {
|
||||
// Check for error response from LoTW fetch
|
||||
if (!adifQSOs) {
|
||||
return { success: false, error: 'Failed to fetch from LoTW', total: 0, added: 0, updated: 0 };
|
||||
}
|
||||
|
||||
// If adifQSOs is an error object, throw it
|
||||
if (adifQSOs.error) {
|
||||
throw new Error(adifQSOs.error);
|
||||
}
|
||||
|
||||
if (adifQSOs.length === 0) {
|
||||
return { success: true, total: 0, added: 0, updated: 0, message: 'No QSOs found in LoTW' };
|
||||
}
|
||||
|
||||
if (jobId) {
|
||||
await updateJobProgress(jobId, {
|
||||
message: `Processing ${adifQSOs.length} QSOs...`,
|
||||
step: 'process',
|
||||
total: adifQSOs.length,
|
||||
processed: 0,
|
||||
});
|
||||
}
|
||||
|
||||
let addedCount = 0;
|
||||
let updatedCount = 0;
|
||||
const errors = [];
|
||||
|
||||
for (const qsoData of adifQSOs) {
|
||||
for (let i = 0; i < adifQSOs.length; i++) {
|
||||
const qsoData = adifQSOs[i];
|
||||
|
||||
try {
|
||||
const dbQSO = convertQSODatabaseFormat(qsoData, userId);
|
||||
|
||||
const existing = await db
|
||||
.select()
|
||||
.from(qsos)
|
||||
.where(eq(qsos.userId, userId))
|
||||
.where(eq(qsos.callsign, dbQSO.callsign))
|
||||
.where(eq(qsos.qsoDate, dbQSO.qsoDate))
|
||||
.where(eq(qsos.band, dbQSO.band))
|
||||
.where(eq(qsos.mode, dbQSO.mode))
|
||||
.where(
|
||||
and(
|
||||
eq(qsos.userId, userId),
|
||||
eq(qsos.callsign, dbQSO.callsign),
|
||||
eq(qsos.qsoDate, dbQSO.qsoDate),
|
||||
eq(qsos.band, dbQSO.band),
|
||||
eq(qsos.mode, dbQSO.mode)
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
if (existing.length > 0) {
|
||||
@@ -299,13 +319,21 @@ async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null) {
|
||||
await db.insert(qsos).values(dbQSO);
|
||||
addedCount++;
|
||||
}
|
||||
|
||||
// Update job progress every 10 QSOs
|
||||
if (jobId && (i + 1) % 10 === 0) {
|
||||
await updateJobProgress(jobId, {
|
||||
processed: i + 1,
|
||||
message: `Processed ${i + 1}/${adifQSOs.length} QSOs...`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error processing QSO', { error: error.message, qso: qsoData });
|
||||
logger.error('Error processing QSO', { error: error.message, jobId, qso: qsoData });
|
||||
errors.push({ qso: qsoData, error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('LoTW sync completed', { total: adifQSOs.length, added: addedCount, updated: updatedCount });
|
||||
logger.info('LoTW sync completed', { total: adifQSOs.length, added: addedCount, updated: updatedCount, jobId });
|
||||
|
||||
return {
|
||||
success: true,
|
||||
@@ -383,7 +411,7 @@ export async function getQSOStats(userId) {
|
||||
/**
|
||||
* Get the date of the last LoTW QSL for a user
|
||||
*/
|
||||
async function getLastLoTWQSLDate(userId) {
|
||||
export async function getLastLoTWQSLDate(userId) {
|
||||
const [result] = await db
|
||||
.select({ maxDate: max(qsos.lotwQslRdate) })
|
||||
.from(qsos)
|
||||
@@ -401,101 +429,6 @@ async function getLastLoTWQSLDate(userId) {
|
||||
return new Date(`${year}-${month}-${day}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* LoTW sync job processor for the job queue
|
||||
*/
|
||||
export async function syncQSOsForJob(jobId, userId, data) {
|
||||
const { lotwUsername, lotwPassword } = data;
|
||||
|
||||
await updateJobProgress(jobId, {
|
||||
message: 'Fetching QSOs from LoTW...',
|
||||
step: 'fetch',
|
||||
});
|
||||
|
||||
const lastQSLDate = await getLastLoTWQSLDate(userId);
|
||||
const sinceDate = lastQSLDate || new Date('2000-01-01');
|
||||
|
||||
if (lastQSLDate) {
|
||||
logger.info(`Job ${jobId}: Incremental sync`, { since: sinceDate.toISOString().split('T')[0] });
|
||||
} else {
|
||||
logger.info(`Job ${jobId}: Full sync`);
|
||||
}
|
||||
|
||||
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
|
||||
|
||||
if (!adifQSOs || adifQSOs.length === 0) {
|
||||
return { success: true, total: 0, added: 0, updated: 0, message: 'No QSOs found in LoTW' };
|
||||
}
|
||||
|
||||
await updateJobProgress(jobId, {
|
||||
message: `Processing ${adifQSOs.length} QSOs...`,
|
||||
step: 'process',
|
||||
total: adifQSOs.length,
|
||||
processed: 0,
|
||||
});
|
||||
|
||||
let addedCount = 0;
|
||||
let updatedCount = 0;
|
||||
const errors = [];
|
||||
|
||||
for (let i = 0; i < adifQSOs.length; i++) {
|
||||
const qsoData = adifQSOs[i];
|
||||
|
||||
try {
|
||||
const dbQSO = convertQSODatabaseFormat(qsoData, userId);
|
||||
|
||||
const existing = await db
|
||||
.select()
|
||||
.from(qsos)
|
||||
.where(
|
||||
and(
|
||||
eq(qsos.userId, userId),
|
||||
eq(qsos.callsign, dbQSO.callsign),
|
||||
eq(qsos.qsoDate, dbQSO.qsoDate),
|
||||
eq(qsos.band, dbQSO.band),
|
||||
eq(qsos.mode, dbQSO.mode)
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
if (existing.length > 0) {
|
||||
await db
|
||||
.update(qsos)
|
||||
.set({
|
||||
lotwQslRdate: dbQSO.lotwQslRdate,
|
||||
lotwQslRstatus: dbQSO.lotwQslRstatus,
|
||||
lotwSyncedAt: dbQSO.lotwSyncedAt,
|
||||
})
|
||||
.where(eq(qsos.id, existing[0].id));
|
||||
updatedCount++;
|
||||
} else {
|
||||
await db.insert(qsos).values(dbQSO);
|
||||
addedCount++;
|
||||
}
|
||||
|
||||
if ((i + 1) % 10 === 0) {
|
||||
await updateJobProgress(jobId, {
|
||||
processed: i + 1,
|
||||
message: `Processed ${i + 1}/${adifQSOs.length} QSOs...`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Job ${jobId}: Error processing QSO`, { error: error.message });
|
||||
errors.push({ qso: qsoData, error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Job ${jobId} completed`, { total: adifQSOs.length, added: addedCount, updated: updatedCount });
|
||||
|
||||
return {
|
||||
success: true,
|
||||
total: adifQSOs.length,
|
||||
added: addedCount,
|
||||
updated: updatedCount,
|
||||
errors: errors.length > 0 ? errors : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete all QSOs for a user
|
||||
*/
|
||||
@@ -504,5 +437,3 @@ export async function deleteQSOs(userId) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Register the LoTW sync processor with the job queue
|
||||
registerProcessor('lotw_sync', syncQSOsForJob);
|
||||
|
||||
Reference in New Issue
Block a user