Add structured logging, navigation bar, and code cleanup
## Backend - Add Pino logging framework with timestamps and structured output - Replace all console.error statements (49+) with proper logging levels - Fix Drizzle ORM bug: replace invalid .get() calls with .limit(1) - Remove unused auth routes file (already in index.js) - Make internal functions private (remove unnecessary exports) - Simplify code by removing excessive debug logging ## Frontend - Add navigation bar to layout with: - User's callsign display - Navigation links (Dashboard, QSOs, Settings) - Logout button with red color distinction - Navigation only shows when user is logged in - Dark themed design matching footer ## Documentation - Update README.md with new project structure - Update docs/DOCUMENTATION.md with logging and nav bar info - Add logger.js to configuration section Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
import { db } from '../config/database.js';
|
||||
import { qsos } from '../db/schema/index.js';
|
||||
import { max, sql } from 'drizzle-orm';
|
||||
import { max, sql, eq, and, desc } from 'drizzle-orm';
|
||||
import { registerProcessor, updateJobProgress } from './job-queue.service.js';
|
||||
import logger from '../config/logger.js';
|
||||
|
||||
/**
|
||||
* LoTW (Logbook of the World) Service
|
||||
@@ -9,41 +10,25 @@ import { registerProcessor, updateJobProgress } from './job-queue.service.js';
|
||||
*/
|
||||
|
||||
// Wavelog-compatible constants
|
||||
const LOTW_CONNECT_TIMEOUT = 30; // CURLOPT_CONNECTTIMEOUT from Wavelog
|
||||
const LOTW_CONNECT_TIMEOUT = 30;
|
||||
|
||||
// Configuration for long-polling
|
||||
const POLLING_CONFIG = {
|
||||
maxRetries: 30, // Maximum number of retry attempts
|
||||
retryDelay: 10000, // Delay between retries in ms (10 seconds)
|
||||
requestTimeout: 60000, // Timeout for individual requests in ms (1 minute)
|
||||
maxTotalTime: 600000, // Maximum total time to wait in ms (10 minutes)
|
||||
maxRetries: 30,
|
||||
retryDelay: 10000,
|
||||
requestTimeout: 60000,
|
||||
maxTotalTime: 600000,
|
||||
};
|
||||
|
||||
/**
|
||||
* Check if LoTW response indicates the report is still being prepared
|
||||
* @param {string} responseData - The response text from LoTW
|
||||
* @returns {boolean} True if report is still pending
|
||||
*/
|
||||
function isReportPending(responseData) {
|
||||
const trimmed = responseData.trim().toLowerCase();
|
||||
|
||||
// LoTW returns various messages when report is not ready:
|
||||
// - Empty responses
|
||||
// - "Report is being prepared" or similar messages
|
||||
// - HTML error pages
|
||||
// - Very short responses that aren't valid ADIF
|
||||
if (trimmed.length < 100) return true;
|
||||
if (trimmed.includes('<html>') || trimmed.includes('<!doctype html>')) return true;
|
||||
|
||||
// Check for empty or very short responses
|
||||
if (trimmed.length < 100) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for HTML responses (error pages)
|
||||
if (trimmed.includes('<html>') || trimmed.includes('<!doctype html>')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for common "not ready" messages
|
||||
const pendingMessages = [
|
||||
'report is being prepared',
|
||||
'your report is being generated',
|
||||
@@ -54,12 +39,9 @@ function isReportPending(responseData) {
|
||||
];
|
||||
|
||||
for (const msg of pendingMessages) {
|
||||
if (trimmed.includes(msg)) {
|
||||
return true;
|
||||
}
|
||||
if (trimmed.includes(msg)) return true;
|
||||
}
|
||||
|
||||
// Check if it looks like valid ADIF data (should start with <ADIF_VER: or have <qso_date)
|
||||
const hasAdifHeader = trimmed.includes('<adif_ver:') ||
|
||||
trimmed.includes('<qso_date') ||
|
||||
trimmed.includes('<call:') ||
|
||||
@@ -68,79 +50,57 @@ function isReportPending(responseData) {
|
||||
return !hasAdifHeader;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleep/delay utility
|
||||
* @param {number} ms - Milliseconds to sleep
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
/**
|
||||
* Fetch QSOs from LoTW with long-polling support
|
||||
* @param {string} lotwUsername - LoTW username
|
||||
* @param {string} lotwPassword - LoTW password
|
||||
* @param {Date} sinceDate - Only fetch QSOs since this date
|
||||
* @returns {Promise<Array>} Array of QSO objects
|
||||
*/
|
||||
export async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||
// LoTW report URL
|
||||
async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
||||
const url = 'https://lotw.arrl.org/lotwuser/lotwreport.adi';
|
||||
|
||||
// Build query parameters - qso_query=1 is REQUIRED to get QSO records!
|
||||
const params = new URLSearchParams({
|
||||
login: lotwUsername,
|
||||
password: lotwPassword,
|
||||
qso_query: '1', // REQUIRED: Without this, no QSO records are returned
|
||||
qso_qsl: 'yes', // Only get QSOs with QSLs (confirmed)
|
||||
qso_qsldetail: 'yes', // Include QSL details (station location)
|
||||
qso_mydetail: 'yes', // Include my station details
|
||||
qso_withown: 'yes', // Include own callsign
|
||||
qso_query: '1',
|
||||
qso_qsl: 'yes',
|
||||
qso_qsldetail: 'yes',
|
||||
qso_mydetail: 'yes',
|
||||
qso_withown: 'yes',
|
||||
});
|
||||
|
||||
// Add date filter - only add qso_qslsince if we have a last QSL date
|
||||
// For first sync (no QSOs in DB), don't filter by date to get ALL QSOs
|
||||
if (sinceDate) {
|
||||
const dateStr = sinceDate.toISOString().split('T')[0];
|
||||
params.append('qso_qslsince', dateStr);
|
||||
console.error('Date filter:', dateStr, '(Incremental sync since last QSL date)');
|
||||
logger.debug('Incremental sync since', { date: dateStr });
|
||||
} else {
|
||||
console.error('No date filter - fetching ALL QSOs (first sync)');
|
||||
logger.debug('Full sync - fetching all QSOs');
|
||||
}
|
||||
|
||||
const fullUrl = `${url}?${params.toString()}`;
|
||||
|
||||
console.error('Fetching from LoTW:', fullUrl.replace(/password=[^&]+/, 'password=***'));
|
||||
logger.debug('Fetching from LoTW', { url: fullUrl.replace(/password=[^&]+/, 'password=***') });
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
// Long-polling loop
|
||||
for (let attempt = 0; attempt < POLLING_CONFIG.maxRetries; attempt++) {
|
||||
const elapsed = Date.now() - startTime;
|
||||
if (elapsed > POLLING_CONFIG.maxTotalTime) {
|
||||
throw new Error(`LoTW sync timeout: exceeded maximum wait time of ${POLLING_CONFIG.maxTotalTime / 1000} seconds`);
|
||||
}
|
||||
|
||||
if (attempt > 0) {
|
||||
logger.debug(`Retry attempt ${attempt + 1}/${POLLING_CONFIG.maxRetries}`, { elapsed: Math.round(elapsed / 1000) });
|
||||
}
|
||||
|
||||
try {
|
||||
// Check if we've exceeded max total time
|
||||
const elapsed = Date.now() - startTime;
|
||||
if (elapsed > POLLING_CONFIG.maxTotalTime) {
|
||||
throw new Error(`LoTW sync timeout: exceeded maximum wait time of ${POLLING_CONFIG.maxTotalTime / 1000} seconds`);
|
||||
}
|
||||
|
||||
if (attempt > 0) {
|
||||
console.error(`Retry attempt ${attempt + 1}/${POLLING_CONFIG.maxRetries} (elapsed: ${Math.round(elapsed / 1000)}s)`);
|
||||
}
|
||||
|
||||
// Make request with timeout
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), POLLING_CONFIG.requestTimeout);
|
||||
|
||||
const response = await fetch(fullUrl, { signal: controller.signal });
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
// Handle HTTP errors
|
||||
if (!response.ok) {
|
||||
if (response.status === 503) {
|
||||
// Service unavailable - might be temporary, retry
|
||||
console.error('LoTW returned 503 (Service Unavailable), waiting before retry...');
|
||||
logger.warn('LoTW returned 503, retrying...');
|
||||
await sleep(POLLING_CONFIG.retryDelay);
|
||||
continue;
|
||||
} else if (response.status === 401) {
|
||||
@@ -148,62 +108,48 @@ export async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate =
|
||||
} else if (response.status === 404) {
|
||||
throw new Error('LoTW service not found (404). The LoTW API URL may have changed.');
|
||||
} else {
|
||||
// Other errors - log but retry
|
||||
console.error(`LoTW returned ${response.status} ${response.statusText}, waiting before retry...`);
|
||||
logger.warn(`LoTW returned ${response.status}, retrying...`);
|
||||
await sleep(POLLING_CONFIG.retryDelay);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Get response text
|
||||
const adifData = await response.text();
|
||||
console.error(`Response length: ${adifData.length} bytes`);
|
||||
|
||||
// Wavelog: Validate response for credential errors
|
||||
if (adifData.toLowerCase().includes('username/password incorrect')) {
|
||||
throw new Error('Username/password incorrect');
|
||||
}
|
||||
|
||||
// Wavelog: Check if file starts with expected header
|
||||
const header = adifData.trim().substring(0, 39).toLowerCase();
|
||||
if (!header.includes('arrl logbook of the world')) {
|
||||
// This might be because the report is still pending
|
||||
if (isReportPending(adifData)) {
|
||||
console.error('LoTW report is still being prepared, waiting...', adifData.substring(0, 100));
|
||||
logger.debug('LoTW report still being prepared, waiting...');
|
||||
await sleep(POLLING_CONFIG.retryDelay);
|
||||
continue;
|
||||
}
|
||||
throw new Error('Downloaded LoTW report is invalid. Check your credentials.');
|
||||
}
|
||||
|
||||
// We have valid data!
|
||||
console.error('LoTW report ready, parsing ADIF data...');
|
||||
console.error('ADIF preview:', adifData.substring(0, 200));
|
||||
logger.info('LoTW report downloaded successfully', { size: adifData.length });
|
||||
|
||||
// Parse ADIF format
|
||||
const qsos = parseADIF(adifData);
|
||||
console.error(`Successfully parsed ${qsos.length} QSOs from LoTW`);
|
||||
logger.info('Parsed QSOs from LoTW', { count: qsos.length });
|
||||
|
||||
return qsos;
|
||||
|
||||
} catch (error) {
|
||||
const elapsed = Date.now() - startTime;
|
||||
|
||||
if (error.name === 'AbortError') {
|
||||
console.error(`Request timeout on attempt ${attempt + 1}, retrying...`);
|
||||
logger.debug('Request timeout, retrying...');
|
||||
await sleep(POLLING_CONFIG.retryDelay);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Re-throw credential/auth errors immediately
|
||||
if (error.message.includes('credentials') || error.message.includes('401') || error.message.includes('404')) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// For other errors, log and retry if we haven't exhausted retries
|
||||
if (attempt < POLLING_CONFIG.maxRetries - 1) {
|
||||
console.error(`Error on attempt ${attempt + 1}: ${error.message}`);
|
||||
console.error('Retrying...');
|
||||
logger.warn(`Error on attempt ${attempt + 1}`, { error: error.message });
|
||||
await sleep(POLLING_CONFIG.retryDelay);
|
||||
continue;
|
||||
} else {
|
||||
@@ -212,73 +158,47 @@ export async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate =
|
||||
}
|
||||
}
|
||||
|
||||
// If we get here, we exhausted all retries
|
||||
const totalTime = Math.round((Date.now() - startTime) / 1000);
|
||||
throw new Error(`LoTW sync failed: Report not ready after ${POLLING_CONFIG.maxRetries} attempts (${totalTime}s). LoTW may be experiencing high load. Please try again later.`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse ADIF (Amateur Data Interchange Format) data
|
||||
* @param {string} adifData - Raw ADIF text data
|
||||
* @returns {Array<Array>} Array of QSOs (each QSO is an array of field objects)
|
||||
*/
|
||||
function parseADIF(adifData) {
|
||||
const qsos = [];
|
||||
const records = adifData.split('<eor>');
|
||||
|
||||
console.error(`Total records after splitting by <eor>: ${records.length}`);
|
||||
|
||||
for (let i = 0; i < records.length; i++) {
|
||||
const record = records[i];
|
||||
|
||||
// Skip empty records or records that are just header info
|
||||
for (const record of records) {
|
||||
if (!record.trim()) continue;
|
||||
if (record.trim().startsWith('<') && !record.includes('<CALL:') && !record.includes('<call:')) continue;
|
||||
|
||||
const qso = {};
|
||||
|
||||
// Match ADIF fields: <FIELDNAME:length[:type]>value
|
||||
// Important: The 'type' part is optional, and field names can contain underscores
|
||||
// We use the length parameter to extract exactly that many characters
|
||||
const regex = /<([A-Z_]+):(\d+)(?::[A-Z]+)?>([\s\S])/gi;
|
||||
let match;
|
||||
|
||||
while ((match = regex.exec(record)) !== null) {
|
||||
const [fullMatch, fieldName, lengthStr, firstChar] = match;
|
||||
const length = parseInt(lengthStr, 10);
|
||||
|
||||
// Extract exactly 'length' characters starting from the position after the '>'
|
||||
const valueStart = match.index + fullMatch.length - 1; // -1 because firstChar is already captured
|
||||
const valueStart = match.index + fullMatch.length - 1;
|
||||
const value = record.substring(valueStart, valueStart + length);
|
||||
|
||||
qso[fieldName.toLowerCase()] = value.trim();
|
||||
|
||||
// Move regex lastIndex to the end of the value so we can find the next tag
|
||||
regex.lastIndex = valueStart + length;
|
||||
}
|
||||
|
||||
// Only add if we have actual QSO data (has CALL or call field)
|
||||
if (Object.keys(qso).length > 0 && (qso.call || qso.call)) {
|
||||
qsos.push(qso);
|
||||
|
||||
// Log first few QSOs for debugging
|
||||
if (qsos.length <= 3) {
|
||||
console.error(`Parsed QSO #${qsos.length}: ${qso.call} on ${qso.qso_date} ${qso.band} ${qso.mode}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.error(`Total QSOs parsed: ${qsos.length}`);
|
||||
return qsos;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert ADIF QSO to database format
|
||||
* @param {Object} adifQSO - QSO object from ADIF parser
|
||||
* @param {number} userId - User ID
|
||||
* @returns {Object} Database-ready QSO object
|
||||
*/
|
||||
export function convertQSODatabaseFormat(adifQSO, userId) {
|
||||
function convertQSODatabaseFormat(adifQSO, userId) {
|
||||
return {
|
||||
userId,
|
||||
callsign: adifQSO.call || '',
|
||||
@@ -303,66 +223,29 @@ export function convertQSODatabaseFormat(adifQSO, userId) {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize band name
|
||||
* @param {string} band - Band from ADIF
|
||||
* @returns {string|null} Normalized band
|
||||
*/
|
||||
function normalizeBand(band) {
|
||||
if (!band) return null;
|
||||
|
||||
const bandMap = {
|
||||
'160m': '160m',
|
||||
'80m': '80m',
|
||||
'60m': '60m',
|
||||
'40m': '40m',
|
||||
'30m': '30m',
|
||||
'20m': '20m',
|
||||
'17m': '17m',
|
||||
'15m': '15m',
|
||||
'12m': '12m',
|
||||
'10m': '10m',
|
||||
'6m': '6m',
|
||||
'4m': '4m',
|
||||
'2m': '2m',
|
||||
'1.25m': '1.25m',
|
||||
'70cm': '70cm',
|
||||
'33cm': '33cm',
|
||||
'23cm': '23cm',
|
||||
'13cm': '13cm',
|
||||
'9cm': '9cm',
|
||||
'6cm': '6cm',
|
||||
'3cm': '3cm',
|
||||
'1.2cm': '1.2cm',
|
||||
'mm': 'mm',
|
||||
'160m': '160m', '80m': '80m', '60m': '60m', '40m': '40m',
|
||||
'30m': '30m', '20m': '20m', '17m': '17m', '15m': '15m',
|
||||
'12m': '12m', '10m': '10m', '6m': '6m', '4m': '4m',
|
||||
'2m': '2m', '1.25m': '1.25m', '70cm': '70cm', '33cm': '33cm',
|
||||
'23cm': '23cm', '13cm': '13cm', '9cm': '9cm', '6cm': '6cm',
|
||||
'3cm': '3cm', '1.2cm': '1.2cm', 'mm': 'mm',
|
||||
};
|
||||
|
||||
return bandMap[band.toLowerCase()] || band;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize mode name
|
||||
* @param {string} mode - Mode from ADIF
|
||||
* @returns {string} Normalized mode
|
||||
*/
|
||||
function normalizeMode(mode) {
|
||||
if (!mode) return '';
|
||||
|
||||
const modeMap = {
|
||||
'cw': 'CW',
|
||||
'ssb': 'SSB',
|
||||
'am': 'AM',
|
||||
'fm': 'FM',
|
||||
'rtty': 'RTTY',
|
||||
'psk31': 'PSK31',
|
||||
'psk63': 'PSK63',
|
||||
'ft8': 'FT8',
|
||||
'ft4': 'FT4',
|
||||
'jt65': 'JT65',
|
||||
'jt9': 'JT9',
|
||||
'js8': 'JS8',
|
||||
'mfsk': 'MFSK',
|
||||
' Olivia': 'OLIVIA',
|
||||
'cw': 'CW', 'ssb': 'SSB', 'am': 'AM', 'fm': 'FM',
|
||||
'rtty': 'RTTY', 'psk31': 'PSK31', 'psk63': 'PSK63',
|
||||
'ft8': 'FT8', 'ft4': 'FT4', 'jt65': 'JT65', 'jt9': 'JT9',
|
||||
'js8': 'JS8', 'mfsk': 'MFSK', 'olivia': 'OLIVIA',
|
||||
};
|
||||
|
||||
const normalized = modeMap[mode.toLowerCase()];
|
||||
@@ -371,139 +254,80 @@ function normalizeMode(mode) {
|
||||
|
||||
/**
|
||||
* Sync QSOs from LoTW to database
|
||||
* @param {number} userId - User ID
|
||||
* @param {string} lotwUsername - LoTW username
|
||||
* @param {string} lotwPassword - LoTW password
|
||||
* @param {Date} sinceDate - Only sync QSOs since this date
|
||||
* @returns {Promise<Object>} Sync result with counts
|
||||
*/
|
||||
export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null) {
|
||||
try {
|
||||
// Fetch QSOs from LoTW
|
||||
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
|
||||
async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null) {
|
||||
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
|
||||
|
||||
if (!adifQSOs || adifQSOs.length === 0) {
|
||||
return {
|
||||
success: true,
|
||||
total: 0,
|
||||
added: 0,
|
||||
updated: 0,
|
||||
message: 'No QSOs found in LoTW',
|
||||
};
|
||||
}
|
||||
|
||||
let addedCount = 0;
|
||||
let updatedCount = 0;
|
||||
const errors = [];
|
||||
|
||||
// Process each QSO
|
||||
for (const qsoData of adifQSOs) {
|
||||
try {
|
||||
console.error('Raw ADIF QSO data:', JSON.stringify(qsoData));
|
||||
const dbQSO = convertQSODatabaseFormat(qsoData, userId);
|
||||
console.error('Converted QSO:', JSON.stringify(dbQSO));
|
||||
console.error('Processing QSO:', dbQSO.callsign, dbQSO.qsoDate, dbQSO.band, dbQSO.mode);
|
||||
|
||||
// Check if QSO already exists (by callsign, date, time, band, mode)
|
||||
const { eq } = await import('drizzle-orm');
|
||||
const existing = await db
|
||||
.select()
|
||||
.from(qsos)
|
||||
.where(eq(qsos.userId, userId))
|
||||
.where(eq(qsos.callsign, dbQSO.callsign))
|
||||
.where(eq(qsos.qsoDate, dbQSO.qsoDate))
|
||||
.where(eq(qsos.band, dbQSO.band))
|
||||
.where(eq(qsos.mode, dbQSO.mode))
|
||||
.limit(1);
|
||||
|
||||
console.error('Existing QSOs found:', existing.length);
|
||||
if (existing.length > 0) {
|
||||
console.error('Existing QSO:', JSON.stringify(existing[0]));
|
||||
}
|
||||
|
||||
if (existing.length > 0) {
|
||||
// Update existing QSO
|
||||
console.error('Updating existing QSO');
|
||||
await db
|
||||
.update(qsos)
|
||||
.set({
|
||||
lotwQslRdate: dbQSO.lotwQslRdate,
|
||||
lotwQslRstatus: dbQSO.lotwQslRstatus,
|
||||
lotwSyncedAt: dbQSO.lotwSyncedAt,
|
||||
})
|
||||
.where(eq(qsos.id, existing[0].id));
|
||||
updatedCount++;
|
||||
} else {
|
||||
// Insert new QSO
|
||||
console.error('Inserting new QSO with data:', JSON.stringify(dbQSO));
|
||||
try {
|
||||
const result = await db.insert(qsos).values(dbQSO);
|
||||
console.error('Insert result:', result);
|
||||
addedCount++;
|
||||
} catch (insertError) {
|
||||
console.error('Insert failed:', insertError.message);
|
||||
console.error('Insert error details:', insertError);
|
||||
throw insertError;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('ERROR processing QSO:', error);
|
||||
errors.push({
|
||||
qso: qsoData,
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
total: adifQSOs.length,
|
||||
added: addedCount,
|
||||
updated: updatedCount,
|
||||
errors: errors.length > 0 ? errors : undefined,
|
||||
};
|
||||
} catch (error) {
|
||||
throw new Error(`LoTW sync failed: ${error.message}`);
|
||||
if (!adifQSOs || adifQSOs.length === 0) {
|
||||
return { success: true, total: 0, added: 0, updated: 0, message: 'No QSOs found in LoTW' };
|
||||
}
|
||||
|
||||
let addedCount = 0;
|
||||
let updatedCount = 0;
|
||||
const errors = [];
|
||||
|
||||
for (const qsoData of adifQSOs) {
|
||||
try {
|
||||
const dbQSO = convertQSODatabaseFormat(qsoData, userId);
|
||||
|
||||
const existing = await db
|
||||
.select()
|
||||
.from(qsos)
|
||||
.where(eq(qsos.userId, userId))
|
||||
.where(eq(qsos.callsign, dbQSO.callsign))
|
||||
.where(eq(qsos.qsoDate, dbQSO.qsoDate))
|
||||
.where(eq(qsos.band, dbQSO.band))
|
||||
.where(eq(qsos.mode, dbQSO.mode))
|
||||
.limit(1);
|
||||
|
||||
if (existing.length > 0) {
|
||||
await db
|
||||
.update(qsos)
|
||||
.set({
|
||||
lotwQslRdate: dbQSO.lotwQslRdate,
|
||||
lotwQslRstatus: dbQSO.lotwQslRstatus,
|
||||
lotwSyncedAt: dbQSO.lotwSyncedAt,
|
||||
})
|
||||
.where(eq(qsos.id, existing[0].id));
|
||||
updatedCount++;
|
||||
} else {
|
||||
await db.insert(qsos).values(dbQSO);
|
||||
addedCount++;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error processing QSO', { error: error.message, qso: qsoData });
|
||||
errors.push({ qso: qsoData, error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('LoTW sync completed', { total: adifQSOs.length, added: addedCount, updated: updatedCount });
|
||||
|
||||
return {
|
||||
success: true,
|
||||
total: adifQSOs.length,
|
||||
added: addedCount,
|
||||
updated: updatedCount,
|
||||
errors: errors.length > 0 ? errors : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get QSOs for a user with pagination
|
||||
* @param {number} userId - User ID
|
||||
* @param {Object} filters - Query filters
|
||||
* @param {Object} options - Pagination options { page, limit }
|
||||
* @returns {Promise<Object>} Paginated QSOs
|
||||
*/
|
||||
export async function getUserQSOs(userId, filters = {}, options = {}) {
|
||||
const { eq, and, desc, sql } = await import('drizzle-orm');
|
||||
|
||||
const { page = 1, limit = 100 } = options;
|
||||
|
||||
console.error('getUserQSOs called with userId:', userId, 'filters:', filters, 'page:', page, 'limit:', limit);
|
||||
|
||||
// Build where conditions
|
||||
const conditions = [eq(qsos.userId, userId)];
|
||||
|
||||
if (filters.band) {
|
||||
conditions.push(eq(qsos.band, filters.band));
|
||||
}
|
||||
if (filters.band) conditions.push(eq(qsos.band, filters.band));
|
||||
if (filters.mode) conditions.push(eq(qsos.mode, filters.mode));
|
||||
if (filters.confirmed) conditions.push(eq(qsos.lotwQslRstatus, 'Y'));
|
||||
|
||||
if (filters.mode) {
|
||||
conditions.push(eq(qsos.mode, filters.mode));
|
||||
}
|
||||
|
||||
if (filters.confirmed) {
|
||||
conditions.push(eq(qsos.lotwQslRstatus, 'Y'));
|
||||
}
|
||||
|
||||
// Get total count for pagination
|
||||
const allResults = await db.select().from(qsos).where(and(...conditions));
|
||||
const totalCount = allResults.length;
|
||||
|
||||
// Calculate offset
|
||||
const offset = (page - 1) * limit;
|
||||
|
||||
// Get paginated results
|
||||
const results = await db
|
||||
.select()
|
||||
.from(qsos)
|
||||
@@ -512,8 +336,6 @@ export async function getUserQSOs(userId, filters = {}, options = {}) {
|
||||
.limit(limit)
|
||||
.offset(offset);
|
||||
|
||||
console.error('getUserQSOs returning', results.length, 'QSOs (page', page, 'of', Math.ceil(totalCount / limit), ')');
|
||||
|
||||
return {
|
||||
qsos: results,
|
||||
pagination: {
|
||||
@@ -529,18 +351,11 @@ export async function getUserQSOs(userId, filters = {}, options = {}) {
|
||||
|
||||
/**
|
||||
* Get QSO statistics for a user
|
||||
* @param {number} userId - User ID
|
||||
* @returns {Promise<Object>} Statistics object
|
||||
*/
|
||||
export async function getQSOStats(userId) {
|
||||
const { eq } = await import('drizzle-orm');
|
||||
const allQSOs = await db.select().from(qsos).where(eq(qsos.userId, userId));
|
||||
|
||||
console.error('getQSOStats called with userId:', userId, 'found', allQSOs.length, 'QSOs in database');
|
||||
|
||||
const confirmed = allQSOs.filter((q) => q.lotwQslRstatus === 'Y');
|
||||
|
||||
// Count unique entities
|
||||
const uniqueEntities = new Set();
|
||||
const uniqueBands = new Set();
|
||||
const uniqueModes = new Set();
|
||||
@@ -551,43 +366,28 @@ export async function getQSOStats(userId) {
|
||||
if (q.mode) uniqueModes.add(q.mode);
|
||||
});
|
||||
|
||||
const stats = {
|
||||
return {
|
||||
total: allQSOs.length,
|
||||
confirmed: confirmed.length,
|
||||
uniqueEntities: uniqueEntities.size,
|
||||
uniqueBands: uniqueBands.size,
|
||||
uniqueModes: uniqueModes.size,
|
||||
};
|
||||
|
||||
console.error('getQSOStats returning:', stats);
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the date of the last LoTW QSL for a user
|
||||
* Used for qso_qslsince parameter to minimize downloads
|
||||
* @param {number} userId - User ID
|
||||
* @returns {Promise<Date|null>} Last QSL date or null
|
||||
*/
|
||||
export async function getLastLoTWQSLDate(userId) {
|
||||
const { eq } = await import('drizzle-orm');
|
||||
|
||||
// Get the most recent lotwQslRdate for this user
|
||||
async function getLastLoTWQSLDate(userId) {
|
||||
const [result] = await db
|
||||
.select({ maxDate: max(qsos.lotwQslRdate) })
|
||||
.from(qsos)
|
||||
.where(eq(qsos.userId, userId));
|
||||
|
||||
if (!result || !result.maxDate) {
|
||||
return null;
|
||||
}
|
||||
if (!result || !result.maxDate) return null;
|
||||
|
||||
// Parse ADIF date format (YYYYMMDD) to Date
|
||||
const dateStr = result.maxDate;
|
||||
if (!dateStr || dateStr === '') {
|
||||
return null;
|
||||
}
|
||||
if (!dateStr || dateStr === '') return null;
|
||||
|
||||
const year = dateStr.substring(0, 4);
|
||||
const month = dateStr.substring(4, 6);
|
||||
@@ -596,172 +396,106 @@ export async function getLastLoTWQSLDate(userId) {
|
||||
return new Date(`${year}-${month}-${day}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate LoTW response following Wavelog logic
|
||||
* @param {string} responseData - Response from LoTW
|
||||
* @returns {Object} { valid: boolean, error?: string }
|
||||
*/
|
||||
function validateLoTWResponse(responseData) {
|
||||
const trimmed = responseData.trim();
|
||||
|
||||
// Wavelog: Check for username/password incorrect
|
||||
if (trimmed.toLowerCase().includes('username/password incorrect')) {
|
||||
return {
|
||||
valid: false,
|
||||
error: 'Username/password incorrect',
|
||||
shouldClearCredentials: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Wavelog: Check if file starts with "ARRL Logbook of the World Status Report"
|
||||
const header = trimmed.substring(0, 39).toLowerCase();
|
||||
if (!header.includes('arrl logbook of the world')) {
|
||||
return {
|
||||
valid: false,
|
||||
error: 'Downloaded LoTW report is invalid. File does not start with expected header.',
|
||||
};
|
||||
}
|
||||
|
||||
return { valid: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* LoTW sync job processor for the job queue
|
||||
* @param {number} jobId - Job ID
|
||||
* @param {number} userId - User ID
|
||||
* @param {Object} data - Job data { lotwUsername, lotwPassword }
|
||||
* @returns {Promise<Object>} Sync result
|
||||
*/
|
||||
export async function syncQSOsForJob(jobId, userId, data) {
|
||||
const { lotwUsername, lotwPassword } = data;
|
||||
|
||||
try {
|
||||
// Update job progress: starting
|
||||
await updateJobProgress(jobId, {
|
||||
message: 'Fetching QSOs from LoTW...',
|
||||
step: 'fetch',
|
||||
});
|
||||
await updateJobProgress(jobId, {
|
||||
message: 'Fetching QSOs from LoTW...',
|
||||
step: 'fetch',
|
||||
});
|
||||
|
||||
// Get last LoTW QSL date for incremental sync
|
||||
const lastQSLDate = await getLastLoTWQSLDate(userId);
|
||||
const lastQSLDate = await getLastLoTWQSLDate(userId);
|
||||
const sinceDate = lastQSLDate || new Date('2000-01-01');
|
||||
|
||||
// If no QSOs exist, use a far past date to get ALL QSOs (first sync)
|
||||
// Otherwise, use the last QSL date for incremental sync
|
||||
const sinceDate = lastQSLDate || new Date('2000-01-01');
|
||||
if (lastQSLDate) {
|
||||
logger.info(`Job ${jobId}: Incremental sync`, { since: sinceDate.toISOString().split('T')[0] });
|
||||
} else {
|
||||
logger.info(`Job ${jobId}: Full sync`);
|
||||
}
|
||||
|
||||
if (lastQSLDate) {
|
||||
console.error(`[Job ${jobId}] Incremental sync since ${sinceDate.toISOString().split('T')[0]}`);
|
||||
} else {
|
||||
console.error(`[Job ${jobId}] Full sync - fetching ALL QSOs since 2000-01-01`);
|
||||
}
|
||||
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
|
||||
|
||||
// Fetch from LoTW
|
||||
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
|
||||
if (!adifQSOs || adifQSOs.length === 0) {
|
||||
return { success: true, total: 0, added: 0, updated: 0, message: 'No QSOs found in LoTW' };
|
||||
}
|
||||
|
||||
if (!adifQSOs || adifQSOs.length === 0) {
|
||||
return {
|
||||
success: true,
|
||||
total: 0,
|
||||
added: 0,
|
||||
updated: 0,
|
||||
message: 'No QSOs found in LoTW',
|
||||
};
|
||||
}
|
||||
await updateJobProgress(jobId, {
|
||||
message: `Processing ${adifQSOs.length} QSOs...`,
|
||||
step: 'process',
|
||||
total: adifQSOs.length,
|
||||
processed: 0,
|
||||
});
|
||||
|
||||
// Update job progress: processing
|
||||
await updateJobProgress(jobId, {
|
||||
message: `Processing ${adifQSOs.length} QSOs...`,
|
||||
step: 'process',
|
||||
total: adifQSOs.length,
|
||||
processed: 0,
|
||||
});
|
||||
let addedCount = 0;
|
||||
let updatedCount = 0;
|
||||
const errors = [];
|
||||
|
||||
let addedCount = 0;
|
||||
let updatedCount = 0;
|
||||
const errors = [];
|
||||
for (let i = 0; i < adifQSOs.length; i++) {
|
||||
const qsoData = adifQSOs[i];
|
||||
|
||||
// Process each QSO
|
||||
for (let i = 0; i < adifQSOs.length; i++) {
|
||||
const qsoData = adifQSOs[i];
|
||||
try {
|
||||
const dbQSO = convertQSODatabaseFormat(qsoData, userId);
|
||||
|
||||
try {
|
||||
const dbQSO = convertQSODatabaseFormat(qsoData, userId);
|
||||
|
||||
// Check if QSO already exists
|
||||
const { eq, and } = await import('drizzle-orm');
|
||||
const existing = await db
|
||||
.select()
|
||||
.from(qsos)
|
||||
.where(
|
||||
and(
|
||||
eq(qsos.userId, userId),
|
||||
eq(qsos.callsign, dbQSO.callsign),
|
||||
eq(qsos.qsoDate, dbQSO.qsoDate),
|
||||
eq(qsos.band, dbQSO.band),
|
||||
eq(qsos.mode, dbQSO.mode)
|
||||
)
|
||||
const existing = await db
|
||||
.select()
|
||||
.from(qsos)
|
||||
.where(
|
||||
and(
|
||||
eq(qsos.userId, userId),
|
||||
eq(qsos.callsign, dbQSO.callsign),
|
||||
eq(qsos.qsoDate, dbQSO.qsoDate),
|
||||
eq(qsos.band, dbQSO.band),
|
||||
eq(qsos.mode, dbQSO.mode)
|
||||
)
|
||||
.limit(1);
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
if (existing.length > 0) {
|
||||
// Update existing QSO
|
||||
await db
|
||||
.update(qsos)
|
||||
.set({
|
||||
lotwQslRdate: dbQSO.lotwQslRdate,
|
||||
lotwQslRstatus: dbQSO.lotwQslRstatus,
|
||||
lotwSyncedAt: dbQSO.lotwSyncedAt,
|
||||
})
|
||||
.where(eq(qsos.id, existing[0].id));
|
||||
updatedCount++;
|
||||
} else {
|
||||
// Insert new QSO
|
||||
await db.insert(qsos).values(dbQSO);
|
||||
addedCount++;
|
||||
}
|
||||
if (existing.length > 0) {
|
||||
await db
|
||||
.update(qsos)
|
||||
.set({
|
||||
lotwQslRdate: dbQSO.lotwQslRdate,
|
||||
lotwQslRstatus: dbQSO.lotwQslRstatus,
|
||||
lotwSyncedAt: dbQSO.lotwSyncedAt,
|
||||
})
|
||||
.where(eq(qsos.id, existing[0].id));
|
||||
updatedCount++;
|
||||
} else {
|
||||
await db.insert(qsos).values(dbQSO);
|
||||
addedCount++;
|
||||
}
|
||||
|
||||
// Update progress every 10 QSOs
|
||||
if ((i + 1) % 10 === 0) {
|
||||
await updateJobProgress(jobId, {
|
||||
processed: i + 1,
|
||||
message: `Processed ${i + 1}/${adifQSOs.length} QSOs...`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[Job ${jobId}] ERROR processing QSO:`, error);
|
||||
errors.push({
|
||||
qso: qsoData,
|
||||
error: error.message,
|
||||
if ((i + 1) % 10 === 0) {
|
||||
await updateJobProgress(jobId, {
|
||||
processed: i + 1,
|
||||
message: `Processed ${i + 1}/${adifQSOs.length} QSOs...`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Job ${jobId}: Error processing QSO`, { error: error.message });
|
||||
errors.push({ qso: qsoData, error: error.message });
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
total: adifQSOs.length,
|
||||
added: addedCount,
|
||||
updated: updatedCount,
|
||||
errors: errors.length > 0 ? errors : undefined,
|
||||
};
|
||||
} catch (error) {
|
||||
// Check if it's a credential error
|
||||
if (error.message.includes('Username/password incorrect')) {
|
||||
throw new Error('Invalid LoTW credentials. Please check your username and password.');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
logger.info(`Job ${jobId} completed`, { total: adifQSOs.length, added: addedCount, updated: updatedCount });
|
||||
|
||||
return {
|
||||
success: true,
|
||||
total: adifQSOs.length,
|
||||
added: addedCount,
|
||||
updated: updatedCount,
|
||||
errors: errors.length > 0 ? errors : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete all QSOs for a user
|
||||
* @param {number} userId - User ID
|
||||
* @returns {Promise<number>} Number of QSOs deleted
|
||||
*/
|
||||
export async function deleteQSOs(userId) {
|
||||
const { eq } = await import('drizzle-orm');
|
||||
|
||||
const result = await db.delete(qsos).where(eq(qsos.userId, userId));
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user