Joerg
44c13e1bdc
- Add detailed documentation covering architecture, components, code structure - Document award system with multiple examples (DXCC, WAS, VUCC, Satellite) - Implement LoTW sync service with ADIF parsing and long-polling - Add QSO logbook page with filtering and statistics - Add settings page for LoTW credentials management - Add API endpoints for LoTW sync, QSO retrieval, and statistics Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
532 lines
16 KiB
JavaScript
532 lines
16 KiB
JavaScript
import { db } from '../config/database.js';
|
|
import { qsos } from '../db/schema/index.js';
|
|
|
|
/**
|
|
* LoTW (Logbook of the World) Service
|
|
* Fetches QSOs from ARRL's LoTW system
|
|
*/
|
|
|
|
// Configuration for long-polling
|
|
const POLLING_CONFIG = {
|
|
maxRetries: 30, // Maximum number of retry attempts
|
|
retryDelay: 10000, // Delay between retries in ms (10 seconds)
|
|
requestTimeout: 60000, // Timeout for individual requests in ms (1 minute)
|
|
maxTotalTime: 600000, // Maximum total time to wait in ms (10 minutes)
|
|
};
|
|
|
|
/**
|
|
* Check if LoTW response indicates the report is still being prepared
|
|
* @param {string} responseData - The response text from LoTW
|
|
* @returns {boolean} True if report is still pending
|
|
*/
|
|
function isReportPending(responseData) {
|
|
const trimmed = responseData.trim().toLowerCase();
|
|
|
|
// LoTW returns various messages when report is not ready:
|
|
// - Empty responses
|
|
// - "Report is being prepared" or similar messages
|
|
// - HTML error pages
|
|
// - Very short responses that aren't valid ADIF
|
|
|
|
// Check for empty or very short responses
|
|
if (trimmed.length < 100) {
|
|
return true;
|
|
}
|
|
|
|
// Check for HTML responses (error pages)
|
|
if (trimmed.includes('<html>') || trimmed.includes('<!doctype html>')) {
|
|
return true;
|
|
}
|
|
|
|
// Check for common "not ready" messages
|
|
const pendingMessages = [
|
|
'report is being prepared',
|
|
'your report is being generated',
|
|
'please try again',
|
|
'report queue',
|
|
'not yet available',
|
|
'temporarily unavailable',
|
|
];
|
|
|
|
for (const msg of pendingMessages) {
|
|
if (trimmed.includes(msg)) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Check if it looks like valid ADIF data (should start with <ADIF_VER: or have <qso_date)
|
|
const hasAdifHeader = trimmed.includes('<adif_ver:') ||
|
|
trimmed.includes('<qso_date') ||
|
|
trimmed.includes('<call:') ||
|
|
trimmed.includes('<band:');
|
|
|
|
return !hasAdifHeader;
|
|
}
|
|
|
|
/**
|
|
* Sleep/delay utility
|
|
* @param {number} ms - Milliseconds to sleep
|
|
* @returns {Promise<void>}
|
|
*/
|
|
function sleep(ms) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
/**
|
|
* Fetch QSOs from LoTW with long-polling support
|
|
* @param {string} lotwUsername - LoTW username
|
|
* @param {string} lotwPassword - LoTW password
|
|
* @param {Date} sinceDate - Only fetch QSOs since this date
|
|
* @returns {Promise<Array>} Array of QSO objects
|
|
*/
|
|
export async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
|
|
// LoTW report URL
|
|
const url = 'https://lotw.arrl.org/lotwuser/lotwreport.adi';
|
|
|
|
// Build query parameters - qso_query=1 is REQUIRED to get QSO records!
|
|
const params = new URLSearchParams({
|
|
login: lotwUsername,
|
|
password: lotwPassword,
|
|
qso_query: '1', // REQUIRED: Without this, no QSO records are returned
|
|
qso_qsl: 'yes', // Only get QSOs with QSLs (confirmed)
|
|
qso_qsldetail: 'yes', // Include QSL details (station location)
|
|
qso_mydetail: 'yes', // Include my station details
|
|
qso_withown: 'yes', // Include own callsign
|
|
});
|
|
|
|
// Add date filter - ALWAYS send qso_qslsince to get all QSOs
|
|
// LoTW default behavior only returns QSOs since last download, so we must
|
|
// explicitly send a date to get QSOs since that date
|
|
const dateStr = sinceDate
|
|
? sinceDate.toISOString().split('T')[0]
|
|
: '2026-01-01'; // Default: Get QSOs since 2026-01-01
|
|
params.append('qso_qslsince', dateStr);
|
|
|
|
console.error('Date filter:', dateStr, sinceDate ? '(User-specified date)' : '(Default: since 2026-01-01)');
|
|
|
|
const fullUrl = `${url}?${params.toString()}`;
|
|
|
|
console.error('Fetching from LoTW:', fullUrl.replace(/password=[^&]+/, 'password=***'));
|
|
|
|
const startTime = Date.now();
|
|
|
|
// Long-polling loop
|
|
for (let attempt = 0; attempt < POLLING_CONFIG.maxRetries; attempt++) {
|
|
try {
|
|
// Check if we've exceeded max total time
|
|
const elapsed = Date.now() - startTime;
|
|
if (elapsed > POLLING_CONFIG.maxTotalTime) {
|
|
throw new Error(`LoTW sync timeout: exceeded maximum wait time of ${POLLING_CONFIG.maxTotalTime / 1000} seconds`);
|
|
}
|
|
|
|
if (attempt > 0) {
|
|
console.error(`Retry attempt ${attempt + 1}/${POLLING_CONFIG.maxRetries} (elapsed: ${Math.round(elapsed / 1000)}s)`);
|
|
}
|
|
|
|
// Make request with timeout
|
|
const controller = new AbortController();
|
|
const timeoutId = setTimeout(() => controller.abort(), POLLING_CONFIG.requestTimeout);
|
|
|
|
const response = await fetch(fullUrl, { signal: controller.signal });
|
|
|
|
clearTimeout(timeoutId);
|
|
|
|
// Handle HTTP errors
|
|
if (!response.ok) {
|
|
if (response.status === 503) {
|
|
// Service unavailable - might be temporary, retry
|
|
console.error('LoTW returned 503 (Service Unavailable), waiting before retry...');
|
|
await sleep(POLLING_CONFIG.retryDelay);
|
|
continue;
|
|
} else if (response.status === 401) {
|
|
throw new Error('Invalid LoTW credentials. Please check your username and password in Settings.');
|
|
} else if (response.status === 404) {
|
|
throw new Error('LoTW service not found (404). The LoTW API URL may have changed.');
|
|
} else {
|
|
// Other errors - log but retry
|
|
console.error(`LoTW returned ${response.status} ${response.statusText}, waiting before retry...`);
|
|
await sleep(POLLING_CONFIG.retryDelay);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Get response text
|
|
const adifData = await response.text();
|
|
console.error(`Response length: ${adifData.length} bytes`);
|
|
|
|
// Check if report is still pending
|
|
if (isReportPending(adifData)) {
|
|
console.error('LoTW report is still being prepared, waiting...', adifData.substring(0, 100));
|
|
|
|
// Wait before retrying
|
|
await sleep(POLLING_CONFIG.retryDelay);
|
|
continue;
|
|
}
|
|
|
|
// We have valid data!
|
|
console.error('LoTW report ready, parsing ADIF data...');
|
|
console.error('ADIF preview:', adifData.substring(0, 200));
|
|
|
|
// Parse ADIF format
|
|
const qsos = parseADIF(adifData);
|
|
console.error(`Successfully parsed ${qsos.length} QSOs from LoTW`);
|
|
|
|
return qsos;
|
|
|
|
} catch (error) {
|
|
const elapsed = Date.now() - startTime;
|
|
|
|
if (error.name === 'AbortError') {
|
|
console.error(`Request timeout on attempt ${attempt + 1}, retrying...`);
|
|
await sleep(POLLING_CONFIG.retryDelay);
|
|
continue;
|
|
}
|
|
|
|
// Re-throw credential/auth errors immediately
|
|
if (error.message.includes('credentials') || error.message.includes('401') || error.message.includes('404')) {
|
|
throw error;
|
|
}
|
|
|
|
// For other errors, log and retry if we haven't exhausted retries
|
|
if (attempt < POLLING_CONFIG.maxRetries - 1) {
|
|
console.error(`Error on attempt ${attempt + 1}: ${error.message}`);
|
|
console.error('Retrying...');
|
|
await sleep(POLLING_CONFIG.retryDelay);
|
|
continue;
|
|
} else {
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
// If we get here, we exhausted all retries
|
|
const totalTime = Math.round((Date.now() - startTime) / 1000);
|
|
throw new Error(`LoTW sync failed: Report not ready after ${POLLING_CONFIG.maxRetries} attempts (${totalTime}s). LoTW may be experiencing high load. Please try again later.`);
|
|
}
|
|
|
|
/**
|
|
* Parse ADIF (Amateur Data Interchange Format) data
|
|
* @param {string} adifData - Raw ADIF text data
|
|
* @returns {Array<Array>} Array of QSOs (each QSO is an array of field objects)
|
|
*/
|
|
function parseADIF(adifData) {
|
|
const qsos = [];
|
|
const records = adifData.split('<eor>');
|
|
|
|
console.error(`Total records after splitting by <eor>: ${records.length}`);
|
|
|
|
for (let i = 0; i < records.length; i++) {
|
|
const record = records[i];
|
|
|
|
// Skip empty records or records that are just header info
|
|
if (!record.trim()) continue;
|
|
if (record.trim().startsWith('<') && !record.includes('<CALL:') && !record.includes('<call:')) continue;
|
|
|
|
const qso = {};
|
|
|
|
// Match ADIF fields: <FIELDNAME:length[:type]>value
|
|
// Important: The 'type' part is optional, and field names can contain underscores
|
|
// We use the length parameter to extract exactly that many characters
|
|
const regex = /<([A-Z_]+):(\d+)(?::[A-Z]+)?>([\s\S])/gi;
|
|
let match;
|
|
|
|
while ((match = regex.exec(record)) !== null) {
|
|
const [fullMatch, fieldName, lengthStr, firstChar] = match;
|
|
const length = parseInt(lengthStr, 10);
|
|
|
|
// Extract exactly 'length' characters starting from the position after the '>'
|
|
const valueStart = match.index + fullMatch.length - 1; // -1 because firstChar is already captured
|
|
const value = record.substring(valueStart, valueStart + length);
|
|
|
|
qso[fieldName.toLowerCase()] = value.trim();
|
|
|
|
// Move regex lastIndex to the end of the value so we can find the next tag
|
|
regex.lastIndex = valueStart + length;
|
|
}
|
|
|
|
// Only add if we have actual QSO data (has CALL or call field)
|
|
if (Object.keys(qso).length > 0 && (qso.call || qso.call)) {
|
|
qsos.push(qso);
|
|
|
|
// Log first few QSOs for debugging
|
|
if (qsos.length <= 3) {
|
|
console.error(`Parsed QSO #${qsos.length}: ${qso.call} on ${qso.qso_date} ${qso.band} ${qso.mode}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
console.error(`Total QSOs parsed: ${qsos.length}`);
|
|
return qsos;
|
|
}
|
|
|
|
/**
|
|
* Convert ADIF QSO to database format
|
|
* @param {Object} adifQSO - QSO object from ADIF parser
|
|
* @param {number} userId - User ID
|
|
* @returns {Object} Database-ready QSO object
|
|
*/
|
|
export function convertQSODatabaseFormat(adifQSO, userId) {
|
|
return {
|
|
userId,
|
|
callsign: adifQSO.call || '',
|
|
qsoDate: adifQSO.qso_date || '',
|
|
timeOn: adifQSO.time_on || adifQSO.time_off || '000000',
|
|
band: normalizeBand(adifQSO.band),
|
|
mode: normalizeMode(adifQSO.mode),
|
|
freq: adifQSO.freq ? parseInt(adifQSO.freq) : null,
|
|
freqRx: adifQSO.freq_rx ? parseInt(adifQSO.freq_rx) : null,
|
|
entity: adifQSO.country || adifQSO.dxcc_country || '',
|
|
entityId: adifQSO.dxcc || null,
|
|
grid: adifQSO.gridsquare || '',
|
|
continent: adifQSO.continent || '',
|
|
cqZone: adifQSO.cq_zone ? parseInt(adifQSO.cq_zone) : null,
|
|
ituZone: adifQSO.itu_zone ? parseInt(adifQSO.itu_zone) : null,
|
|
state: adifQSO.state || adifQSO.us_state || '',
|
|
satName: adifQSO.sat_name || '',
|
|
satMode: adifQSO.sat_mode || '',
|
|
lotwQslRdate: adifQSO.qslrdate || '',
|
|
lotwQslRstatus: adifQSO.qsl_rcvd || 'N',
|
|
lotwSyncedAt: new Date(),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Normalize band name
|
|
* @param {string} band - Band from ADIF
|
|
* @returns {string|null} Normalized band
|
|
*/
|
|
function normalizeBand(band) {
|
|
if (!band) return null;
|
|
|
|
const bandMap = {
|
|
'160m': '160m',
|
|
'80m': '80m',
|
|
'60m': '60m',
|
|
'40m': '40m',
|
|
'30m': '30m',
|
|
'20m': '20m',
|
|
'17m': '17m',
|
|
'15m': '15m',
|
|
'12m': '12m',
|
|
'10m': '10m',
|
|
'6m': '6m',
|
|
'4m': '4m',
|
|
'2m': '2m',
|
|
'1.25m': '1.25m',
|
|
'70cm': '70cm',
|
|
'33cm': '33cm',
|
|
'23cm': '23cm',
|
|
'13cm': '13cm',
|
|
'9cm': '9cm',
|
|
'6cm': '6cm',
|
|
'3cm': '3cm',
|
|
'1.2cm': '1.2cm',
|
|
'mm': 'mm',
|
|
};
|
|
|
|
return bandMap[band.toLowerCase()] || band;
|
|
}
|
|
|
|
/**
|
|
* Normalize mode name
|
|
* @param {string} mode - Mode from ADIF
|
|
* @returns {string} Normalized mode
|
|
*/
|
|
function normalizeMode(mode) {
|
|
if (!mode) return '';
|
|
|
|
const modeMap = {
|
|
'cw': 'CW',
|
|
'ssb': 'SSB',
|
|
'am': 'AM',
|
|
'fm': 'FM',
|
|
'rtty': 'RTTY',
|
|
'psk31': 'PSK31',
|
|
'psk63': 'PSK63',
|
|
'ft8': 'FT8',
|
|
'ft4': 'FT4',
|
|
'jt65': 'JT65',
|
|
'jt9': 'JT9',
|
|
'js8': 'JS8',
|
|
'mfsk': 'MFSK',
|
|
' Olivia': 'OLIVIA',
|
|
};
|
|
|
|
const normalized = modeMap[mode.toLowerCase()];
|
|
return normalized || mode.toUpperCase();
|
|
}
|
|
|
|
/**
|
|
* Sync QSOs from LoTW to database
|
|
* @param {number} userId - User ID
|
|
* @param {string} lotwUsername - LoTW username
|
|
* @param {string} lotwPassword - LoTW password
|
|
* @param {Date} sinceDate - Only sync QSOs since this date
|
|
* @returns {Promise<Object>} Sync result with counts
|
|
*/
|
|
export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null) {
|
|
try {
|
|
// Fetch QSOs from LoTW
|
|
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
|
|
|
|
if (!adifQSOs || adifQSOs.length === 0) {
|
|
return {
|
|
success: true,
|
|
total: 0,
|
|
added: 0,
|
|
updated: 0,
|
|
message: 'No QSOs found in LoTW',
|
|
};
|
|
}
|
|
|
|
let addedCount = 0;
|
|
let updatedCount = 0;
|
|
const errors = [];
|
|
|
|
// Process each QSO
|
|
for (const qsoData of adifQSOs) {
|
|
try {
|
|
console.error('Raw ADIF QSO data:', JSON.stringify(qsoData));
|
|
const dbQSO = convertQSODatabaseFormat(qsoData, userId);
|
|
console.error('Converted QSO:', JSON.stringify(dbQSO));
|
|
console.error('Processing QSO:', dbQSO.callsign, dbQSO.qsoDate, dbQSO.band, dbQSO.mode);
|
|
|
|
// Check if QSO already exists (by callsign, date, time, band, mode)
|
|
const { eq } = await import('drizzle-orm');
|
|
const existing = await db
|
|
.select()
|
|
.from(qsos)
|
|
.where(eq(qsos.userId, userId))
|
|
.where(eq(qsos.callsign, dbQSO.callsign))
|
|
.where(eq(qsos.qsoDate, dbQSO.qsoDate))
|
|
.where(eq(qsos.band, dbQSO.band))
|
|
.where(eq(qsos.mode, dbQSO.mode))
|
|
.limit(1);
|
|
|
|
console.error('Existing QSOs found:', existing.length);
|
|
if (existing.length > 0) {
|
|
console.error('Existing QSO:', JSON.stringify(existing[0]));
|
|
}
|
|
|
|
if (existing.length > 0) {
|
|
// Update existing QSO
|
|
console.error('Updating existing QSO');
|
|
await db
|
|
.update(qsos)
|
|
.set({
|
|
lotwQslRdate: dbQSO.lotwQslRdate,
|
|
lotwQslRstatus: dbQSO.lotwQslRstatus,
|
|
lotwSyncedAt: dbQSO.lotwSyncedAt,
|
|
})
|
|
.where(eq(qsos.id, existing[0].id));
|
|
updatedCount++;
|
|
} else {
|
|
// Insert new QSO
|
|
console.error('Inserting new QSO with data:', JSON.stringify(dbQSO));
|
|
try {
|
|
const result = await db.insert(qsos).values(dbQSO);
|
|
console.error('Insert result:', result);
|
|
addedCount++;
|
|
} catch (insertError) {
|
|
console.error('Insert failed:', insertError.message);
|
|
console.error('Insert error details:', insertError);
|
|
throw insertError;
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error('ERROR processing QSO:', error);
|
|
errors.push({
|
|
qso: qsoData,
|
|
error: error.message,
|
|
});
|
|
}
|
|
}
|
|
|
|
return {
|
|
success: true,
|
|
total: adifQSOs.length,
|
|
added: addedCount,
|
|
updated: updatedCount,
|
|
errors: errors.length > 0 ? errors : undefined,
|
|
};
|
|
} catch (error) {
|
|
throw new Error(`LoTW sync failed: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get QSOs for a user
|
|
* @param {number} userId - User ID
|
|
* @param {Object} filters - Query filters
|
|
* @returns {Promise<Array>} Array of QSOs
|
|
*/
|
|
export async function getUserQSOs(userId, filters = {}) {
|
|
const { eq, and } = await import('drizzle-orm');
|
|
|
|
console.error('getUserQSOs called with userId:', userId, 'filters:', filters);
|
|
|
|
// Build where conditions
|
|
const conditions = [eq(qsos.userId, userId)];
|
|
|
|
if (filters.band) {
|
|
conditions.push(eq(qsos.band, filters.band));
|
|
}
|
|
|
|
if (filters.mode) {
|
|
conditions.push(eq(qsos.mode, filters.mode));
|
|
}
|
|
|
|
if (filters.confirmed) {
|
|
conditions.push(eq(qsos.lotwQslRstatus, 'Y'));
|
|
}
|
|
|
|
// Use and() to combine all conditions
|
|
const results = await db.select().from(qsos).where(and(...conditions));
|
|
|
|
console.error('getUserQSOs returning', results.length, 'QSOs');
|
|
|
|
// Order by date descending, then time
|
|
return results.sort((a, b) => {
|
|
const dateCompare = b.qsoDate.localeCompare(a.qsoDate);
|
|
if (dateCompare !== 0) return dateCompare;
|
|
return b.timeOn.localeCompare(a.timeOn);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Get QSO statistics for a user
|
|
* @param {number} userId - User ID
|
|
* @returns {Promise<Object>} Statistics object
|
|
*/
|
|
export async function getQSOStats(userId) {
|
|
const { eq } = await import('drizzle-orm');
|
|
const allQSOs = await db.select().from(qsos).where(eq(qsos.userId, userId));
|
|
|
|
console.error('getQSOStats called with userId:', userId, 'found', allQSOs.length, 'QSOs in database');
|
|
|
|
const confirmed = allQSOs.filter((q) => q.lotwQslRstatus === 'Y');
|
|
|
|
// Count unique entities
|
|
const uniqueEntities = new Set();
|
|
const uniqueBands = new Set();
|
|
const uniqueModes = new Set();
|
|
|
|
allQSOs.forEach((q) => {
|
|
if (q.entity) uniqueEntities.add(q.entity);
|
|
if (q.band) uniqueBands.add(q.band);
|
|
if (q.mode) uniqueModes.add(q.mode);
|
|
});
|
|
|
|
const stats = {
|
|
total: allQSOs.length,
|
|
confirmed: confirmed.length,
|
|
uniqueEntities: uniqueEntities.size,
|
|
uniqueBands: uniqueBands.size,
|
|
uniqueModes: uniqueModes.size,
|
|
};
|
|
|
|
console.error('getQSOStats returning:', stats);
|
|
|
|
return stats;
|
|
}
|