dj7nt/award
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Add structured logging, navigation bar, and code cleanup

Browse Source 
## Backend
- Add Pino logging framework with timestamps and structured output
- Replace all console.error statements (49+) with proper logging levels
- Fix Drizzle ORM bug: replace invalid .get() calls with .limit(1)
- Remove unused auth routes file (already in index.js)
- Make internal functions private (remove unnecessary exports)
- Simplify code by removing excessive debug logging

## Frontend
- Add navigation bar to layout with:
  - User's callsign display
  - Navigation links (Dashboard, QSOs, Settings)
  - Logout button with red color distinction
- Navigation only shows when user is logged in
- Dark themed design matching footer

## Documentation
- Update README.md with new project structure
- Update docs/DOCUMENTATION.md with logging and nav bar info
- Add logger.js to configuration section

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Joerg
2026-01-16 08:11:57 +01:00
parent 512f4f682e
commit d959235cdd
13 changed files with 388 additions and 683 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/backend/config/database.js
View File

@@ -2,6 +2,7 @@ import Database from 'bun:sqlite';
import { drizzle } from 'drizzle-orm/bun-sqlite';
import * as schema from '../db/schema/index.js';
import { join } from 'path';
import logger from './logger.js';
// Get the directory of this file (src/backend/config/)
const configDir = import.meta.dir || new URL('.', import.meta.url).pathname;
@@ -9,7 +10,7 @@ const configDir = import.meta.dir || new URL('.', import.meta.url).pathname;
// Go up one level to get src/backend/, then to award.db
const dbPath = join(configDir, '..', 'award.db');
console.error('[Database] Using database at:', dbPath);
logger.debug('Database path', { dbPath });
// Create SQLite database connection
const sqlite = new Database(dbPath);

20
src/backend/config/logger.js Normal file
View File

@@ -0,0 +1,20 @@
import pino from 'pino';
const isDevelopment = process.env.NODE_ENV !== 'production';
export const logger = pino({
level: process.env.LOG_LEVEL || (isDevelopment ? 'debug' : 'info'),
transport: isDevelopment
? {
target: 'pino-pretty',
options: {
colorize: true,
translateTime: 'SYS:standard',
ignore: 'pid,hostname',
},
}
: undefined,
timestamp: pino.stdTimeFunctions.isoTime,
});
export default logger;

21
src/backend/index.js
View File

@@ -2,6 +2,7 @@ import { Elysia, t } from 'elysia';
import { cors } from '@elysiajs/cors';
import { jwt } from '@elysiajs/jwt';
import { JWT_SECRET } from './config/jwt.js';
import logger from './config/logger.js';
import {
registerUser,
authenticateUser,
@@ -229,24 +230,16 @@ const app = new Elysia()
*/
.post('/api/lotw/sync', async ({ user, set }) => {
if (!user) {
console.error('[/api/lotw/sync] No user found in request');
logger.warn('/api/lotw/sync: Unauthorized access attempt');
set.status = 401;
return { success: false, error: 'Unauthorized' };
}
console.error('[/api/lotw/sync] User authenticated:', user.id);
try {
// Get user's LoTW credentials from database
const userData = await getUserById(user.id);
console.error('[/api/lotw/sync] User data from DB:', {
id: userData?.id,
lotwUsername: userData?.lotwUsername ? '***' : null,
hasPassword: !!userData?.lotwPassword
});
if (!userData || !userData.lotwUsername || !userData.lotwPassword) {
console.error('[/api/lotw/sync] Missing LoTW credentials');
logger.debug('/api/lotw/sync: Missing LoTW credentials', { userId: user.id });
set.status = 400;
return {
success: false,
@@ -254,13 +247,11 @@ const app = new Elysia()
};
}
// Enqueue the sync job (enqueueJob will check for existing active jobs)
const result = await enqueueJob(user.id, 'lotw_sync', {
lotwUsername: userData.lotwUsername,
lotwPassword: userData.lotwPassword,
});
// If enqueueJob returned existingJob, format the response
if (!result.success && result.existingJob) {
return {
success: true,
@@ -271,7 +262,7 @@ const app = new Elysia()
return result;
} catch (error) {
console.error('Error in /api/lotw/sync:', error);
logger.error('Error in /api/lotw/sync', { error: error.message });
set.status = 500;
return {
success: false,
@@ -488,7 +479,7 @@ const app = new Elysia()
// Start server
.listen(3001);
console.log(`🦊 Backend server running at http://localhost:${app.server?.port}`);
console.log(`📡 API endpoints available at http://localhost:${app.server?.port}/api`);
logger.info(`Backend server running`, { port: app.server?.port, url: `http://localhost:${app.server?.port}` });
logger.info(`API endpoints available`, { url: `http://localhost:${app.server?.port}/api` });
export default app;

21
src/backend/init-db.js
View File

@@ -1,26 +1,15 @@
import Database from 'bun:sqlite';
import { drizzle } from 'drizzle-orm/bun-sqlite';
import * as schema from './db/schema/index.js';
import logger from './config/logger.js';
import { execSync } from 'child_process';
const sqlite = new Database('./award.db');
const db = drizzle({
client: sqlite,
schema,
});
console.log('Creating database tables...');
// Use drizzle-kit to push the schema
// Since we don't have migrations, let's use the push command
const { execSync } = await import('child_process');
logger.info('Initializing database...');
try {
execSync('bun drizzle-kit push', {
cwd: '/Users/joergdorgeist/Dev/award',
stdio: 'inherit'
});
console.log('✓ Database initialized successfully!');
logger.info('Database initialized successfully');
} catch (error) {
console.error('Failed to initialize database:', error);
logger.error('Failed to initialize database', { error: error.message });
process.exit(1);
}

166
src/backend/routes/auth.js
View File

@@ -1,166 +0,0 @@
import { t } from 'elysia';
import {
registerUser,
authenticateUser,
getUserById,
updateLoTWCredentials,
} from '../services/auth.service.js';
/**
* Authentication routes
* Provides endpoints for user registration, login, and profile management
* These routes will be added to the main app which already has authMiddleware
*/
export const authRoutes = (app) => {
console.error('authRoutes function called with app');
return app
/**
* POST /api/auth/register
* Register a new user
*/
.post(
'/api/auth/register',
async ({ body, jwt, set }) => {
try {
// Create user
const user = await registerUser(body);
// Generate JWT token
const token = await jwt.sign({
userId: user.id,
email: user.email,
callsign: user.callsign,
});
set.status = 201;
return {
success: true,
token,
user,
};
} catch (error) {
set.status = 400;
return {
success: false,
error: error.message,
};
}
},
{
body: t.Object({
email: t.String({
format: 'email',
error: 'Invalid email address',
}),
password: t.String({
minLength: 8,
error: 'Password must be at least 8 characters',
}),
callsign: t.String({
minLength: 3,
maxLength: 10,
error: 'Callsign must be 3-10 characters',
}),
}),
}
)
/**
* POST /api/auth/login
* Authenticate user and return JWT token
*/
.post(
'/api/auth/login',
async ({ body, jwt, set }) => {
try {
// Authenticate user
const user = await authenticateUser(body.email, body.password);
// Generate JWT token
const token = await jwt.sign({
userId: user.id,
email: user.email,
callsign: user.callsign,
});
return {
success: true,
token,
user,
};
} catch (error) {
set.status = 401;
return {
success: false,
error: 'Invalid email or password',
};
}
},
{
body: t.Object({
email: t.String({ format: 'email' }),
password: t.String(),
}),
}
)
/**
* GET /api/auth/me
* Get current user profile (requires authentication)
*/
.get('/api/auth/me', async ({ user, set }) => {
console.error('/me endpoint called, user:', user);
if (!user) {
console.error('No user in context - returning 401');
set.status = 401;
return { success: false, error: 'Unauthorized' };
}
// Get full user data from database
const userData = await getUserById(user.id);
if (!userData) {
set.status = 404;
return { success: false, error: 'User not found' };
}
return {
success: true,
user: userData,
};
})
/**
* PUT /api/auth/lotw-credentials
* Update LoTW credentials (requires authentication)
*/
.put(
'/api/auth/lotw-credentials',
async ({ user, body, set }) => {
if (!user) {
set.status = 401;
return { success: false, error: 'Unauthorized' };
}
try {
await updateLoTWCredentials(user.id, body.lotwUsername, body.lotwPassword);
return {
success: true,
message: 'LoTW credentials updated successfully',
};
} catch (error) {
set.status = 500;
return {
success: false,
error: 'Failed to update LoTW credentials',
};
}
},
{
body: t.Object({
lotwUsername: t.String(),
lotwPassword: t.String(),
}),
}
);
};

16
src/backend/services/auth.service.js
View File

@@ -10,7 +10,7 @@ const SALT_ROUNDS = 10;
* @param {string} password - Plain text password
* @returns {Promise<string>} Hashed password
*/
export async function hashPassword(password) {
async function hashPassword(password) {
return bcrypt.hash(password, SALT_ROUNDS);
}
@@ -20,7 +20,7 @@ export async function hashPassword(password) {
* @param {string} hash - Hashed password
* @returns {Promise<boolean>} True if password matches
*/
export async function verifyPassword(password, hash) {
async function verifyPassword(password, hash) {
return bcrypt.compare(password, hash);
}
@@ -35,11 +35,11 @@ export async function verifyPassword(password, hash) {
*/
export async function registerUser({ email, password, callsign }) {
// Check if user already exists
const existingUser = await db
const [existingUser] = await db
.select()
.from(users)
.where(eq(users.email, email))
.get();
.limit(1);
if (existingUser) {
throw new Error('Email already registered');
@@ -72,11 +72,11 @@ export async function registerUser({ email, password, callsign }) {
*/
export async function authenticateUser(email, password) {
// Find user by email
const user = await db
const [user] = await db
.select()
.from(users)
.where(eq(users.email, email))
.get();
.limit(1);
if (!user) {
throw new Error('Invalid email or password');
@@ -99,11 +99,11 @@ export async function authenticateUser(email, password) {
* @returns {Promise<Object|null>} User object (without password) or null
*/
export async function getUserById(userId) {
const user = await db
const [user] = await db
.select()
.from(users)
.where(eq(users.id, userId))
.get();
.limit(1);
if (!user) return null;

39
src/backend/services/job-queue.service.js
View File

@@ -1,6 +1,7 @@
import { db } from '../config/database.js';
import { syncJobs } from '../db/schema/index.js';
import { eq, and, desc, or, lt } from 'drizzle-orm';
import logger from '../config/logger.js';
/**
* Background Job Queue Service
@@ -43,12 +44,12 @@ export function registerProcessor(type, processor) {
* @returns {Promise<Object>} Job object with ID
*/
export async function enqueueJob(userId, type, data = {}) {
console.error('[enqueueJob] Starting job enqueue:', { userId, type, hasData: !!data });
logger.debug('Enqueueing job', { userId, type });
// Check for existing active job of same type for this user
const existingJob = await getUserActiveJob(userId, type);
if (existingJob) {
console.error('[enqueueJob] Found existing active job:', existingJob.id);
logger.debug('Existing active job found', { jobId: existingJob.id });
return {
success: false,
error: `A ${type} job is already running or pending for this user`,
@@ -57,7 +58,6 @@ export async function enqueueJob(userId, type, data = {}) {
}
// Create job record
console.error('[enqueueJob] Creating job record in database...');
const [job] = await db
.insert(syncJobs)
.values({
@@ -68,11 +68,11 @@ export async function enqueueJob(userId, type, data = {}) {
})
.returning();
console.error('[enqueueJob] Job created:', job.id);
logger.info('Job created', { jobId: job.id, type, userId });
// Start processing asynchronously (don't await)
processJobAsync(job.id, userId, type, data).catch((error) => {
console.error(`[enqueueJob] Error processing job ${job.id}:`, error);
logger.error(`Job processing error`, { jobId: job.id, error: error.message });
});
return {
@@ -145,7 +145,7 @@ async function processJobAsync(jobId, userId, type, data) {
* @param {number} jobId - Job ID
* @param {Object} updates - Fields to update
*/
export async function updateJob(jobId, updates) {
async function updateJob(jobId, updates) {
await db.update(syncJobs).set(updates).where(eq(syncJobs.id, jobId));
}
@@ -154,7 +154,7 @@ export async function updateJob(jobId, updates) {
* @param {number} jobId - Job ID
* @returns {Promise<Object|null>} Job object or null
*/
export async function getJob(jobId) {
async function getJob(jobId) {
const [job] = await db.select().from(syncJobs).where(eq(syncJobs.id, jobId)).limit(1);
return job || null;
}
@@ -174,7 +174,7 @@ export async function getJobStatus(jobId) {
try {
parsedResult = JSON.parse(job.result);
} catch (e) {
console.error('Failed to parse job result:', e);
logger.warn('Failed to parse job result', { jobId, error: e.message });
}
}
@@ -198,8 +198,6 @@ export async function getJobStatus(jobId) {
* @returns {Promise<Object|null>} Active job or null
*/
export async function getUserActiveJob(userId, type = null) {
console.error('[getUserActiveJob] Querying for active job:', { userId, type });
// Build the where clause properly with and() and or()
const conditions = [
eq(syncJobs.userId, userId),
@@ -213,20 +211,14 @@ export async function getUserActiveJob(userId, type = null) {
conditions.push(eq(syncJobs.type, type));
}
try {
const [job] = await db
.select()
.from(syncJobs)
.where(and(...conditions))
.orderBy(desc(syncJobs.createdAt))
.limit(1);
const [job] = await db
.select()
.from(syncJobs)
.where(and(...conditions))
.orderBy(desc(syncJobs.createdAt))
.limit(1);
console.error('[getUserActiveJob] Result:', job ? `Found job ${job.id}` : 'No active job');
return job || null;
} catch (error) {
console.error('[getUserActiveJob] Database error:', error);
throw error;
}
return job || null;
}
/**
@@ -284,6 +276,7 @@ export async function cleanupOldJobs(daysOld = 7) {
)
);
logger.info('Cleaned up old jobs', { count: result, daysOld });
return result;
}

624
src/backend/services/lotw.service.js
View File

@@ -1,7 +1,8 @@
import { db } from '../config/database.js';
import { qsos } from '../db/schema/index.js';
import { max, sql } from 'drizzle-orm';
import { max, sql, eq, and, desc } from 'drizzle-orm';
import { registerProcessor, updateJobProgress } from './job-queue.service.js';
import logger from '../config/logger.js';
/**
* LoTW (Logbook of the World) Service
@@ -9,41 +10,25 @@ import { registerProcessor, updateJobProgress } from './job-queue.service.js';
*/
// Wavelog-compatible constants
const LOTW_CONNECT_TIMEOUT = 30; // CURLOPT_CONNECTTIMEOUT from Wavelog
const LOTW_CONNECT_TIMEOUT = 30;
// Configuration for long-polling
const POLLING_CONFIG = {
maxRetries: 30, // Maximum number of retry attempts
retryDelay: 10000, // Delay between retries in ms (10 seconds)
requestTimeout: 60000, // Timeout for individual requests in ms (1 minute)
maxTotalTime: 600000, // Maximum total time to wait in ms (10 minutes)
maxRetries: 30,
retryDelay: 10000,
requestTimeout: 60000,
maxTotalTime: 600000,
};
/**
* Check if LoTW response indicates the report is still being prepared
* @param {string} responseData - The response text from LoTW
* @returns {boolean} True if report is still pending
*/
function isReportPending(responseData) {
const trimmed = responseData.trim().toLowerCase();
// LoTW returns various messages when report is not ready:
// - Empty responses
// - "Report is being prepared" or similar messages
// - HTML error pages
// - Very short responses that aren't valid ADIF
if (trimmed.length < 100) return true;
if (trimmed.includes('<html>') || trimmed.includes('<!doctype html>')) return true;
// Check for empty or very short responses
if (trimmed.length < 100) {
return true;
}
// Check for HTML responses (error pages)
if (trimmed.includes('<html>') || trimmed.includes('<!doctype html>')) {
return true;
}
// Check for common "not ready" messages
const pendingMessages = [
'report is being prepared',
'your report is being generated',
@@ -54,12 +39,9 @@ function isReportPending(responseData) {
];
for (const msg of pendingMessages) {
if (trimmed.includes(msg)) {
return true;
}
if (trimmed.includes(msg)) return true;
}
// Check if it looks like valid ADIF data (should start with <ADIF_VER: or have <qso_date)
const hasAdifHeader = trimmed.includes('<adif_ver:') ||
trimmed.includes('<qso_date') ||
trimmed.includes('<call:') ||
@@ -68,79 +50,57 @@ function isReportPending(responseData) {
return !hasAdifHeader;
}
/**
* Sleep/delay utility
* @param {number} ms - Milliseconds to sleep
* @returns {Promise<void>}
*/
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
/**
* Fetch QSOs from LoTW with long-polling support
* @param {string} lotwUsername - LoTW username
* @param {string} lotwPassword - LoTW password
* @param {Date} sinceDate - Only fetch QSOs since this date
* @returns {Promise<Array>} Array of QSO objects
*/
export async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
// LoTW report URL
async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
const url = 'https://lotw.arrl.org/lotwuser/lotwreport.adi';
// Build query parameters - qso_query=1 is REQUIRED to get QSO records!
const params = new URLSearchParams({
login: lotwUsername,
password: lotwPassword,
qso_query: '1', // REQUIRED: Without this, no QSO records are returned
qso_qsl: 'yes', // Only get QSOs with QSLs (confirmed)
qso_qsldetail: 'yes', // Include QSL details (station location)
qso_mydetail: 'yes', // Include my station details
qso_withown: 'yes', // Include own callsign
qso_query: '1',
qso_qsl: 'yes',
qso_qsldetail: 'yes',
qso_mydetail: 'yes',
qso_withown: 'yes',
});
// Add date filter - only add qso_qslsince if we have a last QSL date
// For first sync (no QSOs in DB), don't filter by date to get ALL QSOs
if (sinceDate) {
const dateStr = sinceDate.toISOString().split('T')[0];
params.append('qso_qslsince', dateStr);
console.error('Date filter:', dateStr, '(Incremental sync since last QSL date)');
logger.debug('Incremental sync since', { date: dateStr });
} else {
console.error('No date filter - fetching ALL QSOs (first sync)');
logger.debug('Full sync - fetching all QSOs');
}
const fullUrl = `${url}?${params.toString()}`;
console.error('Fetching from LoTW:', fullUrl.replace(/password=[^&]+/, 'password=***'));
logger.debug('Fetching from LoTW', { url: fullUrl.replace(/password=[^&]+/, 'password=***') });
const startTime = Date.now();
// Long-polling loop
for (let attempt = 0; attempt < POLLING_CONFIG.maxRetries; attempt++) {
const elapsed = Date.now() - startTime;
if (elapsed > POLLING_CONFIG.maxTotalTime) {
throw new Error(`LoTW sync timeout: exceeded maximum wait time of ${POLLING_CONFIG.maxTotalTime / 1000} seconds`);
}
if (attempt > 0) {
logger.debug(`Retry attempt ${attempt + 1}/${POLLING_CONFIG.maxRetries}`, { elapsed: Math.round(elapsed / 1000) });
}
try {
// Check if we've exceeded max total time
const elapsed = Date.now() - startTime;
if (elapsed > POLLING_CONFIG.maxTotalTime) {
throw new Error(`LoTW sync timeout: exceeded maximum wait time of ${POLLING_CONFIG.maxTotalTime / 1000} seconds`);
}
if (attempt > 0) {
console.error(`Retry attempt ${attempt + 1}/${POLLING_CONFIG.maxRetries} (elapsed: ${Math.round(elapsed / 1000)}s)`);
}
// Make request with timeout
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), POLLING_CONFIG.requestTimeout);
const response = await fetch(fullUrl, { signal: controller.signal });
clearTimeout(timeoutId);
// Handle HTTP errors
if (!response.ok) {
if (response.status === 503) {
// Service unavailable - might be temporary, retry
console.error('LoTW returned 503 (Service Unavailable), waiting before retry...');
logger.warn('LoTW returned 503, retrying...');
await sleep(POLLING_CONFIG.retryDelay);
continue;
} else if (response.status === 401) {
@@ -148,62 +108,48 @@ export async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate =
} else if (response.status === 404) {
throw new Error('LoTW service not found (404). The LoTW API URL may have changed.');
} else {
// Other errors - log but retry
console.error(`LoTW returned ${response.status} ${response.statusText}, waiting before retry...`);
logger.warn(`LoTW returned ${response.status}, retrying...`);
await sleep(POLLING_CONFIG.retryDelay);
continue;
}
}
// Get response text
const adifData = await response.text();
console.error(`Response length: ${adifData.length} bytes`);
// Wavelog: Validate response for credential errors
if (adifData.toLowerCase().includes('username/password incorrect')) {
throw new Error('Username/password incorrect');
}
// Wavelog: Check if file starts with expected header
const header = adifData.trim().substring(0, 39).toLowerCase();
if (!header.includes('arrl logbook of the world')) {
// This might be because the report is still pending
if (isReportPending(adifData)) {
console.error('LoTW report is still being prepared, waiting...', adifData.substring(0, 100));
logger.debug('LoTW report still being prepared, waiting...');
await sleep(POLLING_CONFIG.retryDelay);
continue;
}
throw new Error('Downloaded LoTW report is invalid. Check your credentials.');
}
// We have valid data!
console.error('LoTW report ready, parsing ADIF data...');
console.error('ADIF preview:', adifData.substring(0, 200));
logger.info('LoTW report downloaded successfully', { size: adifData.length });
// Parse ADIF format
const qsos = parseADIF(adifData);
console.error(`Successfully parsed ${qsos.length} QSOs from LoTW`);
logger.info('Parsed QSOs from LoTW', { count: qsos.length });
return qsos;
} catch (error) {
const elapsed = Date.now() - startTime;
if (error.name === 'AbortError') {
console.error(`Request timeout on attempt ${attempt + 1}, retrying...`);
logger.debug('Request timeout, retrying...');
await sleep(POLLING_CONFIG.retryDelay);
continue;
}
// Re-throw credential/auth errors immediately
if (error.message.includes('credentials') || error.message.includes('401') || error.message.includes('404')) {
throw error;
}
// For other errors, log and retry if we haven't exhausted retries
if (attempt < POLLING_CONFIG.maxRetries - 1) {
console.error(`Error on attempt ${attempt + 1}: ${error.message}`);
console.error('Retrying...');
logger.warn(`Error on attempt ${attempt + 1}`, { error: error.message });
await sleep(POLLING_CONFIG.retryDelay);
continue;
} else {
@@ -212,73 +158,47 @@ export async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate =
}
}
// If we get here, we exhausted all retries
const totalTime = Math.round((Date.now() - startTime) / 1000);
throw new Error(`LoTW sync failed: Report not ready after ${POLLING_CONFIG.maxRetries} attempts (${totalTime}s). LoTW may be experiencing high load. Please try again later.`);
}
/**
* Parse ADIF (Amateur Data Interchange Format) data
* @param {string} adifData - Raw ADIF text data
* @returns {Array<Array>} Array of QSOs (each QSO is an array of field objects)
*/
function parseADIF(adifData) {
const qsos = [];
const records = adifData.split('<eor>');
console.error(`Total records after splitting by <eor>: ${records.length}`);
for (let i = 0; i < records.length; i++) {
const record = records[i];
// Skip empty records or records that are just header info
for (const record of records) {
if (!record.trim()) continue;
if (record.trim().startsWith('<') && !record.includes('<CALL:') && !record.includes('<call:')) continue;
const qso = {};
// Match ADIF fields: <FIELDNAME:length[:type]>value
// Important: The 'type' part is optional, and field names can contain underscores
// We use the length parameter to extract exactly that many characters
const regex = /<([A-Z_]+):(\d+)(?::[A-Z]+)?>([\s\S])/gi;
let match;
while ((match = regex.exec(record)) !== null) {
const [fullMatch, fieldName, lengthStr, firstChar] = match;
const length = parseInt(lengthStr, 10);
// Extract exactly 'length' characters starting from the position after the '>'
const valueStart = match.index + fullMatch.length - 1; // -1 because firstChar is already captured
const valueStart = match.index + fullMatch.length - 1;
const value = record.substring(valueStart, valueStart + length);
qso[fieldName.toLowerCase()] = value.trim();
// Move regex lastIndex to the end of the value so we can find the next tag
regex.lastIndex = valueStart + length;
}
// Only add if we have actual QSO data (has CALL or call field)
if (Object.keys(qso).length > 0 && (qso.call || qso.call)) {
qsos.push(qso);
// Log first few QSOs for debugging
if (qsos.length <= 3) {
console.error(`Parsed QSO #${qsos.length}: ${qso.call} on ${qso.qso_date} ${qso.band} ${qso.mode}`);
}
}
}
console.error(`Total QSOs parsed: ${qsos.length}`);
return qsos;
}
/**
* Convert ADIF QSO to database format
* @param {Object} adifQSO - QSO object from ADIF parser
* @param {number} userId - User ID
* @returns {Object} Database-ready QSO object
*/
export function convertQSODatabaseFormat(adifQSO, userId) {
function convertQSODatabaseFormat(adifQSO, userId) {
return {
userId,
callsign: adifQSO.call || '',
@@ -303,66 +223,29 @@ export function convertQSODatabaseFormat(adifQSO, userId) {
};
}
/**
* Normalize band name
* @param {string} band - Band from ADIF
* @returns {string|null} Normalized band
*/
function normalizeBand(band) {
if (!band) return null;
const bandMap = {
'160m': '160m',
'80m': '80m',
'60m': '60m',
'40m': '40m',
'30m': '30m',
'20m': '20m',
'17m': '17m',
'15m': '15m',
'12m': '12m',
'10m': '10m',
'6m': '6m',
'4m': '4m',
'2m': '2m',
'1.25m': '1.25m',
'70cm': '70cm',
'33cm': '33cm',
'23cm': '23cm',
'13cm': '13cm',
'9cm': '9cm',
'6cm': '6cm',
'3cm': '3cm',
'1.2cm': '1.2cm',
'mm': 'mm',
'160m': '160m', '80m': '80m', '60m': '60m', '40m': '40m',
'30m': '30m', '20m': '20m', '17m': '17m', '15m': '15m',
'12m': '12m', '10m': '10m', '6m': '6m', '4m': '4m',
'2m': '2m', '1.25m': '1.25m', '70cm': '70cm', '33cm': '33cm',
'23cm': '23cm', '13cm': '13cm', '9cm': '9cm', '6cm': '6cm',
'3cm': '3cm', '1.2cm': '1.2cm', 'mm': 'mm',
};
return bandMap[band.toLowerCase()] || band;
}
/**
* Normalize mode name
* @param {string} mode - Mode from ADIF
* @returns {string} Normalized mode
*/
function normalizeMode(mode) {
if (!mode) return '';
const modeMap = {
'cw': 'CW',
'ssb': 'SSB',
'am': 'AM',
'fm': 'FM',
'rtty': 'RTTY',
'psk31': 'PSK31',
'psk63': 'PSK63',
'ft8': 'FT8',
'ft4': 'FT4',
'jt65': 'JT65',
'jt9': 'JT9',
'js8': 'JS8',
'mfsk': 'MFSK',
' Olivia': 'OLIVIA',
'cw': 'CW', 'ssb': 'SSB', 'am': 'AM', 'fm': 'FM',
'rtty': 'RTTY', 'psk31': 'PSK31', 'psk63': 'PSK63',
'ft8': 'FT8', 'ft4': 'FT4', 'jt65': 'JT65', 'jt9': 'JT9',
'js8': 'JS8', 'mfsk': 'MFSK', 'olivia': 'OLIVIA',
};
const normalized = modeMap[mode.toLowerCase()];
@@ -371,139 +254,80 @@ function normalizeMode(mode) {
/**
* Sync QSOs from LoTW to database
* @param {number} userId - User ID
* @param {string} lotwUsername - LoTW username
* @param {string} lotwPassword - LoTW password
* @param {Date} sinceDate - Only sync QSOs since this date
* @returns {Promise<Object>} Sync result with counts
*/
export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null) {
try {
// Fetch QSOs from LoTW
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = null) {
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
if (!adifQSOs || adifQSOs.length === 0) {
return {
success: true,
total: 0,
added: 0,
updated: 0,
message: 'No QSOs found in LoTW',
};
}
let addedCount = 0;
let updatedCount = 0;
const errors = [];
// Process each QSO
for (const qsoData of adifQSOs) {
try {
console.error('Raw ADIF QSO data:', JSON.stringify(qsoData));
const dbQSO = convertQSODatabaseFormat(qsoData, userId);
console.error('Converted QSO:', JSON.stringify(dbQSO));
console.error('Processing QSO:', dbQSO.callsign, dbQSO.qsoDate, dbQSO.band, dbQSO.mode);
// Check if QSO already exists (by callsign, date, time, band, mode)
const { eq } = await import('drizzle-orm');
const existing = await db
.select()
.from(qsos)
.where(eq(qsos.userId, userId))
.where(eq(qsos.callsign, dbQSO.callsign))
.where(eq(qsos.qsoDate, dbQSO.qsoDate))
.where(eq(qsos.band, dbQSO.band))
.where(eq(qsos.mode, dbQSO.mode))
.limit(1);
console.error('Existing QSOs found:', existing.length);
if (existing.length > 0) {
console.error('Existing QSO:', JSON.stringify(existing[0]));
}
if (existing.length > 0) {
// Update existing QSO
console.error('Updating existing QSO');
await db
.update(qsos)
.set({
lotwQslRdate: dbQSO.lotwQslRdate,
lotwQslRstatus: dbQSO.lotwQslRstatus,
lotwSyncedAt: dbQSO.lotwSyncedAt,
})
.where(eq(qsos.id, existing[0].id));
updatedCount++;
} else {
// Insert new QSO
console.error('Inserting new QSO with data:', JSON.stringify(dbQSO));
try {
const result = await db.insert(qsos).values(dbQSO);
console.error('Insert result:', result);
addedCount++;
} catch (insertError) {
console.error('Insert failed:', insertError.message);
console.error('Insert error details:', insertError);
throw insertError;
}
}
} catch (error) {
console.error('ERROR processing QSO:', error);
errors.push({
qso: qsoData,
error: error.message,
});
}
}
return {
success: true,
total: adifQSOs.length,
added: addedCount,
updated: updatedCount,
errors: errors.length > 0 ? errors : undefined,
};
} catch (error) {
throw new Error(`LoTW sync failed: ${error.message}`);
if (!adifQSOs || adifQSOs.length === 0) {
return { success: true, total: 0, added: 0, updated: 0, message: 'No QSOs found in LoTW' };
}
let addedCount = 0;
let updatedCount = 0;
const errors = [];
for (const qsoData of adifQSOs) {
try {
const dbQSO = convertQSODatabaseFormat(qsoData, userId);
const existing = await db
.select()
.from(qsos)
.where(eq(qsos.userId, userId))
.where(eq(qsos.callsign, dbQSO.callsign))
.where(eq(qsos.qsoDate, dbQSO.qsoDate))
.where(eq(qsos.band, dbQSO.band))
.where(eq(qsos.mode, dbQSO.mode))
.limit(1);
if (existing.length > 0) {
await db
.update(qsos)
.set({
lotwQslRdate: dbQSO.lotwQslRdate,
lotwQslRstatus: dbQSO.lotwQslRstatus,
lotwSyncedAt: dbQSO.lotwSyncedAt,
})
.where(eq(qsos.id, existing[0].id));
updatedCount++;
} else {
await db.insert(qsos).values(dbQSO);
addedCount++;
}
} catch (error) {
logger.error('Error processing QSO', { error: error.message, qso: qsoData });
errors.push({ qso: qsoData, error: error.message });
}
}
logger.info('LoTW sync completed', { total: adifQSOs.length, added: addedCount, updated: updatedCount });
return {
success: true,
total: adifQSOs.length,
added: addedCount,
updated: updatedCount,
errors: errors.length > 0 ? errors : undefined,
};
}
/**
* Get QSOs for a user with pagination
* @param {number} userId - User ID
* @param {Object} filters - Query filters
* @param {Object} options - Pagination options { page, limit }
* @returns {Promise<Object>} Paginated QSOs
*/
export async function getUserQSOs(userId, filters = {}, options = {}) {
const { eq, and, desc, sql } = await import('drizzle-orm');
const { page = 1, limit = 100 } = options;
console.error('getUserQSOs called with userId:', userId, 'filters:', filters, 'page:', page, 'limit:', limit);
// Build where conditions
const conditions = [eq(qsos.userId, userId)];
if (filters.band) {
conditions.push(eq(qsos.band, filters.band));
}
if (filters.band) conditions.push(eq(qsos.band, filters.band));
if (filters.mode) conditions.push(eq(qsos.mode, filters.mode));
if (filters.confirmed) conditions.push(eq(qsos.lotwQslRstatus, 'Y'));
if (filters.mode) {
conditions.push(eq(qsos.mode, filters.mode));
}
if (filters.confirmed) {
conditions.push(eq(qsos.lotwQslRstatus, 'Y'));
}
// Get total count for pagination
const allResults = await db.select().from(qsos).where(and(...conditions));
const totalCount = allResults.length;
// Calculate offset
const offset = (page - 1) * limit;
// Get paginated results
const results = await db
.select()
.from(qsos)
@@ -512,8 +336,6 @@ export async function getUserQSOs(userId, filters = {}, options = {}) {
.limit(limit)
.offset(offset);
console.error('getUserQSOs returning', results.length, 'QSOs (page', page, 'of', Math.ceil(totalCount / limit), ')');
return {
qsos: results,
pagination: {
@@ -529,18 +351,11 @@ export async function getUserQSOs(userId, filters = {}, options = {}) {
/**
* Get QSO statistics for a user
* @param {number} userId - User ID
* @returns {Promise<Object>} Statistics object
*/
export async function getQSOStats(userId) {
const { eq } = await import('drizzle-orm');
const allQSOs = await db.select().from(qsos).where(eq(qsos.userId, userId));
console.error('getQSOStats called with userId:', userId, 'found', allQSOs.length, 'QSOs in database');
const confirmed = allQSOs.filter((q) => q.lotwQslRstatus === 'Y');
// Count unique entities
const uniqueEntities = new Set();
const uniqueBands = new Set();
const uniqueModes = new Set();
@@ -551,43 +366,28 @@ export async function getQSOStats(userId) {
if (q.mode) uniqueModes.add(q.mode);
});
const stats = {
return {
total: allQSOs.length,
confirmed: confirmed.length,
uniqueEntities: uniqueEntities.size,
uniqueBands: uniqueBands.size,
uniqueModes: uniqueModes.size,
};
console.error('getQSOStats returning:', stats);
return stats;
}
/**
* Get the date of the last LoTW QSL for a user
* Used for qso_qslsince parameter to minimize downloads
* @param {number} userId - User ID
* @returns {Promise<Date|null>} Last QSL date or null
*/
export async function getLastLoTWQSLDate(userId) {
const { eq } = await import('drizzle-orm');
// Get the most recent lotwQslRdate for this user
async function getLastLoTWQSLDate(userId) {
const [result] = await db
.select({ maxDate: max(qsos.lotwQslRdate) })
.from(qsos)
.where(eq(qsos.userId, userId));
if (!result || !result.maxDate) {
return null;
}
if (!result || !result.maxDate) return null;
// Parse ADIF date format (YYYYMMDD) to Date
const dateStr = result.maxDate;
if (!dateStr || dateStr === '') {
return null;
}
if (!dateStr || dateStr === '') return null;
const year = dateStr.substring(0, 4);
const month = dateStr.substring(4, 6);
@@ -596,172 +396,106 @@ export async function getLastLoTWQSLDate(userId) {
return new Date(`${year}-${month}-${day}`);
}
/**
* Validate LoTW response following Wavelog logic
* @param {string} responseData - Response from LoTW
* @returns {Object} { valid: boolean, error?: string }
*/
function validateLoTWResponse(responseData) {
const trimmed = responseData.trim();
// Wavelog: Check for username/password incorrect
if (trimmed.toLowerCase().includes('username/password incorrect')) {
return {
valid: false,
error: 'Username/password incorrect',
shouldClearCredentials: true,
};
}
// Wavelog: Check if file starts with "ARRL Logbook of the World Status Report"
const header = trimmed.substring(0, 39).toLowerCase();
if (!header.includes('arrl logbook of the world')) {
return {
valid: false,
error: 'Downloaded LoTW report is invalid. File does not start with expected header.',
};
}
return { valid: true };
}
/**
* LoTW sync job processor for the job queue
* @param {number} jobId - Job ID
* @param {number} userId - User ID
* @param {Object} data - Job data { lotwUsername, lotwPassword }
* @returns {Promise<Object>} Sync result
*/
export async function syncQSOsForJob(jobId, userId, data) {
const { lotwUsername, lotwPassword } = data;
try {
// Update job progress: starting
await updateJobProgress(jobId, {
message: 'Fetching QSOs from LoTW...',
step: 'fetch',
});
await updateJobProgress(jobId, {
message: 'Fetching QSOs from LoTW...',
step: 'fetch',
});
// Get last LoTW QSL date for incremental sync
const lastQSLDate = await getLastLoTWQSLDate(userId);
const lastQSLDate = await getLastLoTWQSLDate(userId);
const sinceDate = lastQSLDate || new Date('2000-01-01');
// If no QSOs exist, use a far past date to get ALL QSOs (first sync)
// Otherwise, use the last QSL date for incremental sync
const sinceDate = lastQSLDate || new Date('2000-01-01');
if (lastQSLDate) {
logger.info(`Job ${jobId}: Incremental sync`, { since: sinceDate.toISOString().split('T')[0] });
} else {
logger.info(`Job ${jobId}: Full sync`);
}
if (lastQSLDate) {
console.error(`[Job ${jobId}] Incremental sync since ${sinceDate.toISOString().split('T')[0]}`);
} else {
console.error(`[Job ${jobId}] Full sync - fetching ALL QSOs since 2000-01-01`);
}
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
// Fetch from LoTW
const adifQSOs = await fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate);
if (!adifQSOs || adifQSOs.length === 0) {
return { success: true, total: 0, added: 0, updated: 0, message: 'No QSOs found in LoTW' };
}
if (!adifQSOs || adifQSOs.length === 0) {
return {
success: true,
total: 0,
added: 0,
updated: 0,
message: 'No QSOs found in LoTW',
};
}
await updateJobProgress(jobId, {
message: `Processing ${adifQSOs.length} QSOs...`,
step: 'process',
total: adifQSOs.length,
processed: 0,
});
// Update job progress: processing
await updateJobProgress(jobId, {
message: `Processing ${adifQSOs.length} QSOs...`,
step: 'process',
total: adifQSOs.length,
processed: 0,
});
let addedCount = 0;
let updatedCount = 0;
const errors = [];
let addedCount = 0;
let updatedCount = 0;
const errors = [];
for (let i = 0; i < adifQSOs.length; i++) {
const qsoData = adifQSOs[i];
// Process each QSO
for (let i = 0; i < adifQSOs.length; i++) {
const qsoData = adifQSOs[i];
try {
const dbQSO = convertQSODatabaseFormat(qsoData, userId);
try {
const dbQSO = convertQSODatabaseFormat(qsoData, userId);
// Check if QSO already exists
const { eq, and } = await import('drizzle-orm');
const existing = await db
.select()
.from(qsos)
.where(
and(
eq(qsos.userId, userId),
eq(qsos.callsign, dbQSO.callsign),
eq(qsos.qsoDate, dbQSO.qsoDate),
eq(qsos.band, dbQSO.band),
eq(qsos.mode, dbQSO.mode)
)
const existing = await db
.select()
.from(qsos)
.where(
and(
eq(qsos.userId, userId),
eq(qsos.callsign, dbQSO.callsign),
eq(qsos.qsoDate, dbQSO.qsoDate),
eq(qsos.band, dbQSO.band),
eq(qsos.mode, dbQSO.mode)
)
.limit(1);
)
.limit(1);
if (existing.length > 0) {
// Update existing QSO
await db
.update(qsos)
.set({
lotwQslRdate: dbQSO.lotwQslRdate,
lotwQslRstatus: dbQSO.lotwQslRstatus,
lotwSyncedAt: dbQSO.lotwSyncedAt,
})
.where(eq(qsos.id, existing[0].id));
updatedCount++;
} else {
// Insert new QSO
await db.insert(qsos).values(dbQSO);
addedCount++;
}
if (existing.length > 0) {
await db
.update(qsos)
.set({
lotwQslRdate: dbQSO.lotwQslRdate,
lotwQslRstatus: dbQSO.lotwQslRstatus,
lotwSyncedAt: dbQSO.lotwSyncedAt,
})
.where(eq(qsos.id, existing[0].id));
updatedCount++;
} else {
await db.insert(qsos).values(dbQSO);
addedCount++;
}
// Update progress every 10 QSOs
if ((i + 1) % 10 === 0) {
await updateJobProgress(jobId, {
processed: i + 1,
message: `Processed ${i + 1}/${adifQSOs.length} QSOs...`,
});
}
} catch (error) {
console.error(`[Job ${jobId}] ERROR processing QSO:`, error);
errors.push({
qso: qsoData,
error: error.message,
if ((i + 1) % 10 === 0) {
await updateJobProgress(jobId, {
processed: i + 1,
message: `Processed ${i + 1}/${adifQSOs.length} QSOs...`,
});
}
} catch (error) {
logger.error(`Job ${jobId}: Error processing QSO`, { error: error.message });
errors.push({ qso: qsoData, error: error.message });
}
return {
success: true,
total: adifQSOs.length,
added: addedCount,
updated: updatedCount,
errors: errors.length > 0 ? errors : undefined,
};
} catch (error) {
// Check if it's a credential error
if (error.message.includes('Username/password incorrect')) {
throw new Error('Invalid LoTW credentials. Please check your username and password.');
}
throw error;
}
logger.info(`Job ${jobId} completed`, { total: adifQSOs.length, added: addedCount, updated: updatedCount });
return {
success: true,
total: adifQSOs.length,
added: addedCount,
updated: updatedCount,
errors: errors.length > 0 ? errors : undefined,
};
}
/**
* Delete all QSOs for a user
* @param {number} userId - User ID
* @returns {Promise<number>} Number of QSOs deleted
*/
export async function deleteQSOs(userId) {
const { eq } = await import('drizzle-orm');
const result = await db.delete(qsos).where(eq(qsos.userId, userId));
return result;
}