feat: implement DCL ADIF parser and service integration
- Add shared ADIF parser utility (src/backend/utils/adif-parser.js) - parseADIF(): Parse ADIF format into QSO records - parseDCLResponse(): Parse DCL's JSON response format - normalizeBand() and normalizeMode(): Standardize band/mode names - Implement DCL service (src/backend/services/dcl.service.js) - fetchQSOsFromDCL(): Fetch from DCL API (ready for API availability) - parseDCLJSONResponse(): Parse example payload format - syncQSOs(): Update existing QSOs with DCL confirmations - Support DCL-specific fields: DCL_QSL_RCVD, DCL_QSLRDATE, DARC_DOK, MY_DARC_DOK - Refactor LoTW service to use shared ADIF parser - Remove duplicate parseADIF, normalizeBand, normalizeMode functions - Import from shared utility for consistency - Tested with example DCL payload - Successfully parses all 6 QSOs - Correctly extracts DCL confirmation data - Handles ADIF format with <EOR> delimiters Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,184 +2,289 @@ import { db, logger } from '../config.js';
|
||||
import { qsos } from '../db/schema/index.js';
|
||||
import { max, sql, eq, and, desc } from 'drizzle-orm';
|
||||
import { updateJobProgress } from './job-queue.service.js';
|
||||
import { parseDCLResponse, normalizeBand, normalizeMode } from '../utils/adif-parser.js';
|
||||
|
||||
/**
|
||||
* DCL (DARC Community Logbook) Service
|
||||
*
|
||||
* NOTE: DCL does not currently have a public API for downloading QSOs.
|
||||
* This service is prepared as a stub for when DCL adds API support.
|
||||
*
|
||||
* When DCL provides an API, implement:
|
||||
* - fetchQSOsFromDCL() - Download QSOs from DCL
|
||||
* - syncQSOs() - Sync QSOs to database
|
||||
* - getLastDCLQSLDate() - Get last QSL date for incremental sync
|
||||
*
|
||||
* DCL Information:
|
||||
* - Website: https://dcl.darc.de/
|
||||
* - API: Coming soon (currently in development)
|
||||
* - ADIF Export: https://dcl.darc.de/dml/export_adif_form.php (manual only)
|
||||
* - DOK fields: MY_DARC_DOK (user's DOK), DARC_DOK (partner's DOK)
|
||||
*
|
||||
* Expected API Response Format:
|
||||
* {
|
||||
* "adif": "<ADIF_VER:5>3.1.3\\n<CREATED_TIMESTAMP:15>20260117 095453\\n<EOH>\\n..."
|
||||
* }
|
||||
*/
|
||||
|
||||
const REQUEST_TIMEOUT = 60000;
|
||||
|
||||
/**
|
||||
* Fetch QSOs from DCL
|
||||
* Fetch QSOs from DCL API
|
||||
*
|
||||
* TODO: Implement when DCL provides a download API
|
||||
* Expected implementation:
|
||||
* - Use DCL API key for authentication
|
||||
* - Fetch ADIF data with confirmations
|
||||
* - Parse and return QSO records
|
||||
* When DCL provides their API, update the URL and parameters.
|
||||
* Expected response format: { "adif": "<ADIF data>" }
|
||||
*
|
||||
* @param {string} dclApiKey - DCL API key
|
||||
* @param {Date|null} sinceDate - Last sync date for incremental sync
|
||||
* @returns {Promise<Array>} Array of parsed QSO records
|
||||
*/
|
||||
export async function fetchQSOsFromDCL(dclApiKey, sinceDate = null) {
|
||||
logger.info('DCL sync not yet implemented - API endpoint not available', {
|
||||
logger.info('Fetching QSOs from DCL', {
|
||||
hasApiKey: !!dclApiKey,
|
||||
sinceDate: sinceDate?.toISOString(),
|
||||
});
|
||||
|
||||
throw new Error('DCL download API is not yet available. DCL does not currently provide a public API for downloading QSOs. Use the manual ADIF export at https://dcl.darc.de/dml/export_adif_form.php');
|
||||
// TODO: Update URL when DCL publishes their API endpoint
|
||||
const url = 'https://dcl.darc.de/api/export'; // Placeholder URL
|
||||
|
||||
/*
|
||||
* FUTURE IMPLEMENTATION (when DCL provides API):
|
||||
*
|
||||
* const url = 'https://dcl.darc.de/api/...'; // TBA
|
||||
*
|
||||
* const params = new URLSearchParams({
|
||||
* api_key: dclApiKey,
|
||||
* format: 'adif',
|
||||
* qsl: 'yes',
|
||||
* });
|
||||
*
|
||||
* if (sinceDate) {
|
||||
* const dateStr = sinceDate.toISOString().split('T')[0].replace(/-/g, '');
|
||||
* params.append('qso_qslsince', dateStr);
|
||||
* }
|
||||
*
|
||||
* const response = await fetch(`${url}?${params}`, {
|
||||
* headers: {
|
||||
* 'Accept': 'text/plain',
|
||||
* },
|
||||
* timeout: REQUEST_TIMEOUT,
|
||||
* });
|
||||
*
|
||||
* if (!response.ok) {
|
||||
* throw new Error(`DCL API error: ${response.status}`);
|
||||
* }
|
||||
*
|
||||
* const adifData = await response.text();
|
||||
* return parseADIF(adifData);
|
||||
*/
|
||||
const params = new URLSearchParams({
|
||||
api_key: dclApiKey,
|
||||
format: 'json',
|
||||
qsl: 'yes',
|
||||
});
|
||||
|
||||
// Add date filter for incremental sync if provided
|
||||
if (sinceDate) {
|
||||
const dateStr = sinceDate.toISOString().split('T')[0].replace(/-/g, '');
|
||||
params.append('qsl_since', dateStr);
|
||||
}
|
||||
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT);
|
||||
|
||||
const response = await fetch(`${url}?${params}`, {
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 401) {
|
||||
throw new Error('Invalid DCL API key. Please check your DCL credentials in Settings.');
|
||||
} else if (response.status === 404) {
|
||||
throw new Error('DCL API endpoint not found. The DCL API may not be available yet.');
|
||||
} else {
|
||||
throw new Error(`DCL API error: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
// Parse the DCL response format
|
||||
const qsos = parseDCLResponse(data);
|
||||
|
||||
logger.info('Successfully fetched QSOs from DCL', {
|
||||
total: qsos.length,
|
||||
hasConfirmations: qsos.filter(q => qso.dcl_qsl_rcvd === 'Y').length,
|
||||
});
|
||||
|
||||
return qsos;
|
||||
|
||||
} catch (error) {
|
||||
if (error.name === 'AbortError') {
|
||||
throw new Error('DCL API request timed out. Please try again.');
|
||||
}
|
||||
|
||||
logger.error('Failed to fetch from DCL', {
|
||||
error: error.message,
|
||||
url: url.replace(/api_key=[^&]+/, 'api_key=***'),
|
||||
});
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse ADIF data from DCL
|
||||
* Parse DCL API response from JSON
|
||||
* This function exists for testing with example payloads before DCL API is available
|
||||
*
|
||||
* TODO: Implement ADIF parser for DCL format
|
||||
* Should handle DCL-specific fields:
|
||||
* - MY_DARC_DOK
|
||||
* - DARC_DOK
|
||||
*
|
||||
* @param {string} adifData - Raw ADIF data
|
||||
* @param {Object} jsonResponse - JSON response in DCL format
|
||||
* @returns {Array} Array of parsed QSO records
|
||||
*/
|
||||
function parseADIF(adifData) {
|
||||
// TODO: Implement ADIF parser
|
||||
// Should parse standard ADIF fields plus DCL-specific fields:
|
||||
// - MY_DARC_DOK (user's own DOK)
|
||||
// - DARC_DOK (QSO partner's DOK)
|
||||
// - QSL_DATE (confirmation date from DCL)
|
||||
export function parseDCLJSONResponse(jsonResponse) {
|
||||
return parseDCLResponse(jsonResponse);
|
||||
}
|
||||
|
||||
return [];
|
||||
/**
|
||||
* Convert DCL ADIF QSO to database format
|
||||
* @param {Object} adifQSO - Parsed ADIF QSO record
|
||||
* @param {number} userId - User ID
|
||||
* @returns {Object} Database-ready QSO object
|
||||
*/
|
||||
function convertQSODatabaseFormat(adifQSO, userId) {
|
||||
return {
|
||||
userId,
|
||||
callsign: adifQSO.call || '',
|
||||
qsoDate: adifQSO.qso_date || '',
|
||||
timeOn: adifQSO.time_on || adifQSO.time_off || '000000',
|
||||
band: normalizeBand(adifQSO.band),
|
||||
mode: normalizeMode(adifQSO.mode),
|
||||
freq: adifQSO.freq ? parseInt(adifQSO.freq) : null,
|
||||
freqRx: adifQSO.freq_rx ? parseInt(adifQSO.freq_rx) : null,
|
||||
entity: adifQSO.country || adifQSO.dxcc_country || '',
|
||||
entityId: adifQSO.dxcc ? parseInt(adifQSO.dxcc) : null,
|
||||
grid: adifQSO.gridsquare || '',
|
||||
gridSource: adifQSO.gridsquare ? 'DCL' : null,
|
||||
continent: adifQSO.continent || '',
|
||||
cqZone: adifQSO.cq_zone ? parseInt(adifQSO.cq_zone) : null,
|
||||
ituZone: adifQSO.itu_zone ? parseInt(adifQSO.itu_zone) : null,
|
||||
state: adifQSO.state || adifQSO.us_state || '',
|
||||
county: adifQSO.county || '',
|
||||
satName: adifQSO.sat_name || '',
|
||||
satMode: adifQSO.sat_mode || '',
|
||||
myDarcDok: adifQSO.my_darc_dok || '',
|
||||
darcDok: adifQSO.darc_dok || '',
|
||||
// DCL confirmation fields
|
||||
dclQslRdate: adifQSO.dcl_qslrdate || '',
|
||||
dclQslRstatus: adifQSO.dcl_qsl_rcvd || 'N',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync QSOs from DCL to database
|
||||
*
|
||||
* TODO: Implement when DCL provides API
|
||||
* Updates existing QSOs with DCL confirmation data
|
||||
*
|
||||
* @param {number} userId - User ID
|
||||
* @param {string} dclApiKey - DCL API key
|
||||
* @param {Date|null} sinceDate - Last sync date
|
||||
* @param {Date|null} sinceDate - Last sync date for incremental sync
|
||||
* @param {number|null} jobId - Job ID for progress tracking
|
||||
* @returns {Promise<Object>} Sync results
|
||||
*/
|
||||
export async function syncQSOs(userId, dclApiKey, sinceDate = null, jobId = null) {
|
||||
logger.info('DCL sync not yet implemented', { userId, sinceDate, jobId });
|
||||
logger.info('Starting DCL sync', { userId, sinceDate, jobId });
|
||||
|
||||
throw new Error('DCL download API is not yet available');
|
||||
if (jobId) {
|
||||
await updateJobProgress(jobId, {
|
||||
message: 'Fetching QSOs from DCL...',
|
||||
step: 'fetch',
|
||||
});
|
||||
}
|
||||
|
||||
/*
|
||||
* FUTURE IMPLEMENTATION:
|
||||
*
|
||||
* try {
|
||||
* const adifQSOs = await fetchQSOsFromDCL(dclApiKey, sinceDate);
|
||||
*
|
||||
* let addedCount = 0;
|
||||
* let updatedCount = 0;
|
||||
* let errors = [];
|
||||
*
|
||||
* for (const adifQSO of adifQSOs) {
|
||||
* try {
|
||||
* // Map ADIF fields to database schema
|
||||
* const qsoData = mapADIFToDB(adifQSO);
|
||||
*
|
||||
* // Check if QSO already exists
|
||||
* const existing = await db.select()
|
||||
* .from(qsos)
|
||||
* .where(
|
||||
* and(
|
||||
* eq(qsos.userId, userId),
|
||||
* eq(qsos.callsign, adifQSO.call),
|
||||
* eq(qsos.qsoDate, adifQSO.qso_date),
|
||||
* eq(qsos.timeOn, adifQSO.time_on)
|
||||
* )
|
||||
* )
|
||||
* .limit(1);
|
||||
*
|
||||
* if (existing.length > 0) {
|
||||
* // Update existing QSO with DCL confirmation
|
||||
* await db.update(qsos)
|
||||
* .set({
|
||||
* dclQslRdate: adifQSO.qslrdate || null,
|
||||
* dclQslRstatus: adifQSO.qslrdate ? 'Y' : 'N',
|
||||
* darcDok: adifQSO.darc_dok || null,
|
||||
* myDarcDok: adifQSO.my_darc_dok || null,
|
||||
* })
|
||||
* .where(eq(qsos.id, existing[0].id));
|
||||
* updatedCount++;
|
||||
* } else {
|
||||
* // Insert new QSO
|
||||
* await db.insert(qsos).values({
|
||||
* userId,
|
||||
* ...qsoData,
|
||||
* dclQslRdate: adifQSO.qslrdate || null,
|
||||
* dclQslRstatus: adifQSO.qslrdate ? 'Y' : 'N',
|
||||
* });
|
||||
* addedCount++;
|
||||
* }
|
||||
* } catch (err) {
|
||||
* logger.error('Failed to process QSO', { error: err.message, qso: adifQSO });
|
||||
* errors.push(err.message);
|
||||
* }
|
||||
* }
|
||||
*
|
||||
* const result = {
|
||||
* success: true,
|
||||
* total: adifQSOs.length,
|
||||
* added: addedCount,
|
||||
* updated: updatedCount,
|
||||
* errors,
|
||||
* };
|
||||
*
|
||||
* logger.info('DCL sync completed', { ...result, jobId });
|
||||
* return result;
|
||||
*
|
||||
* } catch (error) {
|
||||
* logger.error('DCL sync failed', { error: error.message, userId, jobId });
|
||||
* return { success: false, error: error.message, total: 0, added: 0, updated: 0 };
|
||||
* }
|
||||
*/
|
||||
try {
|
||||
const adifQSOs = await fetchQSOsFromDCL(dclApiKey, sinceDate);
|
||||
|
||||
if (!Array.isArray(adifQSOs) || adifQSOs.length === 0) {
|
||||
logger.info('No QSOs found in DCL response', { userId });
|
||||
return {
|
||||
success: true,
|
||||
total: 0,
|
||||
added: 0,
|
||||
updated: 0,
|
||||
message: 'No QSOs found in DCL',
|
||||
};
|
||||
}
|
||||
|
||||
if (jobId) {
|
||||
await updateJobProgress(jobId, {
|
||||
message: `Processing ${adifQSOs.length} QSOs from DCL...`,
|
||||
step: 'process',
|
||||
total: adifQSOs.length,
|
||||
processed: 0,
|
||||
});
|
||||
}
|
||||
|
||||
let addedCount = 0;
|
||||
let updatedCount = 0;
|
||||
const errors = [];
|
||||
|
||||
for (let i = 0; i < adifQSOs.length; i++) {
|
||||
const adifQSO = adifQSOs[i];
|
||||
|
||||
try {
|
||||
const dbQSO = convertQSODatabaseFormat(adifQSO, userId);
|
||||
|
||||
// Check if QSO already exists (match by callsign, date, time, band, mode)
|
||||
const existing = await db
|
||||
.select()
|
||||
.from(qsos)
|
||||
.where(
|
||||
and(
|
||||
eq(qsos.userId, userId),
|
||||
eq(qsos.callsign, dbQSO.callsign),
|
||||
eq(qsos.qsoDate, dbQSO.qsoDate),
|
||||
eq(qsos.timeOn, dbQSO.timeOn),
|
||||
eq(qsos.band, dbQSO.band),
|
||||
eq(qsos.mode, dbQSO.mode)
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
if (existing.length > 0) {
|
||||
// Update existing QSO with DCL confirmation and DOK data
|
||||
await db
|
||||
.update(qsos)
|
||||
.set({
|
||||
dclQslRdate: dbQSO.dclQslRdate,
|
||||
dclQslRstatus: dbQSO.dclQslRstatus,
|
||||
darcDok: dbQSO.darcDok || existing[0].darcDok,
|
||||
myDarcDok: dbQSO.myDarcDok || existing[0].myDarcDok,
|
||||
grid: dbQSO.grid || existing[0].grid,
|
||||
gridSource: dbQSO.gridSource || existing[0].gridSource,
|
||||
})
|
||||
.where(eq(qsos.id, existing[0].id));
|
||||
updatedCount++;
|
||||
} else {
|
||||
// Insert new QSO
|
||||
await db.insert(qsos).values(dbQSO);
|
||||
addedCount++;
|
||||
}
|
||||
|
||||
// Update job progress every 10 QSOs
|
||||
if (jobId && (i + 1) % 10 === 0) {
|
||||
await updateJobProgress(jobId, {
|
||||
processed: i + 1,
|
||||
message: `Processed ${i + 1}/${adifQSOs.length} QSOs from DCL...`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to process DCL QSO', {
|
||||
error: error.message,
|
||||
qso: adifQSO,
|
||||
userId,
|
||||
});
|
||||
errors.push({ qso: adifQSO, error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
const result = {
|
||||
success: true,
|
||||
total: adifQSOs.length,
|
||||
added: addedCount,
|
||||
updated: updatedCount,
|
||||
confirmed: adifQSOs.filter(q => q.dcl_qsl_rcvd === 'Y').length,
|
||||
errors: errors.length > 0 ? errors : undefined,
|
||||
};
|
||||
|
||||
logger.info('DCL sync completed', {
|
||||
...result,
|
||||
userId,
|
||||
jobId,
|
||||
});
|
||||
|
||||
return result;
|
||||
|
||||
} catch (error) {
|
||||
logger.error('DCL sync failed', {
|
||||
error: error.message,
|
||||
userId,
|
||||
jobId,
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
total: 0,
|
||||
added: 0,
|
||||
updated: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user