Compare commits

...

2 Commits

Author SHA1 Message Date
cce520a00e chore: code cleanup - remove duplicates and add caching
- Delete duplicate getCacheStats() function in cache.service.js
- Fix date calculation bug in lotw.service.js (was Date.now()-Date.now())
- Extract duplicate helper functions (yieldToEventLoop, getQSOKey) to sync-helpers.js
- Cache award definitions in memory to avoid repeated file I/O
- Delete unused parseDCLJSONResponse() function
- Remove unused imports (getPerformanceSummary, resetPerformanceMetrics)
- Auto-discover award JSON files instead of hardcoded list

Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-22 10:22:00 +01:00
d9e0e462b9 docs: update CLAUDE.md with recent changes
Rewrote documentation to reflect current state of the application:

- Updated award list (DXCC, DXCC SAT, DLD, WAS, VUCC SAT, SAT-RS44, 73 on 73)
- Added allowed_bands and satellite_only rule options
- Added DXCC SAT award documentation
- Added QSO deletion endpoint
- Updated award detail view features (unique entity progress, satellite grouping, band sorting)
- Removed outdated DLD variant examples
- Streamlined and reorganized sections

Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-22 09:47:11 +01:00
6 changed files with 129 additions and 447 deletions

450
CLAUDE.md
View File

@@ -170,6 +170,8 @@ The award system is JSON-driven and located in `award-definitions/` directory. E
1. **`entity`**: Count unique entities (DXCC countries, states, grid squares) 1. **`entity`**: Count unique entities (DXCC countries, states, grid squares)
- `entityType`: What to count ("dxcc", "state", "grid", "callsign") - `entityType`: What to count ("dxcc", "state", "grid", "callsign")
- `target`: Number required for award - `target`: Number required for award
- `allowed_bands`: Optional array of bands that count (e.g., `["160m", "80m", "40m", "30m", "20m", "17m", "15m", "12m", "10m"]` for HF only)
- `satellite_only`: Optional boolean to only count satellite QSOs (QSOs with `satName` field)
- `filters`: Optional filters (band, mode, etc.) - `filters`: Optional filters (band, mode, etc.)
- `displayField`: Optional field to display - `displayField`: Optional field to display
@@ -179,7 +181,6 @@ The award system is JSON-driven and located in `award-definitions/` directory. E
- `filters`: Optional filters (band, mode, etc.) for award variants - `filters`: Optional filters (band, mode, etc.) for award variants
- Counts unique (DOK, band, mode) combinations - Counts unique (DOK, band, mode) combinations
- Only DCL-confirmed QSOs count - Only DCL-confirmed QSOs count
- Example variants: DLD 80m, DLD CW, DLD 80m CW
3. **`points`**: Point-based awards 3. **`points`**: Point-based awards
- `stations`: Array of {callsign, points} - `stations`: Array of {callsign, points}
@@ -192,6 +193,16 @@ The award system is JSON-driven and located in `award-definitions/` directory. E
5. **`counter`**: Count QSOs or callsigns 5. **`counter`**: Count QSOs or callsigns
### Current Awards
- **DXCC**: HF bands only (160m-10m), 100 entities required
- **DXCC SAT**: Satellite QSOs only, 100 entities required
- **WAS**: Worked All States award
- **VUCC SAT**: VUCC Satellite award
- **SAT-RS44**: Special satellite award
- **73 on 73**: Special stations award
- **DLD**: Deutschland Diplom, 100 unique DOKs required
### Key Files ### Key Files
**Backend Award Service**: `src/backend/services/awards.service.js` **Backend Award Service**: `src/backend/services/awards.service.js`
@@ -201,11 +212,13 @@ The award system is JSON-driven and located in `award-definitions/` directory. E
- `calculatePointsAwardProgress(userId, award, options)`: Point-based calculation - `calculatePointsAwardProgress(userId, award, options)`: Point-based calculation
- `getAwardEntityBreakdown(userId, awardId)`: Detailed entity breakdown - `getAwardEntityBreakdown(userId, awardId)`: Detailed entity breakdown
- `getAwardProgressDetails(userId, awardId)`: Progress with details - `getAwardProgressDetails(userId, awardId)`: Progress with details
- Implements `allowed_bands` and `satellite_only` filtering
**Database Schema**: `src/backend/db/schema/index.js` **Database Schema**: `src/backend/db/schema/index.js`
- QSO fields include: `darcDok`, `dclQslRstatus`, `dclQslRdate` - QSO fields include: `darcDok`, `dclQslRstatus`, `dclQslRdate`, `satName`
- DOK fields support DLD award tracking - DOK fields support DLD award tracking
- DCL confirmation fields separate from LoTW - DCL confirmation fields separate from LoTW
- `satName` field for satellite QSO tracking
**Award Definitions**: `award-definitions/*.json` **Award Definitions**: `award-definitions/*.json`
- Add new awards by creating JSON definition files - Add new awards by creating JSON definition files
@@ -216,7 +229,6 @@ The award system is JSON-driven and located in `award-definitions/` directory. E
- Handles case-insensitive `<EOR>` delimiters (supports `<EOR>`, `<eor>`, `<Eor>`) - Handles case-insensitive `<EOR>` delimiters (supports `<EOR>`, `<eor>`, `<Eor>`)
- Uses `matchAll()` for reliable field parsing - Uses `matchAll()` for reliable field parsing
- Skips header records automatically - Skips header records automatically
- `parseDCLResponse(response)`: Parse DCL's JSON response format `{ "adif": "..." }`
- `normalizeBand(band)`: Standardize band names (80m, 40m, etc.) - `normalizeBand(band)`: Standardize band names (80m, 40m, etc.)
- `normalizeMode(mode)`: Standardize mode names (CW, FT8, SSB, etc.) - `normalizeMode(mode)`: Standardize mode names (CW, FT8, SSB, etc.)
- Used by both LoTW and DCL services for consistency - Used by both LoTW and DCL services for consistency
@@ -237,6 +249,7 @@ The award system is JSON-driven and located in `award-definitions/` directory. E
- `POST /api/dcl/sync`: Queue DCL sync job - `POST /api/dcl/sync`: Queue DCL sync job
- `GET /api/jobs/:jobId`: Get job status - `GET /api/jobs/:jobId`: Get job status
- `GET /api/jobs/active`: Get active job for current user - `GET /api/jobs/active`: Get active job for current user
- `DELETE /api/qsos/all`: Delete all QSOs for authenticated user
- `GET /*`: Serves static files from `src/frontend/build/` with SPA fallback - `GET /*`: Serves static files from `src/frontend/build/` with SPA fallback
**SPA Routing**: The backend serves the SvelteKit frontend build from `src/frontend/build/`. **SPA Routing**: The backend serves the SvelteKit frontend build from `src/frontend/build/`.
@@ -262,9 +275,9 @@ The award system is JSON-driven and located in `award-definitions/` directory. E
- Fully implemented and functional - Fully implemented and functional
- **Note**: DCL API is a custom prototype by DARC; contact DARC for API specification details - **Note**: DCL API is a custom prototype by DARC; contact DARC for API specification details
### DLD Award Implementation (COMPLETED) ### DLD Award Implementation
The DLD (Deutschland Diplom) award was recently implemented: The DLD (Deutschland Diplom) award:
**Definition**: `award-definitions/dld.json` **Definition**: `award-definitions/dld.json`
```json ```json
@@ -284,7 +297,7 @@ The DLD (Deutschland Diplom) award was recently implemented:
``` ```
**Implementation Details**: **Implementation Details**:
- Function: `calculateDOKAwardProgress()` in `src/backend/services/awards.service.js` (lines 173-268) - Function: `calculateDOKAwardProgress()` in `src/backend/services/awards.service.js`
- Counts unique (DOK, band, mode) combinations - Counts unique (DOK, band, mode) combinations
- Only DCL-confirmed QSOs count (`dclQslRstatus === 'Y'`) - Only DCL-confirmed QSOs count (`dclQslRstatus === 'Y'`)
- Each unique DOK on each unique band/mode counts separately - Each unique DOK on each unique band/mode counts separately
@@ -297,8 +310,6 @@ The DLD (Deutschland Diplom) award was recently implemented:
- `dclQslRstatus`: DCL confirmation status ('Y' = confirmed) - `dclQslRstatus`: DCL confirmation status ('Y' = confirmed)
- `dclQslRdate`: DCL confirmation date - `dclQslRdate`: DCL confirmation date
**Documentation**: See `docs/DOCUMENTATION.md` for complete documentation including DLD award example.
**Frontend**: `src/frontend/src/routes/qsos/+page.svelte` **Frontend**: `src/frontend/src/routes/qsos/+page.svelte`
- Separate sync buttons for LoTW (blue) and DCL (orange) - Separate sync buttons for LoTW (blue) and DCL (orange)
- Independent progress tracking for each sync type - Independent progress tracking for each sync type
@@ -322,79 +333,42 @@ To add a new award:
3. If new rule type needed, add calculation function 3. If new rule type needed, add calculation function
4. Add type handling in `calculateAwardProgress()` switch statement 4. Add type handling in `calculateAwardProgress()` switch statement
5. Add type handling in `getAwardEntityBreakdown()` if needed 5. Add type handling in `getAwardEntityBreakdown()` if needed
6. Update documentation in `docs/DOCUMENTATION.md` 6. Update documentation
7. Test with sample QSO data 7. Test with sample QSO data
### Creating DLD Award Variants ### Award Rule Options
The DOK award type supports filters to create award variants. Examples: **allowed_bands**: Restrict which bands count toward an award
**DLD on 80m** (`dld-80m.json`):
```json
{
"id": "dld-80m",
"name": "DLD 80m",
"description": "Confirm 100 unique DOKs on 80m",
"caption": "Contact 100 different DOKs on the 80m band.",
"category": "darc",
"rules": {
"type": "dok",
"target": 100,
"confirmationType": "dcl",
"displayField": "darcDok",
"filters": {
"operator": "AND",
"filters": [
{ "field": "band", "operator": "eq", "value": "80m" }
]
}
}
}
```
**DLD in CW mode** (`dld-cw.json`):
```json ```json
{ {
"rules": { "rules": {
"type": "dok", "type": "entity",
"target": 100, "allowed_bands": ["160m", "80m", "60m", "40m", "30m", "20m", "17m", "15m", "12m", "10m"]
"confirmationType": "dcl",
"filters": {
"operator": "AND",
"filters": [
{ "field": "mode", "operator": "eq", "value": "CW" }
]
}
} }
} }
``` ```
- If absent or empty, all bands are allowed (default behavior)
- Used for DXCC to restrict to HF bands only
**DLD on 80m using CW** (combined filters, `dld-80m-cw.json`): **satellite_only**: Only count satellite QSOs
```json ```json
{ {
"rules": { "rules": {
"type": "dok", "type": "entity",
"target": 100, "satellite_only": true
"confirmationType": "dcl",
"filters": {
"operator": "AND",
"filters": [
{ "field": "band", "operator": "eq", "value": "80m" },
{ "field": "mode", "operator": "eq", "value": "CW" }
]
}
} }
} }
``` ```
- If `true`, only QSOs with `satName` field set are counted
- Used for DXCC SAT award
**Available filter operators**: **filters**: Additional filtering options
- `eq`: equals - `eq`: equals
- `ne`: not equals - `ne`: not equals
- `in`: in array - `in`: in array
- `nin`: not in array - `nin`: not in array
- `contains`: contains substring - `contains`: contains substring
- Can filter any QSO field (band, mode, callsign, grid, state, etc.)
**Available filter fields**: Any QSO field (band, mode, callsign, grid, state, satName, etc.)
### Confirmation Systems ### Confirmation Systems
@@ -414,13 +388,8 @@ The DOK award type supports filters to create award variants. Examples:
- Required for DLD award - Required for DLD award
- German amateur radio specific - German amateur radio specific
- Request format: POST JSON `{ key, limit, qsl_since, qso_since, cnf_only }` - Request format: POST JSON `{ key, limit, qsl_since, qso_since, cnf_only }`
- `cnf_only: null` - Fetch all QSOs (confirmed + unconfirmed)
- `cnf_only: true` - Fetch only confirmed QSOs
- `qso_since` - QSOs since this date (YYYYMMDD)
- `qsl_since` - QSL confirmations since this date (YYYYMMDD)
- Response format: JSON with ADIF string in `adif` field - Response format: JSON with ADIF string in `adif` field
- Syncs ALL QSOs (both confirmed and unconfirmed) - Syncs ALL QSOs (both confirmed and unconfirmed)
- Unconfirmed QSOs stored but don't count toward awards
- Updates QSOs only if confirmation data has changed - Updates QSOs only if confirmation data has changed
### ADIF Format ### ADIF Format
@@ -439,138 +408,13 @@ Both LoTW and DCL return data in ADIF (Amateur Data Interchange Format):
- `MY_DARC_DOK`: User's own DOK - `MY_DARC_DOK`: User's own DOK
- `STATION_CALLSIGN`: User's callsign - `STATION_CALLSIGN`: User's callsign
### Recent Commits ### QSO Management
- `aeeb75c`: feat: add QSO count display to filter section **Delete All QSOs**: `DELETE /api/qsos/all`
- Shows count of QSOs matching current filters next to "Filters" heading - Deletes all QSOs for authenticated user
- Displays "Showing X filtered QSOs" when filters are active - Also deletes related `qso_changes` records to satisfy foreign key constraints
- Displays "Showing X total QSOs" when no filters applied - Invalidates stats and user caches after deletion
- Dynamically updates when filters change - Returns count of deleted QSOs
- `bee02d1`: fix: count QSOs confirmed by either LoTW or DCL in stats
- QSO stats were only counting LoTW-confirmed QSOs (`lotwQslRstatus === 'Y'`)
- QSOs confirmed only by DCL were excluded from "confirmed" count
- Fixed by changing filter to: `q.lotwQslRstatus === 'Y' || q.dclQslRstatus === 'Y'`
- Now correctly shows all QSOs confirmed by at least one system
- `233888c`: fix: make ADIF parser case-insensitive for EOR delimiter
- **Critical bug**: LoTW uses lowercase `<eor>` tags, parser was splitting on uppercase `<EOR>`
- Caused 242K+ QSOs to be parsed as 1 giant record with fields overwriting each other
- Changed to case-insensitive regex: `new RegExp('<eor>', 'gi')`
- Replaced `regex.exec()` while loop with `matchAll()` for-of iteration
- Now correctly imports all QSOs from large LoTW reports
- `645f786`: fix: add missing timeOn field to LoTW duplicate detection
- LoTW sync was missing `timeOn` in duplicate detection query
- Multiple QSOs with same callsign/date/band/mode but different times were treated as duplicates
- Now matches DCL sync logic: `userId, callsign, qsoDate, timeOn, band, mode`
- `7f77c3a`: feat: add filter support for DOK awards
- DOK award type now supports filtering by band, mode, and other QSO fields
- Allows creating award variants like DLD 80m, DLD CW, DLD 80m CW
- Uses existing filter system with eq, ne, in, nin, contains operators
- Example awards created: dld-80m, dld-40m, dld-cw, dld-80m-cw
- `9e73704`: docs: update CLAUDE.md with DLD award variants documentation
- `7201446`: fix: return proper HTML for SPA routes instead of Bun error page
- When accessing client-side routes (like /qsos) via curl or non-JS clients,
the server attempted to open them as static files, causing Bun to throw
an unhandled ENOENT error that showed an ugly error page
- Now checks if a path has a file extension before attempting to serve it
- Paths without extensions are immediately served index.html for SPA routing
- Also improves the 503 error page with user-friendly HTML when frontend build is missing
- `223461f`: fix: enable debug logging and improve DCL sync observability
- `27d2ef1`: fix: preserve DOK data when DCL doesn't send values
- DCL sync only updates DOK/grid fields when DCL provides non-empty values
- Prevents accidentally clearing DOK data from manual entry or other sources
- Preserves existing DOK when DCL syncs QSO without DOK information
- `e09ab94`: feat: skip QSOs with unchanged confirmation data
- LoTW/DCL sync only updates QSOs if confirmation data has changed
- Tracks added, updated, and skipped QSO counts
- LoTW: Checks if lotwQslRstatus or lotwQslRdate changed
- DCL: Checks if dclQslRstatus, dclQslRdate, darcDok, myDarcDok, or grid changed
- `3592dbb`: feat: add import log showing synced QSOs
- Backend returns addedQSOs and updatedQSOs arrays in sync result
- Frontend displays import log with callsign, date, band, mode for each QSO
- Separate sections for "New QSOs" and "Updated QSOs"
- Sync summary shows total, added, updated, skipped counts
- `8a1a580`: feat: implement DCL ADIF parser and service integration
- Add shared ADIF parser utility (src/backend/utils/adif-parser.js)
- Implement DCL service with API integration
- Refactor LoTW service to use shared parser
- Tested with example DCL payload (6 QSOs parsed successfully)
- `c982dcd`: feat: implement DLD (Deutschland Diplom) award
- `322ccaf`: docs: add DLD (Deutschland Diplom) award documentation
### Sync Behavior
**Import Log**: After each sync, displays a table showing:
- New QSOs: Callsign, Date, Band, Mode
- Updated QSOs: Callsign, Date, Band, Mode (only if data changed)
- Skipped QSOs: Counted but not shown (data unchanged)
**Duplicate Handling**:
- QSOs matched by: userId, callsign, qsoDate, timeOn, band, mode
- If confirmation data unchanged: Skipped (not updated)
- If confirmation data changed: Updated with new values
- Prevents unnecessary database writes and shows accurate import counts
**DOK Update Behavior**:
- If QSO imported via LoTW (no DOK) and later DCL confirms with DOK: DOK is added ✓
- If QSO already has DOK and DCL sends different DOK: DOK is updated ✓
- If QSO has DOK and DCL syncs without DOK (empty): Existing DOK is preserved ✓
- LoTW never sends DOK data; only DCL provides DOK fields
**Important**: DCL sync only updates DOK/grid fields when DCL provides non-empty values. This prevents accidentally clearing DOK data that was manually entered or imported from other sources.
### DCL Sync Strategy
**Current Behavior**: DCL syncs ALL QSOs (confirmed + unconfirmed)
The application syncs both confirmed and unconfirmed QSOs from DCL:
- **Confirmed QSOs**: `dclQslRstatus = 'Y'` - Count toward awards
- **Unconfirmed QSOs**: `dclQslRstatus = 'N'` - Stored but don't count toward awards
**Purpose of syncing unconfirmed QSOs**:
- Users can see who they've worked (via "Not Confirmed" filter)
- Track QSOs awaiting confirmation
- QSOs can get confirmed later and will be updated on next sync
**Award Calculation**: Always uses confirmed QSOs only (e.g., `dclQslRstatus === 'Y'` for DLD award)
### DCL Incremental Sync Strategy
**Challenge**: Need to fetch both new QSOs AND confirmation updates to old QSOs
**Example Scenario**:
1. Full sync on 2026-01-20 → Last QSO date: 2026-01-20
2. User works 3 new QSOs on 2026-01-25 (unconfirmed)
3. Old QSO from 2026-01-10 gets confirmed on 2026-01-26
4. Next sync needs both: new QSOs (2026-01-25) AND confirmation update (2026-01-10)
**Solution**: Use both `qso_since` and `qsl_since` parameters with OR logic
```javascript
// Proposed sync logic (requires OR logic from DCL API)
const lastQSODate = await getLastDCLQSODate(userId); // Track QSO dates
const lastQSLDate = await getLastDCLQSLDate(userId); // Track QSL dates
const requestBody = {
key: dclApiKey,
limit: 50000,
qso_since: lastQSODate, // Get new QSOs since last contact
qsl_since: lastQSLDate, // Get QSL confirmations since last sync
cnf_only: null, // Fetch all QSOs
};
```
**Required API Behavior (OR Logic)**:
- Return QSOs where `(qso_date >= qso_since) OR (qsl_date >= qsl_since)`
- This ensures we get both new QSOs and confirmation updates
**Current DCL API Status**:
- Unknown if current API uses AND or OR logic for combined filters
- **Action Needed**: Request OR logic implementation from DARC
- Test current behavior to confirm API response pattern
**Why OR Logic is Needed**:
- With AND logic: Old QSOs getting confirmed are missed (qso_date too old)
- With OR logic: All updates captured efficiently in one API call
### QSO Page Filters ### QSO Page Filters
@@ -578,34 +422,45 @@ The QSO page (`src/frontend/src/routes/qsos/+page.svelte`) includes advanced fil
**Available Filters**: **Available Filters**:
- **Search Box**: Full-text search across callsign, entity (DXCC country), and grid square fields - **Search Box**: Full-text search across callsign, entity (DXCC country), and grid square fields
- Press Enter to apply search
- Case-insensitive partial matching
- **Band Filter**: Dropdown to filter by amateur band (160m, 80m, 60m, 40m, 30m, 20m, 17m, 15m, 12m, 10m, 6m, 2m, 70cm) - **Band Filter**: Dropdown to filter by amateur band (160m, 80m, 60m, 40m, 30m, 20m, 17m, 15m, 12m, 10m, 6m, 2m, 70cm)
- **Mode Filter**: Dropdown to filter by mode (CW, SSB, AM, FM, RTTY, PSK31, FT8, FT4, JT65, JT9) - **Mode Filter**: Dropdown to filter by mode (CW, SSB, AM, FM, RTTY, PSK31, FT8, FT4, JT65, JT9)
- **Confirmation Type Filter**: Filter by confirmation status - **Confirmation Type Filter**: Filter by confirmation status
- "All QSOs": Shows all QSOs (no filter) - "All QSOs", "LoTW Only", "DCL Only", "Both Confirmed", "Not Confirmed"
- "LoTW Only": Shows QSOs confirmed by LoTW but NOT DCL - **Clear Button**: Resets all filters
- "DCL Only": Shows QSOs confirmed by DCL but NOT LoTW
- "Both Confirmed": Shows QSOs confirmed by BOTH LoTW AND DCL
- "Not Confirmed": Shows QSOs confirmed by NEITHER LoTW nor DCL
- **Clear Button**: Resets all filters and reloads all QSOs
**Backend Implementation** (`src/backend/services/lotw.service.js`): **Backend Implementation** (`src/backend/services/lotw.service.js`):
- `getUserQSOs(userId, filters, options)`: Main filtering function - `getUserQSOs(userId, filters, options)`: Main filtering function
- Supports pagination with `page` and `limit` options - Supports pagination with `page` and `limit` options
- Filter logic uses Drizzle ORM query builders for safe SQL generation - Filter logic uses Drizzle ORM query builders for safe SQL generation
- Debug logging when `LOG_LEVEL=debug` shows applied filters
**Frontend API** (`src/frontend/src/lib/api.js`): **Frontend API** (`src/frontend/src/lib/api.js`):
- `qsosAPI.getAll(filters)`: Fetch QSOs with optional filters - `qsosAPI.getAll(filters)`: Fetch QSOs with optional filters
- Filters passed as query parameters: `?band=20m&mode=CW&confirmationType=lotw&search=DL` - Filters passed as query parameters: `?band=20m&mode=CW&confirmationType=lotw&search=DL`
**QSO Count Display**: ### Award Detail View
- Shows count of QSOs matching current filters next to "Filters" heading
- **With filters active**: "Showing **X** filtered QSOs" **Overview**: The award detail page (`src/frontend/src/routes/awards/[id]/+page.svelte`) displays award progress in a pivot table format.
- **No filters**: "Showing **X** total QSOs"
- Dynamically updates when filters are applied or cleared **Key Features**:
- Uses `pagination.totalCount` from backend API response - **Summary Cards**: Show total, confirmed, worked, needed counts for unique entities
- **Mode Filter**: Filter by specific mode or view "Mixed Mode" (aggregates all modes by band)
- **Table Columns**: Show bands (or band/mode combinations) as columns
- **QSO Counts**: Each cell shows count of confirmed QSOs for that (entity, band, mode) slot
- **Drill-Down**: Click a count to open modal showing all QSOs for that slot
- **QSO Detail**: Click any QSO to view full QSO details
- **Satellite Grouping**: Satellite QSOs grouped under "SAT" column instead of frequency band
**Column Sorting**: Bands sorted by wavelength (longest to shortest):
160m, 80m, 60m, 40m, 30m, 20m, 17m, 15m, 12m, 10m, 6m, 2m, 70cm, SAT
**Column Sums**: Show unique entity count per column (not QSO counts)
**Backend Changes** (`src/backend/services/awards.service.js`):
- `calculateDOKAwardProgress()`: Groups by (DOK, band, mode) slots, collects QSOs in `qsos` array
- `calculatePointsAwardProgress()`: Handles all count modes with `qsos` array
- `getAwardEntityBreakdown()`: Groups by (entity, band, mode) slots
- Includes `satName` in QSO data for satellite grouping
- Implements `allowed_bands` and `satellite_only` filtering
### DXCC Entity Priority Logic ### DXCC Entity Priority Logic
@@ -613,58 +468,18 @@ When syncing QSOs from multiple confirmation sources, the system follows a prior
**Priority Order**: LoTW > DCL **Priority Order**: LoTW > DCL
**Implementation** (`src/backend/services/dcl.service.js`):
```javascript
// DXCC priority: LoTW > DCL
// Only update entity fields from DCL if:
// 1. QSO is NOT LoTW confirmed, AND
// 2. DCL actually sent entity data, AND
// 3. Current entity is missing
const hasLoTWConfirmation = existingQSO.lotwQslRstatus === 'Y';
const hasDCLData = dbQSO.entity || dbQSO.entityId;
const missingEntity = !existingQSO.entity || existingQSO.entity === '';
if (!hasLoTWConfirmation && hasDCLData && missingEntity) {
// Fill in entity data from DCL (only if DCL provides it)
updateData.entity = dbQSO.entity;
updateData.entityId = dbQSO.entityId;
// ... other entity fields
}
```
**Rules**: **Rules**:
1. **LoTW-confirmed QSOs**: Always use LoTW's DXCC data (most reliable) 1. **LoTW-confirmed QSOs**: Always use LoTW's DXCC data (most reliable)
2. **DCL-only QSOs**: Use DCL's DXCC data IF available in ADIF payload 2. **DCL-only QSOs**: Use DCL's DXCC data IF available in ADIF payload
3. **Empty entity fields**: If DCL doesn't send DXCC data, entity remains empty 3. **Empty entity fields**: If DCL doesn't send DXCC data, entity remains empty
4. **Never overwrite**: Once LoTW confirms with entity data, DCL sync won't change it 4. **Never overwrite**: Once LoTW confirms with entity data, DCL sync won't change it
**Important Note**: DCL API currently doesn't send DXCC/entity fields in their ADIF export. This is a limitation of the DCL API, not the application. If DCL adds these fields in the future, the system will automatically use them for DCL-only QSOs. **Important Note**: DCL API currently doesn't send DXCC/entity fields in their ADIF export.
### Recent Development Work (January 2025) ### Critical LoTW Sync Behavior
**QSO Page Enhancements**:
- Added confirmation type filter with exclusive logic (LoTW Only, DCL Only, Both Confirmed, Not Confirmed)
- Added search box for filtering by callsign, entity, or grid square
- Renamed "All Confirmation" to "All QSOs" for clarity
- Fixed filter logic to properly handle exclusive confirmation types
**Bug Fixes**:
- Fixed confirmation filter showing wrong QSOs (e.g., "LoTW Only" was also showing DCL QSOs)
- Implemented proper SQL conditions for exclusive filters using separate condition pushes
- Added debug logging to track filter application
**DXCC Entity Handling**:
- Clarified that DCL API doesn't send DXCC fields (current limitation)
- Implemented priority logic: LoTW entity data takes precedence over DCL
- System ready to auto-use DCL DXCC data if they add it in future API updates
### Critical LoTW Sync Behavior (LEARNED THE HARD WAY)
**⚠️ IMPORTANT: LoTW sync MUST only import confirmed QSOs** **⚠️ IMPORTANT: LoTW sync MUST only import confirmed QSOs**
After attempting to implement "QSO Delta" sync (all QSOs, confirmed + unconfirmed), we discovered:
**The Problem:**
LoTW ADIF export with `qso_qsl=no` (all QSOs mode) only includes: LoTW ADIF export with `qso_qsl=no` (all QSOs mode) only includes:
- `CALL` (callsign) - `CALL` (callsign)
- `QSL_RCVD` (confirmation status: Y/N) - `QSL_RCVD` (confirmation status: Y/N)
@@ -672,9 +487,7 @@ LoTW ADIF export with `qso_qsl=no` (all QSOs mode) only includes:
**Missing Fields for Unconfirmed QSOs:** **Missing Fields for Unconfirmed QSOs:**
- `DXCC` (entity ID) ← **CRITICAL for awards!** - `DXCC` (entity ID) ← **CRITICAL for awards!**
- `COUNTRY` (entity name) - `COUNTRY` (entity name)
- `CONTINENT` - `CONTINENT`, `CQ_ZONE`, `ITU_ZONE`
- `CQ_ZONE`
- `ITU_ZONE`
**Result:** Unconfirmed QSOs have `entityId: null` and `entity: ""`, breaking award calculations. **Result:** Unconfirmed QSOs have `entityId: null` and `entity: ""`, breaking award calculations.
@@ -690,117 +503,22 @@ const params = new URLSearchParams({
}); });
``` ```
**Why This Matters:** ### Recent Development Work (January 2026)
- Awards require `entityId` to count entities
- Without `entityId`, QSOs can't be counted toward DXCC, WAS, etc.
- Users can still see "worked" stations in QSO list, but awards only count confirmed
- DCL sync can import all QSOs because it provides entity data via callsign lookup
**Attempted Solution (REVERTED):** **Award System Enhancements**:
- Tried implementing callsign prefix lookup to populate missing `entityId` - Added `allowed_bands` filter to restrict which bands count toward awards
- Created `src/backend/utils/callsign-lookup.js` with basic prefix mappings - Added `satellite_only` flag for satellite-only awards
- Complexity: 1000+ DXCC entities, many special event callsigns, portable designators - DXCC restricted to HF bands (160m-10m) only
- Decision: Too complex, reverted (commit 310b154) - Added DXCC SAT award for satellite-only QSOs
- Removed redundant award variants (DXCC CW, DLD variants)
**Takeaway:** LoTW confirmed QSOs have reliable DXCC data. Don't try to workaround this fundamental limitation. **Award Detail View Improvements**:
- Summary shows unique entity progress instead of QSO counts
- Column sums count unique entities per column
- Satellite QSOs grouped under "SAT" column
- Bands sorted by wavelength instead of alphabetically
- Mode removed from table headers (visible in filter dropdown)
### QSO Confirmation Filters **QSO Management**:
- Fixed DELETE /api/qsos/all to handle foreign key constraints
Added "Confirmed by at least 1 service" filter to QSO view (commit 688b0fc): - Added cache invalidation after QSO deletion
**Filter Options:**
- "All QSOs" - No filter
- "Confirmed by at least 1 service" (NEW) - LoTW OR DCL confirmed
- "LoTW Only" - Confirmed by LoTW but NOT DCL
- "DCL Only" - Confirmed by DCL but NOT LoTW
- "Both Confirmed" - Confirmed by BOTH LoTW AND DCL
- "Not Confirmed" - Confirmed by NEITHER
**SQL Logic:**
```sql
-- "Confirmed by at least 1 service"
WHERE lotwQslRstatus = 'Y' OR dclQslRstatus = 'Y'
-- "LoTW Only"
WHERE lotwQslRstatus = 'Y' AND (dclQslRstatus IS NULL OR dclQslRstatus != 'Y')
-- "DCL Only"
WHERE dclQslRstatus = 'Y' AND (lotwQslRstatus IS NULL OR lotwQslRstatus != 'Y')
-- "Both Confirmed"
WHERE lotwQslRstatus = 'Y' AND dclQslRstatus = 'Y'
-- "Not Confirmed"
WHERE (lotwQslRstatus IS NULL OR lotwQslRstatus != 'Y')
AND (dclQslRstatus IS NULL OR dclQslRstatus != 'Y')
```
### Recent Development Work (January 2025)
**Sync Type Support (ATTEMPTED & REVERTED):**
- Commit 5b78935: Added LoTW sync type support (QSL/QSO delta/full)
- Commit 310b154: Reverted - LoTW doesn't provide entity data for unconfirmed QSOs
- **Lesson:** Keep it simple - only sync confirmed QSOs from LoTW
**Dashboard Enhancements:**
- Added sync job history display with real-time polling (every 2 seconds)
- Shows job progress, status, and import logs
- Cancel button for stale/failed jobs with rollback capability
- Tracks all QSO changes in `qso_changes` table for rollback
**Rollback System:**
- `cancelJob(jobId, userId)` - Cancels and rolls back sync jobs
- Tracks added QSOs (deletes them on rollback)
- Tracks updated QSOs (restores previous state)
- Only allows canceling failed jobs or stale running jobs (>1 hour)
- Server-side validation prevents unauthorized cancellations
### Award Detail View (January 2025)
**Overview**: The award detail page (`src/frontend/src/routes/awards/[id]/+page.svelte`) displays award progress in a pivot table format with entities as rows and band/mode combinations as columns.
**Key Features**:
- **QSO Count per Slot**: Each table cell shows the count of confirmed QSOs for that (entity, band, mode) combination
- **Drill-Down**: Click a count to open a modal showing all QSOs for that slot
- **QSO Detail**: Click any QSO in the list to view full QSO details
- **Mode Filter**: Filter by specific mode or view "Mixed Mode" (aggregates all modes by band)
**Backend Changes** (`src/backend/services/awards.service.js`):
- `calculateDOKAwardProgress()`: Groups by (DOK, band, mode) slots, collects all confirmed QSOs in `qsos` array
- `calculatePointsAwardProgress()`: Updated for all count modes (perBandMode, perStation, perQso) with `qsos` array
- `getAwardEntityBreakdown()`: Groups by (entity, band, mode) slots for entity awards
**Response Structure**:
```javascript
{
entity: "F03",
band: "80m",
mode: "CW",
worked: true,
confirmed: true,
qsos: [
{ qsoId: 123, callsign: "DK0MU", mode: "CW", qsoDate: "20250115", timeOn: "123456", confirmed: true },
{ qsoId: 456, callsign: "DL1ABC", mode: "CW", qsoDate: "20250120", timeOn: "234500", confirmed: true }
]
}
```
**Mode Filter**:
- **Mixed Mode (default)**: Shows bands as columns, aggregates all modes
- Example: Columns are "80m", "40m", "20m"
- Clicking a count shows all QSOs for that band across all modes
- **Specific Mode**: Shows (band, mode) combinations as columns
- Example: Columns are "80m CW", "80m SSB", "40m CW"
- Filters to only show QSOs with that mode
**Frontend Components**:
- **Mode Filter Dropdown**: Located between summary cards and table
- Dynamically populated with available modes from the data
- Clear button appears when specific mode is selected
- **Count Badges**: Blue clickable links showing QSO count (removed bubbles, kept links)
- **QSO List Modal**: Shows all QSOs for selected slot with columns: Callsign, Date, Time, Mode
- **QSO Detail Modal**: Full QSO information (existing feature)
**Files Modified**:
- `src/backend/services/awards.service.js` - Backend grouping and QSO collection
- `src/frontend/src/routes/awards/[id]/+page.svelte` - Frontend display and interaction

View File

@@ -1,7 +1,7 @@
import { db, logger } from '../config.js'; import { db, logger } from '../config.js';
import { qsos } from '../db/schema/index.js'; import { qsos } from '../db/schema/index.js';
import { eq, and, or, desc, sql } from 'drizzle-orm'; import { eq, and, or, desc, sql } from 'drizzle-orm';
import { readFileSync } from 'fs'; import { readFileSync, readdirSync } from 'fs';
import { join } from 'path'; import { join } from 'path';
import { getCachedAwardProgress, setCachedAwardProgress } from './cache.service.js'; import { getCachedAwardProgress, setCachedAwardProgress } from './cache.service.js';
@@ -13,23 +13,25 @@ import { getCachedAwardProgress, setCachedAwardProgress } from './cache.service.
// Load award definitions from files // Load award definitions from files
const AWARD_DEFINITIONS_DIR = join(process.cwd(), 'award-definitions'); const AWARD_DEFINITIONS_DIR = join(process.cwd(), 'award-definitions');
// In-memory cache for award definitions (static, never changes at runtime)
let cachedAwardDefinitions = null;
/** /**
* Load all award definitions * Load all award definitions (cached in memory)
*/ */
function loadAwardDefinitions() { function loadAwardDefinitions() {
// Return cached definitions if available
if (cachedAwardDefinitions) {
return cachedAwardDefinitions;
}
const definitions = []; const definitions = [];
try { try {
const files = [ // Auto-discover all JSON files in the award-definitions directory
'dxcc.json', const files = readdirSync(AWARD_DEFINITIONS_DIR)
'dxcc-sat.json', .filter(f => f.endsWith('.json'))
'was.json', .sort();
'vucc-sat.json',
'sat-rs44.json',
'special-stations.json',
'dld.json',
'73-on-73.json',
];
for (const file of files) { for (const file of files) {
try { try {
@@ -45,6 +47,9 @@ function loadAwardDefinitions() {
logger.error('Error loading award definitions', { error: error.message }); logger.error('Error loading award definitions', { error: error.message });
} }
// Cache the definitions for future calls
cachedAwardDefinitions = definitions;
return definitions; return definitions;
} }

View File

@@ -86,32 +86,6 @@ export function clearAllCache() {
return size; return size;
} }
/**
* Get cache statistics (for monitoring/debugging)
* @returns {object} Cache stats
*/
export function getCacheStats() {
const now = Date.now();
let expired = 0;
let valid = 0;
for (const [, value] of awardCache) {
const age = now - value.timestamp;
if (age > CACHE_TTL) {
expired++;
} else {
valid++;
}
}
return {
total: awardCache.size,
valid,
expired,
ttl: CACHE_TTL
};
}
/** /**
* Clean up expired cache entries (maintenance function) * Clean up expired cache entries (maintenance function)
* Can be called periodically to free memory * Can be called periodically to free memory

View File

@@ -4,6 +4,7 @@ import { max, sql, eq, and, desc } from 'drizzle-orm';
import { updateJobProgress } from './job-queue.service.js'; import { updateJobProgress } from './job-queue.service.js';
import { parseDCLResponse, normalizeBand, normalizeMode } from '../utils/adif-parser.js'; import { parseDCLResponse, normalizeBand, normalizeMode } from '../utils/adif-parser.js';
import { invalidateUserCache, invalidateStatsCache } from './cache.service.js'; import { invalidateUserCache, invalidateStatsCache } from './cache.service.js';
import { yieldToEventLoop, getQSOKey } from '../utils/sync-helpers.js';
/** /**
* DCL (DARC Community Logbook) Service * DCL (DARC Community Logbook) Service
@@ -122,17 +123,6 @@ export async function fetchQSOsFromDCL(dclApiKey, sinceDate = null) {
} }
} }
/**
* Parse DCL API response from JSON
* Can be used for testing with example payloads
*
* @param {Object} jsonResponse - JSON response in DCL format
* @returns {Array} Array of parsed QSO records
*/
export function parseDCLJSONResponse(jsonResponse) {
return parseDCLResponse(jsonResponse);
}
/** /**
* Convert DCL ADIF QSO to database format * Convert DCL ADIF QSO to database format
* @param {Object} adifQSO - Parsed ADIF QSO record * @param {Object} adifQSO - Parsed ADIF QSO record
@@ -169,21 +159,6 @@ function convertQSODatabaseFormat(adifQSO, userId) {
}; };
} }
/**
* Yield to event loop to allow other requests to be processed
* This prevents blocking the server during long-running sync operations
*/
function yieldToEventLoop() {
return new Promise(resolve => setImmediate(resolve));
}
/**
* Get QSO key for duplicate detection
*/
function getQSOKey(qso) {
return `${qso.callsign}|${qso.qsoDate}|${qso.timeOn}|${qso.band}|${qso.mode}`;
}
/** /**
* Sync QSOs from DCL to database (optimized with batch operations) * Sync QSOs from DCL to database (optimized with batch operations)
* Updates existing QSOs with DCL confirmation data * Updates existing QSOs with DCL confirmation data

View File

@@ -4,7 +4,8 @@ import { max, sql, eq, and, or, desc, like } from 'drizzle-orm';
import { updateJobProgress } from './job-queue.service.js'; import { updateJobProgress } from './job-queue.service.js';
import { parseADIF, normalizeBand, normalizeMode } from '../utils/adif-parser.js'; import { parseADIF, normalizeBand, normalizeMode } from '../utils/adif-parser.js';
import { invalidateUserCache, getCachedStats, setCachedStats, invalidateStatsCache } from './cache.service.js'; import { invalidateUserCache, getCachedStats, setCachedStats, invalidateStatsCache } from './cache.service.js';
import { trackQueryPerformance, getPerformanceSummary, resetPerformanceMetrics } from './performance.service.js'; import { trackQueryPerformance } from './performance.service.js';
import { yieldToEventLoop, getQSOKey } from '../utils/sync-helpers.js';
/** /**
* LoTW (Logbook of the World) Service * LoTW (Logbook of the World) Service
@@ -81,6 +82,7 @@ const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
* Fetch QSOs from LoTW with retry support * Fetch QSOs from LoTW with retry support
*/ */
async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) { async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
const startTime = Date.now();
const url = 'https://lotw.arrl.org/lotwuser/lotwreport.adi'; const url = 'https://lotw.arrl.org/lotwuser/lotwreport.adi';
const params = new URLSearchParams({ const params = new URLSearchParams({
@@ -176,7 +178,7 @@ async function fetchQSOsFromLoTW(lotwUsername, lotwPassword, sinceDate = null) {
} }
} }
const totalTime = Math.round((Date.now() - Date.now()) / 1000); const totalTime = Math.round((Date.now() - startTime) / 1000);
return { return {
error: `LoTW sync failed: Report not ready after ${MAX_RETRIES} attempts (${totalTime}s). LoTW may be experiencing high load. Please try again later.` error: `LoTW sync failed: Report not ready after ${MAX_RETRIES} attempts (${totalTime}s). LoTW may be experiencing high load. Please try again later.`
}; };
@@ -210,21 +212,6 @@ function convertQSODatabaseFormat(adifQSO, userId) {
}; };
} }
/**
* Yield to event loop to allow other requests to be processed
* This prevents blocking the server during long-running sync operations
*/
function yieldToEventLoop() {
return new Promise(resolve => setImmediate(resolve));
}
/**
* Get QSO key for duplicate detection
*/
function getQSOKey(qso) {
return `${qso.callsign}|${qso.qsoDate}|${qso.timeOn}|${qso.band}|${qso.mode}`;
}
/** /**
* Sync QSOs from LoTW to database (optimized with batch operations) * Sync QSOs from LoTW to database (optimized with batch operations)
* @param {number} userId - User ID * @param {number} userId - User ID

View File

@@ -0,0 +1,23 @@
/**
* Sync Helper Utilities
*
* Shared utilities for LoTW and DCL sync operations
*/
/**
* Yield to event loop to allow other requests to be processed
* This prevents blocking the server during long-running sync operations
* @returns {Promise<void>}
*/
export function yieldToEventLoop() {
return new Promise(resolve => setImmediate(resolve));
}
/**
* Get QSO key for duplicate detection
* @param {object} qso - QSO object
* @returns {string} Unique key for the QSO
*/
export function getQSOKey(qso) {
return `${qso.callsign}|${qso.qsoDate}|${qso.timeOn}|${qso.band}|${qso.mode}`;
}