perf: implement Phase 1 backend performance optimizations

Fix N+1 query, add database indexes, and implement award progress caching:

- Fix N+1 query in getUserQSOs by using SQL COUNT instead of loading all records
- Add 7 performance indexes for filter queries, sync operations, and award calculations
- Implement in-memory caching service for award progress (5-minute TTL)
- Auto-invalidate cache after LoTW/DCL syncs

Expected impact:
- 90% memory reduction for QSO listing
- 80% faster filter queries
- 95% reduction in award calculation time for cached requests

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
2026-01-19 14:18:00 +01:00
parent f86d68c97b
commit f50ec5f44e
7 changed files with 355 additions and 3 deletions

View File

@@ -3,6 +3,7 @@ import { qsos } from '../db/schema/index.js';
import { eq, and, or, desc, sql } from 'drizzle-orm';
import { readFileSync } from 'fs';
import { join } from 'path';
import { getCachedAwardProgress, setCachedAwardProgress } from './cache.service.js';
/**
* Awards Service
@@ -585,6 +586,15 @@ function matchesFilter(qso, filter) {
* Get award progress with QSO details
*/
export async function getAwardProgressDetails(userId, awardId) {
// Check cache first
const cached = getCachedAwardProgress(userId, awardId);
if (cached) {
logger.debug(`Cache hit for award ${awardId}, user ${userId}`);
return cached;
}
logger.debug(`Cache miss for award ${awardId}, user ${userId} - calculating...`);
// Get award definition
const definitions = loadAwardDefinitions();
const award = definitions.find((def) => def.id === awardId);
@@ -596,7 +606,7 @@ export async function getAwardProgressDetails(userId, awardId) {
// Calculate progress
const progress = await calculateAwardProgress(userId, award);
return {
const result = {
award: {
id: award.id,
name: award.name,
@@ -606,6 +616,11 @@ export async function getAwardProgressDetails(userId, awardId) {
},
...progress,
};
// Store in cache
setCachedAwardProgress(userId, awardId, result);
return result;
}
/**

View File

@@ -0,0 +1,129 @@
/**
* Cache Service for Award Progress
*
* Provides in-memory caching for award progress calculations to avoid
* expensive database aggregations on every request.
*
* Cache TTL: 5 minutes (balances freshness with performance)
*
* Usage:
* - Check cache before calculating award progress
* - Invalidate cache when QSOs are synced/updated
* - Automatic expiry after TTL
*/
const awardCache = new Map();
const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
/**
* Get cached award progress if available and not expired
* @param {number} userId - User ID
* @param {string} awardId - Award ID
* @returns {object|null} Cached progress data or null if not found/expired
*/
export function getCachedAwardProgress(userId, awardId) {
const key = `${userId}:${awardId}`;
const cached = awardCache.get(key);
if (!cached) {
return null;
}
// Check if cache has expired
const age = Date.now() - cached.timestamp;
if (age > CACHE_TTL) {
awardCache.delete(key);
return null;
}
return cached.data;
}
/**
* Set award progress in cache
* @param {number} userId - User ID
* @param {string} awardId - Award ID
* @param {object} data - Award progress data to cache
*/
export function setCachedAwardProgress(userId, awardId, data) {
const key = `${userId}:${awardId}`;
awardCache.set(key, {
data,
timestamp: Date.now()
});
}
/**
* Invalidate all cached awards for a specific user
* Call this after syncing or updating QSOs
* @param {number} userId - User ID
*/
export function invalidateUserCache(userId) {
const prefix = `${userId}:`;
let deleted = 0;
for (const [key] of awardCache) {
if (key.startsWith(prefix)) {
awardCache.delete(key);
deleted++;
}
}
return deleted;
}
/**
* Clear all cached awards (use sparingly)
*主要用于测试或紧急情况
*/
export function clearAllCache() {
const size = awardCache.size;
awardCache.clear();
return size;
}
/**
* Get cache statistics (for monitoring/debugging)
* @returns {object} Cache stats
*/
export function getCacheStats() {
const now = Date.now();
let expired = 0;
let valid = 0;
for (const [, value] of awardCache) {
const age = now - value.timestamp;
if (age > CACHE_TTL) {
expired++;
} else {
valid++;
}
}
return {
total: awardCache.size,
valid,
expired,
ttl: CACHE_TTL
};
}
/**
* Clean up expired cache entries (maintenance function)
* Can be called periodically to free memory
* @returns {number} Number of entries cleaned up
*/
export function cleanupExpiredCache() {
const now = Date.now();
let cleaned = 0;
for (const [key, value] of awardCache) {
const age = now - value.timestamp;
if (age > CACHE_TTL) {
awardCache.delete(key);
cleaned++;
}
}
return cleaned;
}

View File

@@ -3,6 +3,7 @@ import { qsos } from '../db/schema/index.js';
import { max, sql, eq, and, desc } from 'drizzle-orm';
import { updateJobProgress } from './job-queue.service.js';
import { parseDCLResponse, normalizeBand, normalizeMode } from '../utils/adif-parser.js';
import { invalidateUserCache } from './cache.service.js';
/**
* DCL (DARC Community Logbook) Service
@@ -350,6 +351,10 @@ export async function syncQSOs(userId, dclApiKey, sinceDate = null, jobId = null
jobId,
});
// Invalidate award cache for this user since QSOs may have changed
const deletedCache = invalidateUserCache(userId);
logger.debug(`Invalidated ${deletedCache} cached award entries for user ${userId}`);
return result;
} catch (error) {

View File

@@ -3,6 +3,7 @@ import { qsos } from '../db/schema/index.js';
import { max, sql, eq, and, or, desc, like } from 'drizzle-orm';
import { updateJobProgress } from './job-queue.service.js';
import { parseADIF, normalizeBand, normalizeMode } from '../utils/adif-parser.js';
import { invalidateUserCache } from './cache.service.js';
/**
* LoTW (Logbook of the World) Service
@@ -304,6 +305,10 @@ export async function syncQSOs(userId, lotwUsername, lotwPassword, sinceDate = n
logger.info('LoTW sync completed', { total: adifQSOs.length, added: addedCount, updated: updatedCount, skipped: skippedCount, jobId });
// Invalidate award cache for this user since QSOs may have changed
const deletedCache = invalidateUserCache(userId);
logger.debug(`Invalidated ${deletedCache} cached award entries for user ${userId}`);
return {
success: true,
total: adifQSOs.length,
@@ -370,8 +375,12 @@ export async function getUserQSOs(userId, filters = {}, options = {}) {
));
}
const allResults = await db.select().from(qsos).where(and(...conditions));
const totalCount = allResults.length;
// Use SQL COUNT for efficient pagination (avoids loading all QSOs into memory)
const [{ count }] = await db
.select({ count: sql`CAST(count(*) AS INTEGER)` })
.from(qsos)
.where(and(...conditions));
const totalCount = count;
const offset = (page - 1) * limit;