Compare commits

..

1 Commits

Author SHA1 Message Date
vhaudiquet 5d8320fe8a fix/mathc_collector: fix splitting of boots and first backs
pipeline / build-and-push-images (push) Has been cancelled
pipeline / lint-and-format (push) Has been cancelled
2026-04-30 20:15:44 +02:00
14 changed files with 54 additions and 437 deletions
+8 -12
View File
@@ -1,25 +1,21 @@
services: services:
# Development MongoDB with memory optimizations # Development MongoDB with performance optimizations
mongodb: mongodb:
image: mongo:8.3.4 image: mongo:latest
container_name: buildpath-mongodb container_name: buildpath-mongodb
ports: ports:
- "27017:27017" - "27017:27017"
environment: environment:
MONGO_INITDB_ROOT_USERNAME: ${MONGO_USER:-root} MONGO_INITDB_ROOT_USERNAME: ${MONGO_USER:-root}
MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASS:-password} MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASS:-password}
GLIBC_TUNABLES: glibc.pthread.rseq=1
volumes: volumes:
- ./data/db:/data/db - ./data/db:/data/db
# Reduced cache size to leave more RAM for the import script command: mongod --wiredTigerCacheSizeGB 4 --quiet
# WiredTiger cache is now 2GB (was 4GB) to prevent OOM during large imports healthcheck:
command: mongod --wiredTigerCacheSizeGB 2 --quiet test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet
deploy: interval: 5s
resources: timeout: 2s
limits: retries: 30
memory: 4G
reservations:
memory: 2G
mongo-express: mongo-express:
image: mongo-express image: mongo-express
+8 -29
View File
@@ -31,29 +31,19 @@ async function importLargeJsonFile(filePath, collectionName, batchSize = 1000) {
const collection = db.collection(collectionName); const collection = db.collection(collectionName);
try { try {
// Check file size first // Create indexes first for better performance
await collection.createIndex({ "metadata.matchId": 1 }, { unique: true });
await collection.createIndex({ "info.gameDuration": 1 });
await collection.createIndex({ "info.participants.championId": 1 });
await collection.createIndex({ "info.participants.win": 1 });
// Check file size
const fileStats = fs.statSync(filePath); const fileStats = fs.statSync(filePath);
const fileSize = (fileStats.size / (1024 * 1024 * 1024)).toFixed(2); const fileSize = (fileStats.size / (1024 * 1024 * 1024)).toFixed(2);
console.log(` 📊 File size: ${fileSize} GB`); console.log(` 📊 File size: ${fileSize} GB`);
// Defer index creation to after import to reduce memory pressure
// Only create the unique matchId index before import to prevent duplicates
console.log(` 📇 Creating unique matchId index...`);
await collection.createIndex({ "metadata.matchId": 1 }, { unique: true, background: false });
await processLineDelimitedFormat(filePath, collection, batchSize, startTime); await processLineDelimitedFormat(filePath, collection, batchSize, startTime);
// Create additional indexes after import to reduce memory pressure
console.log(`\n 📇 Creating additional indexes (this may take a while)...`);
try {
await collection.createIndex({ "info.gameDuration": 1 }, { background: true });
await collection.createIndex({ "info.participants.championId": 1 }, { background: true });
await collection.createIndex({ "info.participants.win": 1 }, { background: true });
console.log(` ✅ Indexes created successfully`);
} catch (indexError) {
console.log(` ⚠️ Warning: Could not create additional indexes: ${indexError.message}`);
}
const totalTime = ((Date.now() - startTime) / 1000).toFixed(1); const totalTime = ((Date.now() - startTime) / 1000).toFixed(1);
console.log(`🎉 Import complete in ${totalTime} seconds`); console.log(`🎉 Import complete in ${totalTime} seconds`);
console.log(`✅ Processed: ${processed.toLocaleString()} matches`); console.log(`✅ Processed: ${processed.toLocaleString()} matches`);
@@ -76,7 +66,6 @@ async function importLargeJsonFile(filePath, collectionName, batchSize = 1000) {
let batch = []; let batch = [];
let lineCount = 0; let lineCount = 0;
let batchCount = 0;
for await (const line of rl) { for await (const line of rl) {
lineCount++; lineCount++;
@@ -99,16 +88,9 @@ async function importLargeJsonFile(filePath, collectionName, batchSize = 1000) {
batch.push(match); batch.push(match);
if (batch.length >= batchSize) { if (batch.length >= batchSize) {
batchCount++; process.stdout.write(`\r Inserting batch into MongoDB... `);
process.stdout.write(`\r Inserting batch #${batchCount} (${batch.length} matches)... `);
await insertBatch(batch, collection); await insertBatch(batch, collection);
batch = []; batch = [];
// Force garbage collection hint every 10 batches by yielding to the event loop
// This helps reduce memory pressure when processing large files
if (batchCount % 10 === 0) {
await new Promise(resolve => setImmediate(resolve));
}
} }
} catch (error) { } catch (error) {
skipped++; skipped++;
@@ -117,11 +99,8 @@ async function importLargeJsonFile(filePath, collectionName, batchSize = 1000) {
// Insert remaining matches // Insert remaining matches
if (batch.length > 0) { if (batch.length > 0) {
process.stdout.write(`\r Inserting final batch (${batch.length} matches)... `);
await insertBatch(batch, collection); await insertBatch(batch, collection);
} }
console.log(`\n 📊 Total batches inserted: ${batchCount + 1}`);
} }
async function insertBatch(batch, collection) { async function insertBatch(batch, collection) {
-11
View File
@@ -107,15 +107,4 @@ defineProps<{
opacity: 0.8; opacity: 0.8;
white-space: nowrap; white-space: nowrap;
} }
@media only screen and (max-width: 650px) {
.item-row {
align-items: center;
max-width: 100%;
margin: 0 20px;
}
.first-back-content {
justify-content: center;
}
}
</style> </style>
+1
View File
@@ -124,6 +124,7 @@ const itemIconPath = computed(() => CDRAGON_BASE + mapPath(props.item.iconPath))
border-radius: 4px; border-radius: 4px;
border: 1px solid var(--color-on-surface); border: 1px solid var(--color-on-surface);
overflow: hidden; overflow: hidden;
cursor: help;
position: relative; position: relative;
} }
+1 -13
View File
@@ -86,11 +86,7 @@ refreshStyles()
<div class="rune-spacer-bar" /> <div class="rune-spacer-bar" />
<div class="rune-holder" style="align-content: end"> <div class="rune-holder" style="align-content: end">
<div class="rune-slot"> <div class="rune-slot">
<img <img style="margin: auto" :src="CDRAGON_BASE + mapPath(secondaryStyle.iconPath)" />
class="rune-style-img"
style="margin: auto"
:src="CDRAGON_BASE + mapPath(secondaryStyle.iconPath)"
/>
</div> </div>
<div <div
v-for="(slot, slotIndex) in secondaryStyle.slots.slice(1, 4)" v-for="(slot, slotIndex) in secondaryStyle.slots.slice(1, 4)"
@@ -132,10 +128,6 @@ refreshStyles()
margin-right: 20px; margin-right: 20px;
border: 1px var(--color-on-surface) solid; border: 1px var(--color-on-surface) solid;
} }
.rune-style-img {
width: 48px;
height: 48px;
}
@media only screen and (max-width: 650px) { @media only screen and (max-width: 650px) {
.rune-slot { .rune-slot {
@@ -151,9 +143,5 @@ refreshStyles()
margin-left: 10px; margin-left: 10px;
margin-right: 10px; margin-right: 10px;
} }
.rune-icon {
width: 24px !important;
height: 24px !important;
}
} }
</style> </style>
+1
View File
@@ -114,6 +114,7 @@ const perkIconPath = computed(() => CDRAGON_BASE + mapPath(props.perk.iconPath))
border-radius: 50%; border-radius: 50%;
border: 1px solid var(--color-on-surface); border: 1px solid var(--color-on-surface);
overflow: hidden; overflow: hidden;
cursor: help;
position: relative; position: relative;
} }
+6 -30
View File
@@ -16,18 +16,12 @@ import type { LaneData } from 'match_collector'
// Register // Register
ChartJS.register(Title, Tooltip, Legend, BarElement, CategoryScale, LinearScale) ChartJS.register(Title, Tooltip, Legend, BarElement, CategoryScale, LinearScale)
const props = withDefaults( const props = defineProps<{
defineProps<{
data: Array<{ title: string; data: Array<{ lane: LaneData; champion: Champion }> }> data: Array<{ title: string; data: Array<{ lane: LaneData; champion: Champion }> }>
metric?: 'pickrate' | 'winrate' }>()
}>(),
{
metric: 'pickrate'
}
)
const labels: Array<string> = [] const labels: Array<string> = []
const values: Array<number> = [] const pickrates: Array<number> = []
const images: Array<string> = [] const images: Array<string> = []
const backgroundColors: Array<string> = [] const backgroundColors: Array<string> = []
const CHAMPION_CUT_THRESHOLD = 32 const CHAMPION_CUT_THRESHOLD = 32
@@ -40,7 +34,7 @@ for (const tier of props.data) {
if (count > CHAMPION_CUT_THRESHOLD) break if (count > CHAMPION_CUT_THRESHOLD) break
labels.push(champion.name) labels.push(champion.name)
values.push(lane[props.metric] * 100) pickrates.push(lane.pickrate * 100)
images.push(CDRAGON_BASE + mapPath(champion.squarePortraitPath)) images.push(CDRAGON_BASE + mapPath(champion.squarePortraitPath))
backgroundColors.push(TIER_COLORS[colorIndex]) backgroundColors.push(TIER_COLORS[colorIndex])
@@ -53,10 +47,10 @@ const chartData = ref({
labels: labels, labels: labels,
datasets: [ datasets: [
{ {
label: props.metric === 'pickrate' ? 'Pickrate' : 'Winrate', label: 'Pickrate',
backgroundColor: backgroundColors, backgroundColor: backgroundColors,
barPercentage: 1.0, barPercentage: 1.0,
data: values data: pickrates
} }
] ]
}) })
@@ -68,24 +62,6 @@ const chartOptions = ref({
ticks: { ticks: {
callback: () => '' callback: () => ''
} }
},
y: {
title: {
display: true,
text: props.metric === 'winrate' ? 'Winrate (%)' : 'Pickrate (%)'
},
...(props.metric === 'winrate'
? {
min: 40,
max: 60,
grid: {
color: (context: { tick: { value: number } }) =>
context.tick.value === 50 ? '#666' : 'rgba(0, 0, 0, 0.1)',
lineWidth: (context: { tick: { value: number } }) =>
context.tick.value === 50 ? 2 : 1
}
}
: {})
} }
}, },
plugins: { plugins: {
+3 -11
View File
@@ -110,21 +110,14 @@ tiers.push({ title: 'F', data: tierFromScaledPickrate(0, 0.1) })
:tier="tier.data" :tier="tier.data"
/> />
<h2 style="margin-left: 10px; margin-top: 20px; font-size: 2rem; font-weight: 300"> <TierlistChart id="chart" :data="tiers" />
Pickrates
</h2>
<TierlistChart id="chart-pickrate" :data="tiers" metric="pickrate" />
<h2 style="margin-left: 10px; font-size: 2rem; font-weight: 300">Winrates</h2>
<TierlistChart id="chart-winrate" :data="tiers" metric="winrate" />
</div> </div>
</div> </div>
</div> </div>
</template> </template>
<style scoped> <style scoped>
#chart-pickrate, #chart {
#chart-winrate {
margin-left: 100px; margin-left: 100px;
margin-right: 100px; margin-right: 100px;
margin-bottom: 100px; margin-bottom: 100px;
@@ -132,8 +125,7 @@ tiers.push({ title: 'F', data: tierFromScaledPickrate(0, 0.1) })
} }
@media only screen and (max-width: 450px) { @media only screen and (max-width: 450px) {
#chart-pickrate, #chart {
#chart-winrate {
margin-left: 2px; margin-left: 2px;
margin-right: 12px; margin-right: 12px;
margin-bottom: 40px; margin-bottom: 40px;
+2 -17
View File
@@ -1,29 +1,14 @@
import type { MongoClient } from 'mongodb' import type { MongoClient } from 'mongodb'
import { connectToDatabase, fetchLatestPatch, getAvailablePlatforms } from '../utils/mongo' import { connectToDatabase, fetchLatestPatch, getAvailablePlatforms } from '../utils/mongo'
interface StatsDocument {
patch: string
total: number
platforms: Record<string, number>
updatedAt: Date
}
async function fetchGameCount(client: MongoClient, patch: string) { async function fetchGameCount(client: MongoClient, patch: string) {
const database = client.db('matches') const database = client.db('matches')
const statsCollection = database.collection<StatsDocument>('stats')
// Try to get stats from the pre-computed stats collection // Check for platform-specific collections
const stats = await statsCollection.findOne({ patch })
if (stats) {
return { total: stats.total, platforms: stats.platforms }
}
// Fallback: compute stats from collections if stats document doesn't exist
// This handles the migration case where stats weren't pre-computed
const platforms = await getAvailablePlatforms(client, patch) const platforms = await getAvailablePlatforms(client, patch)
if (platforms.length > 0) { if (platforms.length > 0) {
// Sum counts from all platform-specific collections
let totalCount = 0 let totalCount = 0
const platformCounts: Record<string, number> = {} const platformCounts: Record<string, number> = {}
+2 -7
View File
@@ -20,14 +20,9 @@ RUN npm install
COPY --chown=node:node match_collector/. . COPY --chown=node:node match_collector/. .
FROM node:current-alpine FROM node:current-alpine
# Install su-exec for dropping privileges
RUN apk add --no-cache su-exec
RUN mkdir -p /home/node/app && chown -R node:node /home/node/app RUN mkdir -p /home/node/app && chown -R node:node /home/node/app
WORKDIR /home/node/app WORKDIR /home/node/app
USER node
COPY --from=build --chown=node:node /home/node/app/match_collector/node_modules ./node_modules COPY --from=build --chown=node:node /home/node/app/match_collector/node_modules ./node_modules
COPY --from=build --chown=node:node /home/node/app/match_collector/. . COPY --from=build --chown=node:node /home/node/app/match_collector/. .
COPY --chown=node:node match_collector/docker-entrypoint.sh /usr/local/bin/ CMD ["/bin/sh", "-c", "node --import=tsx src/index.ts; sleep 20h"]
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
# Run entrypoint as root to fix permissions, then drop to node user
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
CMD ["/bin/sh", "-c", "node --import=tsx src/index.ts; sleep 12h"]
-9
View File
@@ -1,9 +0,0 @@
#!/bin/sh
# Fix permissions on the cdragon cache directory if it exists
if [ -d "/cdragon" ]; then
# Ensure the node user owns the cdragon directory
chown -R node:node /cdragon 2>/dev/null || true
fi
# Execute the main command as the node user
exec su-exec node "$@"
+13 -76
View File
@@ -219,10 +219,17 @@ function handleMatchBuilds(
} }
if (event.type != 'ITEM_PURCHASED') continue if (event.type != 'ITEM_PURCHASED') continue
// Handle boots upgrades
if (
itemInfo.requiredBuffCurrencyName == 'Feats_NoxianBootPurchaseBuff' ||
itemInfo.requiredBuffCurrencyName == 'Feats_SpecialQuestBootBuff'
) {
continue
}
// Handle boots differently // Handle boots differently
if (itemInfo.categories.includes('Boots')) { if (itemInfo.categories.includes('Boots')) {
// Ignore basic boots, only count Tier 2 boots if (itemInfo.to.length == 0 || (itemInfo.to[0] >= 3171 && itemInfo.to[0] <= 3176)) {
if (event.itemId != 1001) {
// Check for bootsFirst // Check for bootsFirst
if (items.length < 2) { if (items.length < 2) {
build.bootsFirstCount += 1 build.bootsFirstCount += 1
@@ -391,35 +398,16 @@ async function handleMatchList(
const database = client.db('matches') const database = client.db('matches')
const collectionName = platform ? `${patch}_${platform}` : patch const collectionName = platform ? `${patch}_${platform}` : patch
const matches = database.collection(collectionName) const matches = database.collection(collectionName)
const allMatches = matches.find()
const totalMatches: number = await matches.countDocuments() const totalMatches: number = await matches.countDocuments()
// Process matches in batches to limit memory usage
const BATCH_SIZE = 1000
let currentMatch = 0 let currentMatch = 0
let processedInBatch = 0 for await (const match of allMatches) {
// Use cursor with batch size to limit memory consumption
const cursor = matches.find().batchSize(BATCH_SIZE)
try {
for await (const match of cursor) {
process.stdout.write( process.stdout.write(
'\rComputing champion stats, game entry ' + currentMatch + '/' + totalMatches + ' ... ' '\rComputing champion stats, game entry ' + currentMatch + '/' + totalMatches + ' ... '
) )
currentMatch += 1 currentMatch += 1
processedInBatch += 1
handleMatch(match as unknown as Match, champions, platform) handleMatch(match as unknown as Match, champions, platform)
// Periodically yield to allow garbage collection and log progress
if (processedInBatch >= BATCH_SIZE) {
processedInBatch = 0
// Small delay to allow garbage collection
await new Promise(resolve => setImmediate(resolve))
}
}
} finally {
// Ensure cursor is closed
await cursor.close()
} }
return totalMatches return totalMatches
@@ -707,8 +695,8 @@ async function finalizeChampionStats(champion: ChampionData, totalMatches: numbe
// Sort matchups by score (games * winrate) in descending order // Sort matchups by score (games * winrate) in descending order
for (const lane of champion.lanes) { for (const lane of champion.lanes) {
if (lane.matchups && lane.matchups.length > 0) { if (lane.matchups && lane.matchups.length > 0) {
// Filter out matchups with insufficient games (minimum 30 games to avoid statistical bias) // Filter out matchups with insufficient games (minimum 5 games)
const filteredMatchups = lane.matchups.filter(m => m.games >= 30) const filteredMatchups = lane.matchups.filter(m => m.games >= 5)
// Sort by score (games * (winrate - 0.5)^2) descending // Sort by score (games * (winrate - 0.5)^2) descending
filteredMatchups.sort((a, b) => { filteredMatchups.sort((a, b) => {
@@ -755,48 +743,6 @@ async function championList() {
return list.slice(1) return list.slice(1)
} }
/**
* Compact matches collections to release memory back to the OS.
* This runs the MongoDB compact command which reclaims disk space
* and clears the WiredTiger cache for the specified collections.
*/
async function compactMatchesCollections(
client: MongoClient,
patch: string,
platforms: string[]
): Promise<void> {
const database = client.db('matches')
console.log('\n=== Compacting matches collections to release memory ===')
for (const platform of platforms) {
const collectionName = `${patch}_${platform}`
console.log(`Compacting collection: ${collectionName}...`)
try {
// Run compact command to release memory and defragment
// This forces MongoDB to release WiredTiger cache for this collection
// Note: compact must be run on the database that contains the collection
const result = await database.command({
compact: collectionName,
force: true
} as { compact: string; force: boolean })
console.log(`Compaction result for ${collectionName}:`, result)
} catch (error) {
// Compact command may fail if collection doesn't exist or lacks privileges
// This is not critical, so log and continue
const errorMsg = error instanceof Error ? error.message : String(error)
if (errorMsg.includes('NamespaceNotFound')) {
console.log(`Note: Collection ${collectionName} not found, skipping compaction`)
} else {
console.log(`Note: Could not compact ${collectionName}:`, errorMsg)
}
}
}
console.log('Compaction complete.')
}
async function makeChampionsStats(client: MongoClient, patch: string, platforms: string[] = []) { async function makeChampionsStats(client: MongoClient, patch: string, platforms: string[] = []) {
const globalItems = await itemList() const globalItems = await itemList()
for (const item of globalItems) { for (const item of globalItems) {
@@ -827,12 +773,6 @@ async function makeChampionsStats(client: MongoClient, patch: string, platforms:
const platformMatches = await handleMatchList(client, patch, champions, platform) const platformMatches = await handleMatchList(client, patch, champions, platform)
totalMatches += platformMatches totalMatches += platformMatches
console.log(`Processed ${platformMatches} matches from ${platform}`) console.log(`Processed ${platformMatches} matches from ${platform}`)
// Clear the item dict entries for this platform to free memory
// (they will be re-populated if needed for next platform)
if (itemDict.size > 0) {
console.log(`Clearing item cache to free memory...`)
}
} }
console.log(`\n=== Total matches processed: ${totalMatches} ===`) console.log(`\n=== Total matches processed: ${totalMatches} ===`)
@@ -848,9 +788,6 @@ async function makeChampionsStats(client: MongoClient, patch: string, platforms:
// Create alias-index for better key-find // Create alias-index for better key-find
await collection.createIndex({ alias: 1 }) await collection.createIndex({ alias: 1 })
console.log(`Stats saved to collection: ${patch}`) console.log(`Stats saved to collection: ${patch}`)
// Compact matches collections to release memory back to the OS
await compactMatchesCollections(client, patch, platforms)
} }
export default { makeChampionsStats } export default { makeChampionsStats }
+3 -64
View File
@@ -4,7 +4,6 @@ const sleep_minutes = 12
import { MongoClient } from 'mongodb' import { MongoClient } from 'mongodb'
import champion_stat from './champion_stat' import champion_stat from './champion_stat'
import stats from './stats'
import { Match } from './api' import { Match } from './api'
import { PLATFORMS, getPlatformBaseUrl, getRegionalBaseUrl, getRegionForPlatform } from './platform' import { PLATFORMS, getPlatformBaseUrl, getRegionalBaseUrl, getRegionForPlatform } from './platform'
import { downloadCDragonAssets } from './cdragon_cache' import { downloadCDragonAssets } from './cdragon_cache'
@@ -28,14 +27,6 @@ function extractPatchFromGameVersion(gameVersion: string): string {
* Collections are named like "15.1_EUW1", "15.2_NA1", etc. * Collections are named like "15.1_EUW1", "15.2_NA1", etc.
*/ */
async function getLatestPatchFromCollections(client: MongoClient): Promise<string | null> { async function getLatestPatchFromCollections(client: MongoClient): Promise<string | null> {
const patches = await getAllPatchesFromCollections(client)
return patches.length > 0 ? patches[0] : null
}
/**
* Get all patches from existing match collections, sorted from latest to oldest.
*/
async function getAllPatchesFromCollections(client: MongoClient): Promise<string[]> {
const matchesDb = client.db('matches') const matchesDb = client.db('matches')
const collections = await matchesDb.listCollections().toArray() const collections = await matchesDb.listCollections().toArray()
const collectionNames = collections.map(c => c.name) const collectionNames = collections.map(c => c.name)
@@ -51,10 +42,10 @@ async function getAllPatchesFromCollections(client: MongoClient): Promise<string
} }
if (patches.size === 0) { if (patches.size === 0) {
return [] return null
} }
// Sort patches from latest to oldest (highest version number first) // Sort patches and return the latest (highest version number)
const sortedPatches = Array.from(patches).sort((a, b) => { const sortedPatches = Array.from(patches).sort((a, b) => {
const [aMajor, aMinor] = a.split('.').map(Number) const [aMajor, aMinor] = a.split('.').map(Number)
const [bMajor, bMinor] = b.split('.').map(Number) const [bMajor, bMinor] = b.split('.').map(Number)
@@ -62,47 +53,7 @@ async function getAllPatchesFromCollections(client: MongoClient): Promise<string
return bMinor - aMinor return bMinor - aMinor
}) })
return sortedPatches return sortedPatches[0]
}
/**
* Clean up match collections that are more than 2 patches old.
* This helps reduce database storage by removing outdated match data.
* @param client MongoDB client
* @param keepPatches Number of recent patches to keep (default: 2)
*/
async function cleanupOldPatches(client: MongoClient, keepPatches: number): Promise<void> {
const matchesDb = client.db('matches')
const allPatches = await getAllPatchesFromCollections(client)
if (allPatches.length <= keepPatches) {
console.log(`Cleanup: Only ${allPatches.length} patch(es) found, nothing to clean up.`)
return
}
// Get patches to remove (everything after the first `keepPatches` patches)
const patchesToRemove = allPatches.slice(keepPatches)
console.log(`Cleanup: Found ${allPatches.length} patches, keeping ${keepPatches} most recent.`)
console.log(`Cleanup: Patches to remove: ${patchesToRemove.join(', ')}`)
// Get all collections to find ones that match patches to remove
const collections = await matchesDb.listCollections().toArray()
const collectionNames = collections.map(c => c.name)
let droppedCount = 0
for (const collectionName of collectionNames) {
const patch = collectionName.split('_')[0]
if (patchesToRemove.includes(patch)) {
console.log(`Cleanup: Dropping collection '${collectionName}'...`)
await matchesDb.dropCollection(collectionName)
droppedCount++
}
}
// Delete stats for removed patches
await stats.deleteStats(client, patchesToRemove)
console.log(`Cleanup: Dropped ${droppedCount} collection(s).`)
} }
async function main() { async function main() {
@@ -167,10 +118,6 @@ async function main() {
} }
} }
// Clean up old patches (keep only 2 most recent patches)
console.log('\n=== Cleaning up old patches ===')
await cleanupOldPatches(client, 2)
// Get the latest patch from collections and generate stats for it // Get the latest patch from collections and generate stats for it
const latestPatch = await getLatestPatchFromCollections(client) const latestPatch = await getLatestPatchFromCollections(client)
if (latestPatch) { if (latestPatch) {
@@ -304,9 +251,6 @@ async function saveMatch(client: MongoClient, match: Match, patch: string, platf
const collectionName = `${patch}_${platform}` const collectionName = `${patch}_${platform}`
const matches = database.collection(collectionName) const matches = database.collection(collectionName)
await matches.insertOne(match) await matches.insertOne(match)
// Increment stats counter for this patch/platform
await stats.incrementMatchCount(client, patch, platform)
} }
/** /**
@@ -355,11 +299,6 @@ async function runWithPreloadedData() {
.filter(name => name.startsWith(`${latestPatch}_`)) .filter(name => name.startsWith(`${latestPatch}_`))
.map(name => name.replace(`${latestPatch}_`, '')) .map(name => name.replace(`${latestPatch}_`, ''))
// Recalculate match count stats from existing collections
if (platforms.length > 0) {
await stats.recalculateStats(client, latestPatch, platforms)
}
// Generate stats for each platform // Generate stats for each platform
if (platforms.length > 0) { if (platforms.length > 0) {
await champion_stat.makeChampionsStats(client, latestPatch, platforms) await champion_stat.makeChampionsStats(client, latestPatch, platforms)
-152
View File
@@ -1,152 +0,0 @@
import { MongoClient } from 'mongodb'
/**
* Stats document structure stored in the 'stats' collection.
* One document per patch, containing total and per-platform match counts.
*/
interface StatsDocument {
patch: string
total: number
platforms: Record<string, number>
updatedAt: Date
}
const STATS_COLLECTION = 'stats'
const STATS_DATABASE = 'matches'
/**
* Initialize stats document for a patch if it doesn't exist.
* This should be called before processing matches for a new patch.
*/
async function initStats(client: MongoClient, patch: string): Promise<void> {
const database = client.db(STATS_DATABASE)
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
await collection.updateOne(
{ patch },
{
$setOnInsert: {
patch,
total: 0,
platforms: {},
updatedAt: new Date()
}
},
{ upsert: true }
)
}
/**
* Increment match count for a specific patch and platform.
* This should be called each time a new match is saved.
*/
async function incrementMatchCount(
client: MongoClient,
patch: string,
platform: string
): Promise<void> {
const database = client.db(STATS_DATABASE)
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
await collection.updateOne(
{ patch },
{
$inc: { total: 1, [`platforms.${platform}`]: 1 },
$set: { updatedAt: new Date() }
},
{ upsert: true }
)
}
/**
* Get stats for a specific patch.
* Returns null if no stats exist for the patch.
*/
async function getStats(client: MongoClient, patch: string): Promise<StatsDocument | null> {
const database = client.db(STATS_DATABASE)
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
return await collection.findOne({ patch })
}
/**
* Get stats for all patches, sorted from latest to oldest.
*/
async function getAllStats(client: MongoClient): Promise<StatsDocument[]> {
const database = client.db(STATS_DATABASE)
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
const stats = await collection.find({}).toArray()
// Sort patches from latest to oldest
return stats.sort((a, b) => {
const [aMajor, aMinor] = a.patch.split('.').map(Number)
const [bMajor, bMinor] = b.patch.split('.').map(Number)
if (aMajor !== bMajor) return bMajor - aMajor
return bMinor - aMinor
})
}
/**
* Delete stats for patches that are being cleaned up.
* This should be called when old patch collections are dropped.
*/
async function deleteStats(client: MongoClient, patches: string[]): Promise<void> {
if (patches.length === 0) return
const database = client.db(STATS_DATABASE)
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
await collection.deleteMany({ patch: { $in: patches } })
console.log(`Stats: Deleted stats for patches: ${patches.join(', ')}`)
}
/**
* Recalculate stats from existing match collections.
* This is useful for migration or fixing inconsistent stats.
*/
async function recalculateStats(
client: MongoClient,
patch: string,
platforms: string[]
): Promise<void> {
const database = client.db(STATS_DATABASE)
let total = 0
const platformCounts: Record<string, number> = {}
for (const platform of platforms) {
const collectionName = `${patch}_${platform}`
const collection = database.collection(collectionName)
const count = await collection.countDocuments()
platformCounts[platform] = count
total += count
}
const statsCollection = database.collection<StatsDocument>(STATS_COLLECTION)
await statsCollection.updateOne(
{ patch },
{
$set: {
patch,
total,
platforms: platformCounts,
updatedAt: new Date()
}
},
{ upsert: true }
)
console.log(`Stats: Recalculated stats for patch ${patch}: ${total} total matches`)
}
export default {
initStats,
incrementMatchCount,
getStats,
getAllStats,
deleteStats,
recalculateStats
}
export type { StatsDocument }