Compare commits
1 Commits
main
..
5d8320fe8a
| Author | SHA1 | Date | |
|---|---|---|---|
|
5d8320fe8a
|
+8
-12
@@ -1,25 +1,21 @@
|
||||
services:
|
||||
# Development MongoDB with memory optimizations
|
||||
# Development MongoDB with performance optimizations
|
||||
mongodb:
|
||||
image: mongo:8.3.4
|
||||
image: mongo:latest
|
||||
container_name: buildpath-mongodb
|
||||
ports:
|
||||
- "27017:27017"
|
||||
environment:
|
||||
MONGO_INITDB_ROOT_USERNAME: ${MONGO_USER:-root}
|
||||
MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASS:-password}
|
||||
GLIBC_TUNABLES: glibc.pthread.rseq=1
|
||||
volumes:
|
||||
- ./data/db:/data/db
|
||||
# Reduced cache size to leave more RAM for the import script
|
||||
# WiredTiger cache is now 2GB (was 4GB) to prevent OOM during large imports
|
||||
command: mongod --wiredTigerCacheSizeGB 2 --quiet
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 4G
|
||||
reservations:
|
||||
memory: 2G
|
||||
command: mongod --wiredTigerCacheSizeGB 4 --quiet
|
||||
healthcheck:
|
||||
test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
mongo-express:
|
||||
image: mongo-express
|
||||
|
||||
@@ -31,29 +31,19 @@ async function importLargeJsonFile(filePath, collectionName, batchSize = 1000) {
|
||||
const collection = db.collection(collectionName);
|
||||
|
||||
try {
|
||||
// Check file size first
|
||||
// Create indexes first for better performance
|
||||
await collection.createIndex({ "metadata.matchId": 1 }, { unique: true });
|
||||
await collection.createIndex({ "info.gameDuration": 1 });
|
||||
await collection.createIndex({ "info.participants.championId": 1 });
|
||||
await collection.createIndex({ "info.participants.win": 1 });
|
||||
|
||||
// Check file size
|
||||
const fileStats = fs.statSync(filePath);
|
||||
const fileSize = (fileStats.size / (1024 * 1024 * 1024)).toFixed(2);
|
||||
console.log(` 📊 File size: ${fileSize} GB`);
|
||||
|
||||
// Defer index creation to after import to reduce memory pressure
|
||||
// Only create the unique matchId index before import to prevent duplicates
|
||||
console.log(` 📇 Creating unique matchId index...`);
|
||||
await collection.createIndex({ "metadata.matchId": 1 }, { unique: true, background: false });
|
||||
|
||||
await processLineDelimitedFormat(filePath, collection, batchSize, startTime);
|
||||
|
||||
// Create additional indexes after import to reduce memory pressure
|
||||
console.log(`\n 📇 Creating additional indexes (this may take a while)...`);
|
||||
try {
|
||||
await collection.createIndex({ "info.gameDuration": 1 }, { background: true });
|
||||
await collection.createIndex({ "info.participants.championId": 1 }, { background: true });
|
||||
await collection.createIndex({ "info.participants.win": 1 }, { background: true });
|
||||
console.log(` ✅ Indexes created successfully`);
|
||||
} catch (indexError) {
|
||||
console.log(` ⚠️ Warning: Could not create additional indexes: ${indexError.message}`);
|
||||
}
|
||||
|
||||
const totalTime = ((Date.now() - startTime) / 1000).toFixed(1);
|
||||
console.log(`🎉 Import complete in ${totalTime} seconds`);
|
||||
console.log(`✅ Processed: ${processed.toLocaleString()} matches`);
|
||||
@@ -76,7 +66,6 @@ async function importLargeJsonFile(filePath, collectionName, batchSize = 1000) {
|
||||
|
||||
let batch = [];
|
||||
let lineCount = 0;
|
||||
let batchCount = 0;
|
||||
|
||||
for await (const line of rl) {
|
||||
lineCount++;
|
||||
@@ -99,16 +88,9 @@ async function importLargeJsonFile(filePath, collectionName, batchSize = 1000) {
|
||||
batch.push(match);
|
||||
|
||||
if (batch.length >= batchSize) {
|
||||
batchCount++;
|
||||
process.stdout.write(`\r Inserting batch #${batchCount} (${batch.length} matches)... `);
|
||||
process.stdout.write(`\r Inserting batch into MongoDB... `);
|
||||
await insertBatch(batch, collection);
|
||||
batch = [];
|
||||
|
||||
// Force garbage collection hint every 10 batches by yielding to the event loop
|
||||
// This helps reduce memory pressure when processing large files
|
||||
if (batchCount % 10 === 0) {
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
skipped++;
|
||||
@@ -117,11 +99,8 @@ async function importLargeJsonFile(filePath, collectionName, batchSize = 1000) {
|
||||
|
||||
// Insert remaining matches
|
||||
if (batch.length > 0) {
|
||||
process.stdout.write(`\r Inserting final batch (${batch.length} matches)... `);
|
||||
await insertBatch(batch, collection);
|
||||
}
|
||||
|
||||
console.log(`\n 📊 Total batches inserted: ${batchCount + 1}`);
|
||||
}
|
||||
|
||||
async function insertBatch(batch, collection) {
|
||||
|
||||
@@ -107,15 +107,4 @@ defineProps<{
|
||||
opacity: 0.8;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
@media only screen and (max-width: 650px) {
|
||||
.item-row {
|
||||
align-items: center;
|
||||
max-width: 100%;
|
||||
margin: 0 20px;
|
||||
}
|
||||
.first-back-content {
|
||||
justify-content: center;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -124,6 +124,7 @@ const itemIconPath = computed(() => CDRAGON_BASE + mapPath(props.item.iconPath))
|
||||
border-radius: 4px;
|
||||
border: 1px solid var(--color-on-surface);
|
||||
overflow: hidden;
|
||||
cursor: help;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
|
||||
@@ -86,11 +86,7 @@ refreshStyles()
|
||||
<div class="rune-spacer-bar" />
|
||||
<div class="rune-holder" style="align-content: end">
|
||||
<div class="rune-slot">
|
||||
<img
|
||||
class="rune-style-img"
|
||||
style="margin: auto"
|
||||
:src="CDRAGON_BASE + mapPath(secondaryStyle.iconPath)"
|
||||
/>
|
||||
<img style="margin: auto" :src="CDRAGON_BASE + mapPath(secondaryStyle.iconPath)" />
|
||||
</div>
|
||||
<div
|
||||
v-for="(slot, slotIndex) in secondaryStyle.slots.slice(1, 4)"
|
||||
@@ -132,10 +128,6 @@ refreshStyles()
|
||||
margin-right: 20px;
|
||||
border: 1px var(--color-on-surface) solid;
|
||||
}
|
||||
.rune-style-img {
|
||||
width: 48px;
|
||||
height: 48px;
|
||||
}
|
||||
|
||||
@media only screen and (max-width: 650px) {
|
||||
.rune-slot {
|
||||
@@ -151,9 +143,5 @@ refreshStyles()
|
||||
margin-left: 10px;
|
||||
margin-right: 10px;
|
||||
}
|
||||
.rune-icon {
|
||||
width: 24px !important;
|
||||
height: 24px !important;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -114,6 +114,7 @@ const perkIconPath = computed(() => CDRAGON_BASE + mapPath(props.perk.iconPath))
|
||||
border-radius: 50%;
|
||||
border: 1px solid var(--color-on-surface);
|
||||
overflow: hidden;
|
||||
cursor: help;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
|
||||
@@ -16,18 +16,12 @@ import type { LaneData } from 'match_collector'
|
||||
// Register
|
||||
ChartJS.register(Title, Tooltip, Legend, BarElement, CategoryScale, LinearScale)
|
||||
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
data: Array<{ title: string; data: Array<{ lane: LaneData; champion: Champion }> }>
|
||||
metric?: 'pickrate' | 'winrate'
|
||||
}>(),
|
||||
{
|
||||
metric: 'pickrate'
|
||||
}
|
||||
)
|
||||
const props = defineProps<{
|
||||
data: Array<{ title: string; data: Array<{ lane: LaneData; champion: Champion }> }>
|
||||
}>()
|
||||
|
||||
const labels: Array<string> = []
|
||||
const values: Array<number> = []
|
||||
const pickrates: Array<number> = []
|
||||
const images: Array<string> = []
|
||||
const backgroundColors: Array<string> = []
|
||||
const CHAMPION_CUT_THRESHOLD = 32
|
||||
@@ -40,7 +34,7 @@ for (const tier of props.data) {
|
||||
if (count > CHAMPION_CUT_THRESHOLD) break
|
||||
|
||||
labels.push(champion.name)
|
||||
values.push(lane[props.metric] * 100)
|
||||
pickrates.push(lane.pickrate * 100)
|
||||
images.push(CDRAGON_BASE + mapPath(champion.squarePortraitPath))
|
||||
backgroundColors.push(TIER_COLORS[colorIndex])
|
||||
|
||||
@@ -53,10 +47,10 @@ const chartData = ref({
|
||||
labels: labels,
|
||||
datasets: [
|
||||
{
|
||||
label: props.metric === 'pickrate' ? 'Pickrate' : 'Winrate',
|
||||
label: 'Pickrate',
|
||||
backgroundColor: backgroundColors,
|
||||
barPercentage: 1.0,
|
||||
data: values
|
||||
data: pickrates
|
||||
}
|
||||
]
|
||||
})
|
||||
@@ -68,24 +62,6 @@ const chartOptions = ref({
|
||||
ticks: {
|
||||
callback: () => ''
|
||||
}
|
||||
},
|
||||
y: {
|
||||
title: {
|
||||
display: true,
|
||||
text: props.metric === 'winrate' ? 'Winrate (%)' : 'Pickrate (%)'
|
||||
},
|
||||
...(props.metric === 'winrate'
|
||||
? {
|
||||
min: 40,
|
||||
max: 60,
|
||||
grid: {
|
||||
color: (context: { tick: { value: number } }) =>
|
||||
context.tick.value === 50 ? '#666' : 'rgba(0, 0, 0, 0.1)',
|
||||
lineWidth: (context: { tick: { value: number } }) =>
|
||||
context.tick.value === 50 ? 2 : 1
|
||||
}
|
||||
}
|
||||
: {})
|
||||
}
|
||||
},
|
||||
plugins: {
|
||||
|
||||
@@ -110,21 +110,14 @@ tiers.push({ title: 'F', data: tierFromScaledPickrate(0, 0.1) })
|
||||
:tier="tier.data"
|
||||
/>
|
||||
|
||||
<h2 style="margin-left: 10px; margin-top: 20px; font-size: 2rem; font-weight: 300">
|
||||
Pickrates
|
||||
</h2>
|
||||
<TierlistChart id="chart-pickrate" :data="tiers" metric="pickrate" />
|
||||
|
||||
<h2 style="margin-left: 10px; font-size: 2rem; font-weight: 300">Winrates</h2>
|
||||
<TierlistChart id="chart-winrate" :data="tiers" metric="winrate" />
|
||||
<TierlistChart id="chart" :data="tiers" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
#chart-pickrate,
|
||||
#chart-winrate {
|
||||
#chart {
|
||||
margin-left: 100px;
|
||||
margin-right: 100px;
|
||||
margin-bottom: 100px;
|
||||
@@ -132,8 +125,7 @@ tiers.push({ title: 'F', data: tierFromScaledPickrate(0, 0.1) })
|
||||
}
|
||||
|
||||
@media only screen and (max-width: 450px) {
|
||||
#chart-pickrate,
|
||||
#chart-winrate {
|
||||
#chart {
|
||||
margin-left: 2px;
|
||||
margin-right: 12px;
|
||||
margin-bottom: 40px;
|
||||
|
||||
@@ -1,29 +1,14 @@
|
||||
import type { MongoClient } from 'mongodb'
|
||||
import { connectToDatabase, fetchLatestPatch, getAvailablePlatforms } from '../utils/mongo'
|
||||
|
||||
interface StatsDocument {
|
||||
patch: string
|
||||
total: number
|
||||
platforms: Record<string, number>
|
||||
updatedAt: Date
|
||||
}
|
||||
|
||||
async function fetchGameCount(client: MongoClient, patch: string) {
|
||||
const database = client.db('matches')
|
||||
const statsCollection = database.collection<StatsDocument>('stats')
|
||||
|
||||
// Try to get stats from the pre-computed stats collection
|
||||
const stats = await statsCollection.findOne({ patch })
|
||||
|
||||
if (stats) {
|
||||
return { total: stats.total, platforms: stats.platforms }
|
||||
}
|
||||
|
||||
// Fallback: compute stats from collections if stats document doesn't exist
|
||||
// This handles the migration case where stats weren't pre-computed
|
||||
// Check for platform-specific collections
|
||||
const platforms = await getAvailablePlatforms(client, patch)
|
||||
|
||||
if (platforms.length > 0) {
|
||||
// Sum counts from all platform-specific collections
|
||||
let totalCount = 0
|
||||
const platformCounts: Record<string, number> = {}
|
||||
|
||||
|
||||
@@ -20,14 +20,9 @@ RUN npm install
|
||||
COPY --chown=node:node match_collector/. .
|
||||
|
||||
FROM node:current-alpine
|
||||
# Install su-exec for dropping privileges
|
||||
RUN apk add --no-cache su-exec
|
||||
RUN mkdir -p /home/node/app && chown -R node:node /home/node/app
|
||||
WORKDIR /home/node/app
|
||||
USER node
|
||||
COPY --from=build --chown=node:node /home/node/app/match_collector/node_modules ./node_modules
|
||||
COPY --from=build --chown=node:node /home/node/app/match_collector/. .
|
||||
COPY --chown=node:node match_collector/docker-entrypoint.sh /usr/local/bin/
|
||||
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
|
||||
# Run entrypoint as root to fix permissions, then drop to node user
|
||||
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
||||
CMD ["/bin/sh", "-c", "node --import=tsx src/index.ts; sleep 12h"]
|
||||
CMD ["/bin/sh", "-c", "node --import=tsx src/index.ts; sleep 20h"]
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Fix permissions on the cdragon cache directory if it exists
|
||||
if [ -d "/cdragon" ]; then
|
||||
# Ensure the node user owns the cdragon directory
|
||||
chown -R node:node /cdragon 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Execute the main command as the node user
|
||||
exec su-exec node "$@"
|
||||
@@ -219,10 +219,17 @@ function handleMatchBuilds(
|
||||
}
|
||||
if (event.type != 'ITEM_PURCHASED') continue
|
||||
|
||||
// Handle boots upgrades
|
||||
if (
|
||||
itemInfo.requiredBuffCurrencyName == 'Feats_NoxianBootPurchaseBuff' ||
|
||||
itemInfo.requiredBuffCurrencyName == 'Feats_SpecialQuestBootBuff'
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Handle boots differently
|
||||
if (itemInfo.categories.includes('Boots')) {
|
||||
// Ignore basic boots, only count Tier 2 boots
|
||||
if (event.itemId != 1001) {
|
||||
if (itemInfo.to.length == 0 || (itemInfo.to[0] >= 3171 && itemInfo.to[0] <= 3176)) {
|
||||
// Check for bootsFirst
|
||||
if (items.length < 2) {
|
||||
build.bootsFirstCount += 1
|
||||
@@ -391,35 +398,16 @@ async function handleMatchList(
|
||||
const database = client.db('matches')
|
||||
const collectionName = platform ? `${patch}_${platform}` : patch
|
||||
const matches = database.collection(collectionName)
|
||||
const allMatches = matches.find()
|
||||
const totalMatches: number = await matches.countDocuments()
|
||||
|
||||
// Process matches in batches to limit memory usage
|
||||
const BATCH_SIZE = 1000
|
||||
let currentMatch = 0
|
||||
let processedInBatch = 0
|
||||
|
||||
// Use cursor with batch size to limit memory consumption
|
||||
const cursor = matches.find().batchSize(BATCH_SIZE)
|
||||
|
||||
try {
|
||||
for await (const match of cursor) {
|
||||
process.stdout.write(
|
||||
'\rComputing champion stats, game entry ' + currentMatch + '/' + totalMatches + ' ... '
|
||||
)
|
||||
currentMatch += 1
|
||||
processedInBatch += 1
|
||||
handleMatch(match as unknown as Match, champions, platform)
|
||||
|
||||
// Periodically yield to allow garbage collection and log progress
|
||||
if (processedInBatch >= BATCH_SIZE) {
|
||||
processedInBatch = 0
|
||||
// Small delay to allow garbage collection
|
||||
await new Promise(resolve => setImmediate(resolve))
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
// Ensure cursor is closed
|
||||
await cursor.close()
|
||||
for await (const match of allMatches) {
|
||||
process.stdout.write(
|
||||
'\rComputing champion stats, game entry ' + currentMatch + '/' + totalMatches + ' ... '
|
||||
)
|
||||
currentMatch += 1
|
||||
handleMatch(match as unknown as Match, champions, platform)
|
||||
}
|
||||
|
||||
return totalMatches
|
||||
@@ -707,8 +695,8 @@ async function finalizeChampionStats(champion: ChampionData, totalMatches: numbe
|
||||
// Sort matchups by score (games * winrate) in descending order
|
||||
for (const lane of champion.lanes) {
|
||||
if (lane.matchups && lane.matchups.length > 0) {
|
||||
// Filter out matchups with insufficient games (minimum 30 games to avoid statistical bias)
|
||||
const filteredMatchups = lane.matchups.filter(m => m.games >= 30)
|
||||
// Filter out matchups with insufficient games (minimum 5 games)
|
||||
const filteredMatchups = lane.matchups.filter(m => m.games >= 5)
|
||||
|
||||
// Sort by score (games * (winrate - 0.5)^2) descending
|
||||
filteredMatchups.sort((a, b) => {
|
||||
@@ -755,48 +743,6 @@ async function championList() {
|
||||
return list.slice(1)
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact matches collections to release memory back to the OS.
|
||||
* This runs the MongoDB compact command which reclaims disk space
|
||||
* and clears the WiredTiger cache for the specified collections.
|
||||
*/
|
||||
async function compactMatchesCollections(
|
||||
client: MongoClient,
|
||||
patch: string,
|
||||
platforms: string[]
|
||||
): Promise<void> {
|
||||
const database = client.db('matches')
|
||||
|
||||
console.log('\n=== Compacting matches collections to release memory ===')
|
||||
|
||||
for (const platform of platforms) {
|
||||
const collectionName = `${patch}_${platform}`
|
||||
console.log(`Compacting collection: ${collectionName}...`)
|
||||
|
||||
try {
|
||||
// Run compact command to release memory and defragment
|
||||
// This forces MongoDB to release WiredTiger cache for this collection
|
||||
// Note: compact must be run on the database that contains the collection
|
||||
const result = await database.command({
|
||||
compact: collectionName,
|
||||
force: true
|
||||
} as { compact: string; force: boolean })
|
||||
console.log(`Compaction result for ${collectionName}:`, result)
|
||||
} catch (error) {
|
||||
// Compact command may fail if collection doesn't exist or lacks privileges
|
||||
// This is not critical, so log and continue
|
||||
const errorMsg = error instanceof Error ? error.message : String(error)
|
||||
if (errorMsg.includes('NamespaceNotFound')) {
|
||||
console.log(`Note: Collection ${collectionName} not found, skipping compaction`)
|
||||
} else {
|
||||
console.log(`Note: Could not compact ${collectionName}:`, errorMsg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Compaction complete.')
|
||||
}
|
||||
|
||||
async function makeChampionsStats(client: MongoClient, patch: string, platforms: string[] = []) {
|
||||
const globalItems = await itemList()
|
||||
for (const item of globalItems) {
|
||||
@@ -827,12 +773,6 @@ async function makeChampionsStats(client: MongoClient, patch: string, platforms:
|
||||
const platformMatches = await handleMatchList(client, patch, champions, platform)
|
||||
totalMatches += platformMatches
|
||||
console.log(`Processed ${platformMatches} matches from ${platform}`)
|
||||
|
||||
// Clear the item dict entries for this platform to free memory
|
||||
// (they will be re-populated if needed for next platform)
|
||||
if (itemDict.size > 0) {
|
||||
console.log(`Clearing item cache to free memory...`)
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n=== Total matches processed: ${totalMatches} ===`)
|
||||
@@ -848,9 +788,6 @@ async function makeChampionsStats(client: MongoClient, patch: string, platforms:
|
||||
// Create alias-index for better key-find
|
||||
await collection.createIndex({ alias: 1 })
|
||||
console.log(`Stats saved to collection: ${patch}`)
|
||||
|
||||
// Compact matches collections to release memory back to the OS
|
||||
await compactMatchesCollections(client, patch, platforms)
|
||||
}
|
||||
|
||||
export default { makeChampionsStats }
|
||||
|
||||
@@ -4,7 +4,6 @@ const sleep_minutes = 12
|
||||
import { MongoClient } from 'mongodb'
|
||||
|
||||
import champion_stat from './champion_stat'
|
||||
import stats from './stats'
|
||||
import { Match } from './api'
|
||||
import { PLATFORMS, getPlatformBaseUrl, getRegionalBaseUrl, getRegionForPlatform } from './platform'
|
||||
import { downloadCDragonAssets } from './cdragon_cache'
|
||||
@@ -28,14 +27,6 @@ function extractPatchFromGameVersion(gameVersion: string): string {
|
||||
* Collections are named like "15.1_EUW1", "15.2_NA1", etc.
|
||||
*/
|
||||
async function getLatestPatchFromCollections(client: MongoClient): Promise<string | null> {
|
||||
const patches = await getAllPatchesFromCollections(client)
|
||||
return patches.length > 0 ? patches[0] : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all patches from existing match collections, sorted from latest to oldest.
|
||||
*/
|
||||
async function getAllPatchesFromCollections(client: MongoClient): Promise<string[]> {
|
||||
const matchesDb = client.db('matches')
|
||||
const collections = await matchesDb.listCollections().toArray()
|
||||
const collectionNames = collections.map(c => c.name)
|
||||
@@ -51,10 +42,10 @@ async function getAllPatchesFromCollections(client: MongoClient): Promise<string
|
||||
}
|
||||
|
||||
if (patches.size === 0) {
|
||||
return []
|
||||
return null
|
||||
}
|
||||
|
||||
// Sort patches from latest to oldest (highest version number first)
|
||||
// Sort patches and return the latest (highest version number)
|
||||
const sortedPatches = Array.from(patches).sort((a, b) => {
|
||||
const [aMajor, aMinor] = a.split('.').map(Number)
|
||||
const [bMajor, bMinor] = b.split('.').map(Number)
|
||||
@@ -62,47 +53,7 @@ async function getAllPatchesFromCollections(client: MongoClient): Promise<string
|
||||
return bMinor - aMinor
|
||||
})
|
||||
|
||||
return sortedPatches
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up match collections that are more than 2 patches old.
|
||||
* This helps reduce database storage by removing outdated match data.
|
||||
* @param client MongoDB client
|
||||
* @param keepPatches Number of recent patches to keep (default: 2)
|
||||
*/
|
||||
async function cleanupOldPatches(client: MongoClient, keepPatches: number): Promise<void> {
|
||||
const matchesDb = client.db('matches')
|
||||
const allPatches = await getAllPatchesFromCollections(client)
|
||||
|
||||
if (allPatches.length <= keepPatches) {
|
||||
console.log(`Cleanup: Only ${allPatches.length} patch(es) found, nothing to clean up.`)
|
||||
return
|
||||
}
|
||||
|
||||
// Get patches to remove (everything after the first `keepPatches` patches)
|
||||
const patchesToRemove = allPatches.slice(keepPatches)
|
||||
console.log(`Cleanup: Found ${allPatches.length} patches, keeping ${keepPatches} most recent.`)
|
||||
console.log(`Cleanup: Patches to remove: ${patchesToRemove.join(', ')}`)
|
||||
|
||||
// Get all collections to find ones that match patches to remove
|
||||
const collections = await matchesDb.listCollections().toArray()
|
||||
const collectionNames = collections.map(c => c.name)
|
||||
|
||||
let droppedCount = 0
|
||||
for (const collectionName of collectionNames) {
|
||||
const patch = collectionName.split('_')[0]
|
||||
if (patchesToRemove.includes(patch)) {
|
||||
console.log(`Cleanup: Dropping collection '${collectionName}'...`)
|
||||
await matchesDb.dropCollection(collectionName)
|
||||
droppedCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Delete stats for removed patches
|
||||
await stats.deleteStats(client, patchesToRemove)
|
||||
|
||||
console.log(`Cleanup: Dropped ${droppedCount} collection(s).`)
|
||||
return sortedPatches[0]
|
||||
}
|
||||
|
||||
async function main() {
|
||||
@@ -167,10 +118,6 @@ async function main() {
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up old patches (keep only 2 most recent patches)
|
||||
console.log('\n=== Cleaning up old patches ===')
|
||||
await cleanupOldPatches(client, 2)
|
||||
|
||||
// Get the latest patch from collections and generate stats for it
|
||||
const latestPatch = await getLatestPatchFromCollections(client)
|
||||
if (latestPatch) {
|
||||
@@ -304,9 +251,6 @@ async function saveMatch(client: MongoClient, match: Match, patch: string, platf
|
||||
const collectionName = `${patch}_${platform}`
|
||||
const matches = database.collection(collectionName)
|
||||
await matches.insertOne(match)
|
||||
|
||||
// Increment stats counter for this patch/platform
|
||||
await stats.incrementMatchCount(client, patch, platform)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -355,11 +299,6 @@ async function runWithPreloadedData() {
|
||||
.filter(name => name.startsWith(`${latestPatch}_`))
|
||||
.map(name => name.replace(`${latestPatch}_`, ''))
|
||||
|
||||
// Recalculate match count stats from existing collections
|
||||
if (platforms.length > 0) {
|
||||
await stats.recalculateStats(client, latestPatch, platforms)
|
||||
}
|
||||
|
||||
// Generate stats for each platform
|
||||
if (platforms.length > 0) {
|
||||
await champion_stat.makeChampionsStats(client, latestPatch, platforms)
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
import { MongoClient } from 'mongodb'
|
||||
|
||||
/**
|
||||
* Stats document structure stored in the 'stats' collection.
|
||||
* One document per patch, containing total and per-platform match counts.
|
||||
*/
|
||||
interface StatsDocument {
|
||||
patch: string
|
||||
total: number
|
||||
platforms: Record<string, number>
|
||||
updatedAt: Date
|
||||
}
|
||||
|
||||
const STATS_COLLECTION = 'stats'
|
||||
const STATS_DATABASE = 'matches'
|
||||
|
||||
/**
|
||||
* Initialize stats document for a patch if it doesn't exist.
|
||||
* This should be called before processing matches for a new patch.
|
||||
*/
|
||||
async function initStats(client: MongoClient, patch: string): Promise<void> {
|
||||
const database = client.db(STATS_DATABASE)
|
||||
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
|
||||
|
||||
await collection.updateOne(
|
||||
{ patch },
|
||||
{
|
||||
$setOnInsert: {
|
||||
patch,
|
||||
total: 0,
|
||||
platforms: {},
|
||||
updatedAt: new Date()
|
||||
}
|
||||
},
|
||||
{ upsert: true }
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment match count for a specific patch and platform.
|
||||
* This should be called each time a new match is saved.
|
||||
*/
|
||||
async function incrementMatchCount(
|
||||
client: MongoClient,
|
||||
patch: string,
|
||||
platform: string
|
||||
): Promise<void> {
|
||||
const database = client.db(STATS_DATABASE)
|
||||
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
|
||||
|
||||
await collection.updateOne(
|
||||
{ patch },
|
||||
{
|
||||
$inc: { total: 1, [`platforms.${platform}`]: 1 },
|
||||
$set: { updatedAt: new Date() }
|
||||
},
|
||||
{ upsert: true }
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get stats for a specific patch.
|
||||
* Returns null if no stats exist for the patch.
|
||||
*/
|
||||
async function getStats(client: MongoClient, patch: string): Promise<StatsDocument | null> {
|
||||
const database = client.db(STATS_DATABASE)
|
||||
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
|
||||
|
||||
return await collection.findOne({ patch })
|
||||
}
|
||||
|
||||
/**
|
||||
* Get stats for all patches, sorted from latest to oldest.
|
||||
*/
|
||||
async function getAllStats(client: MongoClient): Promise<StatsDocument[]> {
|
||||
const database = client.db(STATS_DATABASE)
|
||||
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
|
||||
|
||||
const stats = await collection.find({}).toArray()
|
||||
|
||||
// Sort patches from latest to oldest
|
||||
return stats.sort((a, b) => {
|
||||
const [aMajor, aMinor] = a.patch.split('.').map(Number)
|
||||
const [bMajor, bMinor] = b.patch.split('.').map(Number)
|
||||
if (aMajor !== bMajor) return bMajor - aMajor
|
||||
return bMinor - aMinor
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete stats for patches that are being cleaned up.
|
||||
* This should be called when old patch collections are dropped.
|
||||
*/
|
||||
async function deleteStats(client: MongoClient, patches: string[]): Promise<void> {
|
||||
if (patches.length === 0) return
|
||||
|
||||
const database = client.db(STATS_DATABASE)
|
||||
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
|
||||
|
||||
await collection.deleteMany({ patch: { $in: patches } })
|
||||
console.log(`Stats: Deleted stats for patches: ${patches.join(', ')}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Recalculate stats from existing match collections.
|
||||
* This is useful for migration or fixing inconsistent stats.
|
||||
*/
|
||||
async function recalculateStats(
|
||||
client: MongoClient,
|
||||
patch: string,
|
||||
platforms: string[]
|
||||
): Promise<void> {
|
||||
const database = client.db(STATS_DATABASE)
|
||||
|
||||
let total = 0
|
||||
const platformCounts: Record<string, number> = {}
|
||||
|
||||
for (const platform of platforms) {
|
||||
const collectionName = `${patch}_${platform}`
|
||||
const collection = database.collection(collectionName)
|
||||
const count = await collection.countDocuments()
|
||||
platformCounts[platform] = count
|
||||
total += count
|
||||
}
|
||||
|
||||
const statsCollection = database.collection<StatsDocument>(STATS_COLLECTION)
|
||||
await statsCollection.updateOne(
|
||||
{ patch },
|
||||
{
|
||||
$set: {
|
||||
patch,
|
||||
total,
|
||||
platforms: platformCounts,
|
||||
updatedAt: new Date()
|
||||
}
|
||||
},
|
||||
{ upsert: true }
|
||||
)
|
||||
|
||||
console.log(`Stats: Recalculated stats for patch ${patch}: ${total} total matches`)
|
||||
}
|
||||
|
||||
export default {
|
||||
initStats,
|
||||
incrementMatchCount,
|
||||
getStats,
|
||||
getAllStats,
|
||||
deleteStats,
|
||||
recalculateStats
|
||||
}
|
||||
|
||||
export type { StatsDocument }
|
||||
Reference in New Issue
Block a user