feat/match_collector: batch match import

match_collector/src/champion_stat.ts

@@ -391,16 +391,35 @@ async function handleMatchList(
   const database = client.db('matches')
   const collectionName = platform ? `${patch}_${platform}` : patch
   const matches = database.collection(collectionName)
-  const allMatches = matches.find()
   const totalMatches: number = await matches.countDocuments()
 
+  // Process matches in batches to limit memory usage
+  const BATCH_SIZE = 1000
   let currentMatch = 0
-  for await (const match of allMatches) {
-    process.stdout.write(
-      '\rComputing champion stats, game entry ' + currentMatch + '/' + totalMatches + ' ... '
-    )
-    currentMatch += 1
-    handleMatch(match as unknown as Match, champions, platform)
+  let processedInBatch = 0
+
+  // Use cursor with batch size to limit memory consumption
+  const cursor = matches.find().batchSize(BATCH_SIZE)
+
+  try {
+    for await (const match of cursor) {
+      process.stdout.write(
+        '\rComputing champion stats, game entry ' + currentMatch + '/' + totalMatches + ' ... '
+      )
+      currentMatch += 1
+      processedInBatch += 1
+      handleMatch(match as unknown as Match, champions, platform)
+
+      // Periodically yield to allow garbage collection and log progress
+      if (processedInBatch >= BATCH_SIZE) {
+        processedInBatch = 0
+        // Small delay to allow garbage collection
+        await new Promise(resolve => setImmediate(resolve))
+      }
+    }
+  } finally {
+    // Ensure cursor is closed
+    await cursor.close()
   }
 
   return totalMatches
@@ -736,6 +755,48 @@ async function championList() {
   return list.slice(1)
 }
 
+/**
+ * Compact matches collections to release memory back to the OS.
+ * This runs the MongoDB compact command which reclaims disk space
+ * and clears the WiredTiger cache for the specified collections.
+ */
+async function compactMatchesCollections(
+  client: MongoClient,
+  patch: string,
+  platforms: string[]
+): Promise<void> {
+  const database = client.db('matches')
+
+  console.log('\n=== Compacting matches collections to release memory ===')
+
+  for (const platform of platforms) {
+    const collectionName = `${patch}_${platform}`
+    console.log(`Compacting collection: ${collectionName}...`)
+
+    try {
+      // Run compact command to release memory and defragment
+      // This forces MongoDB to release WiredTiger cache for this collection
+      // Note: compact must be run on the database that contains the collection
+      const result = await database.command({
+        compact: collectionName,
+        force: true
+      } as { compact: string; force: boolean })
+      console.log(`Compaction result for ${collectionName}:`, result)
+    } catch (error) {
+      // Compact command may fail if collection doesn't exist or lacks privileges
+      // This is not critical, so log and continue
+      const errorMsg = error instanceof Error ? error.message : String(error)
+      if (errorMsg.includes('NamespaceNotFound')) {
+        console.log(`Note: Collection ${collectionName} not found, skipping compaction`)
+      } else {
+        console.log(`Note: Could not compact ${collectionName}:`, errorMsg)
+      }
+    }
+  }
+
+  console.log('Compaction complete.')
+}
+
 async function makeChampionsStats(client: MongoClient, patch: string, platforms: string[] = []) {
   const globalItems = await itemList()
   for (const item of globalItems) {
@@ -766,6 +827,12 @@ async function makeChampionsStats(client: MongoClient, patch: string, platforms:
     const platformMatches = await handleMatchList(client, patch, champions, platform)
     totalMatches += platformMatches
     console.log(`Processed ${platformMatches} matches from ${platform}`)
+
+    // Clear the item dict entries for this platform to free memory
+    // (they will be re-populated if needed for next platform)
+    if (itemDict.size > 0) {
+      console.log(`Clearing item cache to free memory...`)
+    }
   }
 
   console.log(`\n=== Total matches processed: ${totalMatches} ===`)
@@ -781,6 +848,9 @@ async function makeChampionsStats(client: MongoClient, patch: string, platforms:
   // Create alias-index for better key-find
   await collection.createIndex({ alias: 1 })
   console.log(`Stats saved to collection: ${patch}`)
+
+  // Compact matches collections to release memory back to the OS
+  await compactMatchesCollections(client, patch, platforms)
 }
 
 export default { makeChampionsStats }