Compare commits
1 Commits
main
..
5d8320fe8a
| Author | SHA1 | Date | |
|---|---|---|---|
|
5d8320fe8a
|
+8
-12
@@ -1,25 +1,21 @@
|
|||||||
services:
|
services:
|
||||||
# Development MongoDB with memory optimizations
|
# Development MongoDB with performance optimizations
|
||||||
mongodb:
|
mongodb:
|
||||||
image: mongo:8.3.4
|
image: mongo:latest
|
||||||
container_name: buildpath-mongodb
|
container_name: buildpath-mongodb
|
||||||
ports:
|
ports:
|
||||||
- "27017:27017"
|
- "27017:27017"
|
||||||
environment:
|
environment:
|
||||||
MONGO_INITDB_ROOT_USERNAME: ${MONGO_USER:-root}
|
MONGO_INITDB_ROOT_USERNAME: ${MONGO_USER:-root}
|
||||||
MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASS:-password}
|
MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASS:-password}
|
||||||
GLIBC_TUNABLES: glibc.pthread.rseq=1
|
|
||||||
volumes:
|
volumes:
|
||||||
- ./data/db:/data/db
|
- ./data/db:/data/db
|
||||||
# Reduced cache size to leave more RAM for the import script
|
command: mongod --wiredTigerCacheSizeGB 4 --quiet
|
||||||
# WiredTiger cache is now 2GB (was 4GB) to prevent OOM during large imports
|
healthcheck:
|
||||||
command: mongod --wiredTigerCacheSizeGB 2 --quiet
|
test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet
|
||||||
deploy:
|
interval: 5s
|
||||||
resources:
|
timeout: 2s
|
||||||
limits:
|
retries: 30
|
||||||
memory: 4G
|
|
||||||
reservations:
|
|
||||||
memory: 2G
|
|
||||||
|
|
||||||
mongo-express:
|
mongo-express:
|
||||||
image: mongo-express
|
image: mongo-express
|
||||||
|
|||||||
@@ -31,29 +31,19 @@ async function importLargeJsonFile(filePath, collectionName, batchSize = 1000) {
|
|||||||
const collection = db.collection(collectionName);
|
const collection = db.collection(collectionName);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Check file size first
|
// Create indexes first for better performance
|
||||||
|
await collection.createIndex({ "metadata.matchId": 1 }, { unique: true });
|
||||||
|
await collection.createIndex({ "info.gameDuration": 1 });
|
||||||
|
await collection.createIndex({ "info.participants.championId": 1 });
|
||||||
|
await collection.createIndex({ "info.participants.win": 1 });
|
||||||
|
|
||||||
|
// Check file size
|
||||||
const fileStats = fs.statSync(filePath);
|
const fileStats = fs.statSync(filePath);
|
||||||
const fileSize = (fileStats.size / (1024 * 1024 * 1024)).toFixed(2);
|
const fileSize = (fileStats.size / (1024 * 1024 * 1024)).toFixed(2);
|
||||||
console.log(` 📊 File size: ${fileSize} GB`);
|
console.log(` 📊 File size: ${fileSize} GB`);
|
||||||
|
|
||||||
// Defer index creation to after import to reduce memory pressure
|
|
||||||
// Only create the unique matchId index before import to prevent duplicates
|
|
||||||
console.log(` 📇 Creating unique matchId index...`);
|
|
||||||
await collection.createIndex({ "metadata.matchId": 1 }, { unique: true, background: false });
|
|
||||||
|
|
||||||
await processLineDelimitedFormat(filePath, collection, batchSize, startTime);
|
await processLineDelimitedFormat(filePath, collection, batchSize, startTime);
|
||||||
|
|
||||||
// Create additional indexes after import to reduce memory pressure
|
|
||||||
console.log(`\n 📇 Creating additional indexes (this may take a while)...`);
|
|
||||||
try {
|
|
||||||
await collection.createIndex({ "info.gameDuration": 1 }, { background: true });
|
|
||||||
await collection.createIndex({ "info.participants.championId": 1 }, { background: true });
|
|
||||||
await collection.createIndex({ "info.participants.win": 1 }, { background: true });
|
|
||||||
console.log(` ✅ Indexes created successfully`);
|
|
||||||
} catch (indexError) {
|
|
||||||
console.log(` ⚠️ Warning: Could not create additional indexes: ${indexError.message}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const totalTime = ((Date.now() - startTime) / 1000).toFixed(1);
|
const totalTime = ((Date.now() - startTime) / 1000).toFixed(1);
|
||||||
console.log(`🎉 Import complete in ${totalTime} seconds`);
|
console.log(`🎉 Import complete in ${totalTime} seconds`);
|
||||||
console.log(`✅ Processed: ${processed.toLocaleString()} matches`);
|
console.log(`✅ Processed: ${processed.toLocaleString()} matches`);
|
||||||
@@ -76,7 +66,6 @@ async function importLargeJsonFile(filePath, collectionName, batchSize = 1000) {
|
|||||||
|
|
||||||
let batch = [];
|
let batch = [];
|
||||||
let lineCount = 0;
|
let lineCount = 0;
|
||||||
let batchCount = 0;
|
|
||||||
|
|
||||||
for await (const line of rl) {
|
for await (const line of rl) {
|
||||||
lineCount++;
|
lineCount++;
|
||||||
@@ -99,16 +88,9 @@ async function importLargeJsonFile(filePath, collectionName, batchSize = 1000) {
|
|||||||
batch.push(match);
|
batch.push(match);
|
||||||
|
|
||||||
if (batch.length >= batchSize) {
|
if (batch.length >= batchSize) {
|
||||||
batchCount++;
|
process.stdout.write(`\r Inserting batch into MongoDB... `);
|
||||||
process.stdout.write(`\r Inserting batch #${batchCount} (${batch.length} matches)... `);
|
|
||||||
await insertBatch(batch, collection);
|
await insertBatch(batch, collection);
|
||||||
batch = [];
|
batch = [];
|
||||||
|
|
||||||
// Force garbage collection hint every 10 batches by yielding to the event loop
|
|
||||||
// This helps reduce memory pressure when processing large files
|
|
||||||
if (batchCount % 10 === 0) {
|
|
||||||
await new Promise(resolve => setImmediate(resolve));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
skipped++;
|
skipped++;
|
||||||
@@ -117,11 +99,8 @@ async function importLargeJsonFile(filePath, collectionName, batchSize = 1000) {
|
|||||||
|
|
||||||
// Insert remaining matches
|
// Insert remaining matches
|
||||||
if (batch.length > 0) {
|
if (batch.length > 0) {
|
||||||
process.stdout.write(`\r Inserting final batch (${batch.length} matches)... `);
|
|
||||||
await insertBatch(batch, collection);
|
await insertBatch(batch, collection);
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`\n 📊 Total batches inserted: ${batchCount + 1}`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function insertBatch(batch, collection) {
|
async function insertBatch(batch, collection) {
|
||||||
|
|||||||
@@ -107,15 +107,4 @@ defineProps<{
|
|||||||
opacity: 0.8;
|
opacity: 0.8;
|
||||||
white-space: nowrap;
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
@media only screen and (max-width: 650px) {
|
|
||||||
.item-row {
|
|
||||||
align-items: center;
|
|
||||||
max-width: 100%;
|
|
||||||
margin: 0 20px;
|
|
||||||
}
|
|
||||||
.first-back-content {
|
|
||||||
justify-content: center;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -124,6 +124,7 @@ const itemIconPath = computed(() => CDRAGON_BASE + mapPath(props.item.iconPath))
|
|||||||
border-radius: 4px;
|
border-radius: 4px;
|
||||||
border: 1px solid var(--color-on-surface);
|
border: 1px solid var(--color-on-surface);
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
|
cursor: help;
|
||||||
position: relative;
|
position: relative;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -86,11 +86,7 @@ refreshStyles()
|
|||||||
<div class="rune-spacer-bar" />
|
<div class="rune-spacer-bar" />
|
||||||
<div class="rune-holder" style="align-content: end">
|
<div class="rune-holder" style="align-content: end">
|
||||||
<div class="rune-slot">
|
<div class="rune-slot">
|
||||||
<img
|
<img style="margin: auto" :src="CDRAGON_BASE + mapPath(secondaryStyle.iconPath)" />
|
||||||
class="rune-style-img"
|
|
||||||
style="margin: auto"
|
|
||||||
:src="CDRAGON_BASE + mapPath(secondaryStyle.iconPath)"
|
|
||||||
/>
|
|
||||||
</div>
|
</div>
|
||||||
<div
|
<div
|
||||||
v-for="(slot, slotIndex) in secondaryStyle.slots.slice(1, 4)"
|
v-for="(slot, slotIndex) in secondaryStyle.slots.slice(1, 4)"
|
||||||
@@ -132,10 +128,6 @@ refreshStyles()
|
|||||||
margin-right: 20px;
|
margin-right: 20px;
|
||||||
border: 1px var(--color-on-surface) solid;
|
border: 1px var(--color-on-surface) solid;
|
||||||
}
|
}
|
||||||
.rune-style-img {
|
|
||||||
width: 48px;
|
|
||||||
height: 48px;
|
|
||||||
}
|
|
||||||
|
|
||||||
@media only screen and (max-width: 650px) {
|
@media only screen and (max-width: 650px) {
|
||||||
.rune-slot {
|
.rune-slot {
|
||||||
@@ -151,9 +143,5 @@ refreshStyles()
|
|||||||
margin-left: 10px;
|
margin-left: 10px;
|
||||||
margin-right: 10px;
|
margin-right: 10px;
|
||||||
}
|
}
|
||||||
.rune-icon {
|
|
||||||
width: 24px !important;
|
|
||||||
height: 24px !important;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -114,6 +114,7 @@ const perkIconPath = computed(() => CDRAGON_BASE + mapPath(props.perk.iconPath))
|
|||||||
border-radius: 50%;
|
border-radius: 50%;
|
||||||
border: 1px solid var(--color-on-surface);
|
border: 1px solid var(--color-on-surface);
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
|
cursor: help;
|
||||||
position: relative;
|
position: relative;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,18 +16,12 @@ import type { LaneData } from 'match_collector'
|
|||||||
// Register
|
// Register
|
||||||
ChartJS.register(Title, Tooltip, Legend, BarElement, CategoryScale, LinearScale)
|
ChartJS.register(Title, Tooltip, Legend, BarElement, CategoryScale, LinearScale)
|
||||||
|
|
||||||
const props = withDefaults(
|
const props = defineProps<{
|
||||||
defineProps<{
|
data: Array<{ title: string; data: Array<{ lane: LaneData; champion: Champion }> }>
|
||||||
data: Array<{ title: string; data: Array<{ lane: LaneData; champion: Champion }> }>
|
}>()
|
||||||
metric?: 'pickrate' | 'winrate'
|
|
||||||
}>(),
|
|
||||||
{
|
|
||||||
metric: 'pickrate'
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
const labels: Array<string> = []
|
const labels: Array<string> = []
|
||||||
const values: Array<number> = []
|
const pickrates: Array<number> = []
|
||||||
const images: Array<string> = []
|
const images: Array<string> = []
|
||||||
const backgroundColors: Array<string> = []
|
const backgroundColors: Array<string> = []
|
||||||
const CHAMPION_CUT_THRESHOLD = 32
|
const CHAMPION_CUT_THRESHOLD = 32
|
||||||
@@ -40,7 +34,7 @@ for (const tier of props.data) {
|
|||||||
if (count > CHAMPION_CUT_THRESHOLD) break
|
if (count > CHAMPION_CUT_THRESHOLD) break
|
||||||
|
|
||||||
labels.push(champion.name)
|
labels.push(champion.name)
|
||||||
values.push(lane[props.metric] * 100)
|
pickrates.push(lane.pickrate * 100)
|
||||||
images.push(CDRAGON_BASE + mapPath(champion.squarePortraitPath))
|
images.push(CDRAGON_BASE + mapPath(champion.squarePortraitPath))
|
||||||
backgroundColors.push(TIER_COLORS[colorIndex])
|
backgroundColors.push(TIER_COLORS[colorIndex])
|
||||||
|
|
||||||
@@ -53,10 +47,10 @@ const chartData = ref({
|
|||||||
labels: labels,
|
labels: labels,
|
||||||
datasets: [
|
datasets: [
|
||||||
{
|
{
|
||||||
label: props.metric === 'pickrate' ? 'Pickrate' : 'Winrate',
|
label: 'Pickrate',
|
||||||
backgroundColor: backgroundColors,
|
backgroundColor: backgroundColors,
|
||||||
barPercentage: 1.0,
|
barPercentage: 1.0,
|
||||||
data: values
|
data: pickrates
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
})
|
})
|
||||||
@@ -68,24 +62,6 @@ const chartOptions = ref({
|
|||||||
ticks: {
|
ticks: {
|
||||||
callback: () => ''
|
callback: () => ''
|
||||||
}
|
}
|
||||||
},
|
|
||||||
y: {
|
|
||||||
title: {
|
|
||||||
display: true,
|
|
||||||
text: props.metric === 'winrate' ? 'Winrate (%)' : 'Pickrate (%)'
|
|
||||||
},
|
|
||||||
...(props.metric === 'winrate'
|
|
||||||
? {
|
|
||||||
min: 40,
|
|
||||||
max: 60,
|
|
||||||
grid: {
|
|
||||||
color: (context: { tick: { value: number } }) =>
|
|
||||||
context.tick.value === 50 ? '#666' : 'rgba(0, 0, 0, 0.1)',
|
|
||||||
lineWidth: (context: { tick: { value: number } }) =>
|
|
||||||
context.tick.value === 50 ? 2 : 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
: {})
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
plugins: {
|
plugins: {
|
||||||
|
|||||||
@@ -110,21 +110,14 @@ tiers.push({ title: 'F', data: tierFromScaledPickrate(0, 0.1) })
|
|||||||
:tier="tier.data"
|
:tier="tier.data"
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<h2 style="margin-left: 10px; margin-top: 20px; font-size: 2rem; font-weight: 300">
|
<TierlistChart id="chart" :data="tiers" />
|
||||||
Pickrates
|
|
||||||
</h2>
|
|
||||||
<TierlistChart id="chart-pickrate" :data="tiers" metric="pickrate" />
|
|
||||||
|
|
||||||
<h2 style="margin-left: 10px; font-size: 2rem; font-weight: 300">Winrates</h2>
|
|
||||||
<TierlistChart id="chart-winrate" :data="tiers" metric="winrate" />
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
#chart-pickrate,
|
#chart {
|
||||||
#chart-winrate {
|
|
||||||
margin-left: 100px;
|
margin-left: 100px;
|
||||||
margin-right: 100px;
|
margin-right: 100px;
|
||||||
margin-bottom: 100px;
|
margin-bottom: 100px;
|
||||||
@@ -132,8 +125,7 @@ tiers.push({ title: 'F', data: tierFromScaledPickrate(0, 0.1) })
|
|||||||
}
|
}
|
||||||
|
|
||||||
@media only screen and (max-width: 450px) {
|
@media only screen and (max-width: 450px) {
|
||||||
#chart-pickrate,
|
#chart {
|
||||||
#chart-winrate {
|
|
||||||
margin-left: 2px;
|
margin-left: 2px;
|
||||||
margin-right: 12px;
|
margin-right: 12px;
|
||||||
margin-bottom: 40px;
|
margin-bottom: 40px;
|
||||||
|
|||||||
@@ -1,29 +1,14 @@
|
|||||||
import type { MongoClient } from 'mongodb'
|
import type { MongoClient } from 'mongodb'
|
||||||
import { connectToDatabase, fetchLatestPatch, getAvailablePlatforms } from '../utils/mongo'
|
import { connectToDatabase, fetchLatestPatch, getAvailablePlatforms } from '../utils/mongo'
|
||||||
|
|
||||||
interface StatsDocument {
|
|
||||||
patch: string
|
|
||||||
total: number
|
|
||||||
platforms: Record<string, number>
|
|
||||||
updatedAt: Date
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchGameCount(client: MongoClient, patch: string) {
|
async function fetchGameCount(client: MongoClient, patch: string) {
|
||||||
const database = client.db('matches')
|
const database = client.db('matches')
|
||||||
const statsCollection = database.collection<StatsDocument>('stats')
|
|
||||||
|
|
||||||
// Try to get stats from the pre-computed stats collection
|
// Check for platform-specific collections
|
||||||
const stats = await statsCollection.findOne({ patch })
|
|
||||||
|
|
||||||
if (stats) {
|
|
||||||
return { total: stats.total, platforms: stats.platforms }
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback: compute stats from collections if stats document doesn't exist
|
|
||||||
// This handles the migration case where stats weren't pre-computed
|
|
||||||
const platforms = await getAvailablePlatforms(client, patch)
|
const platforms = await getAvailablePlatforms(client, patch)
|
||||||
|
|
||||||
if (platforms.length > 0) {
|
if (platforms.length > 0) {
|
||||||
|
// Sum counts from all platform-specific collections
|
||||||
let totalCount = 0
|
let totalCount = 0
|
||||||
const platformCounts: Record<string, number> = {}
|
const platformCounts: Record<string, number> = {}
|
||||||
|
|
||||||
|
|||||||
@@ -20,14 +20,9 @@ RUN npm install
|
|||||||
COPY --chown=node:node match_collector/. .
|
COPY --chown=node:node match_collector/. .
|
||||||
|
|
||||||
FROM node:current-alpine
|
FROM node:current-alpine
|
||||||
# Install su-exec for dropping privileges
|
|
||||||
RUN apk add --no-cache su-exec
|
|
||||||
RUN mkdir -p /home/node/app && chown -R node:node /home/node/app
|
RUN mkdir -p /home/node/app && chown -R node:node /home/node/app
|
||||||
WORKDIR /home/node/app
|
WORKDIR /home/node/app
|
||||||
|
USER node
|
||||||
COPY --from=build --chown=node:node /home/node/app/match_collector/node_modules ./node_modules
|
COPY --from=build --chown=node:node /home/node/app/match_collector/node_modules ./node_modules
|
||||||
COPY --from=build --chown=node:node /home/node/app/match_collector/. .
|
COPY --from=build --chown=node:node /home/node/app/match_collector/. .
|
||||||
COPY --chown=node:node match_collector/docker-entrypoint.sh /usr/local/bin/
|
CMD ["/bin/sh", "-c", "node --import=tsx src/index.ts; sleep 20h"]
|
||||||
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
|
|
||||||
# Run entrypoint as root to fix permissions, then drop to node user
|
|
||||||
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
|
||||||
CMD ["/bin/sh", "-c", "node --import=tsx src/index.ts; sleep 12h"]
|
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# Fix permissions on the cdragon cache directory if it exists
|
|
||||||
if [ -d "/cdragon" ]; then
|
|
||||||
# Ensure the node user owns the cdragon directory
|
|
||||||
chown -R node:node /cdragon 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Execute the main command as the node user
|
|
||||||
exec su-exec node "$@"
|
|
||||||
@@ -219,10 +219,17 @@ function handleMatchBuilds(
|
|||||||
}
|
}
|
||||||
if (event.type != 'ITEM_PURCHASED') continue
|
if (event.type != 'ITEM_PURCHASED') continue
|
||||||
|
|
||||||
|
// Handle boots upgrades
|
||||||
|
if (
|
||||||
|
itemInfo.requiredBuffCurrencyName == 'Feats_NoxianBootPurchaseBuff' ||
|
||||||
|
itemInfo.requiredBuffCurrencyName == 'Feats_SpecialQuestBootBuff'
|
||||||
|
) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
// Handle boots differently
|
// Handle boots differently
|
||||||
if (itemInfo.categories.includes('Boots')) {
|
if (itemInfo.categories.includes('Boots')) {
|
||||||
// Ignore basic boots, only count Tier 2 boots
|
if (itemInfo.to.length == 0 || (itemInfo.to[0] >= 3171 && itemInfo.to[0] <= 3176)) {
|
||||||
if (event.itemId != 1001) {
|
|
||||||
// Check for bootsFirst
|
// Check for bootsFirst
|
||||||
if (items.length < 2) {
|
if (items.length < 2) {
|
||||||
build.bootsFirstCount += 1
|
build.bootsFirstCount += 1
|
||||||
@@ -391,35 +398,16 @@ async function handleMatchList(
|
|||||||
const database = client.db('matches')
|
const database = client.db('matches')
|
||||||
const collectionName = platform ? `${patch}_${platform}` : patch
|
const collectionName = platform ? `${patch}_${platform}` : patch
|
||||||
const matches = database.collection(collectionName)
|
const matches = database.collection(collectionName)
|
||||||
|
const allMatches = matches.find()
|
||||||
const totalMatches: number = await matches.countDocuments()
|
const totalMatches: number = await matches.countDocuments()
|
||||||
|
|
||||||
// Process matches in batches to limit memory usage
|
|
||||||
const BATCH_SIZE = 1000
|
|
||||||
let currentMatch = 0
|
let currentMatch = 0
|
||||||
let processedInBatch = 0
|
for await (const match of allMatches) {
|
||||||
|
process.stdout.write(
|
||||||
// Use cursor with batch size to limit memory consumption
|
'\rComputing champion stats, game entry ' + currentMatch + '/' + totalMatches + ' ... '
|
||||||
const cursor = matches.find().batchSize(BATCH_SIZE)
|
)
|
||||||
|
currentMatch += 1
|
||||||
try {
|
handleMatch(match as unknown as Match, champions, platform)
|
||||||
for await (const match of cursor) {
|
|
||||||
process.stdout.write(
|
|
||||||
'\rComputing champion stats, game entry ' + currentMatch + '/' + totalMatches + ' ... '
|
|
||||||
)
|
|
||||||
currentMatch += 1
|
|
||||||
processedInBatch += 1
|
|
||||||
handleMatch(match as unknown as Match, champions, platform)
|
|
||||||
|
|
||||||
// Periodically yield to allow garbage collection and log progress
|
|
||||||
if (processedInBatch >= BATCH_SIZE) {
|
|
||||||
processedInBatch = 0
|
|
||||||
// Small delay to allow garbage collection
|
|
||||||
await new Promise(resolve => setImmediate(resolve))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
// Ensure cursor is closed
|
|
||||||
await cursor.close()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return totalMatches
|
return totalMatches
|
||||||
@@ -707,8 +695,8 @@ async function finalizeChampionStats(champion: ChampionData, totalMatches: numbe
|
|||||||
// Sort matchups by score (games * winrate) in descending order
|
// Sort matchups by score (games * winrate) in descending order
|
||||||
for (const lane of champion.lanes) {
|
for (const lane of champion.lanes) {
|
||||||
if (lane.matchups && lane.matchups.length > 0) {
|
if (lane.matchups && lane.matchups.length > 0) {
|
||||||
// Filter out matchups with insufficient games (minimum 30 games to avoid statistical bias)
|
// Filter out matchups with insufficient games (minimum 5 games)
|
||||||
const filteredMatchups = lane.matchups.filter(m => m.games >= 30)
|
const filteredMatchups = lane.matchups.filter(m => m.games >= 5)
|
||||||
|
|
||||||
// Sort by score (games * (winrate - 0.5)^2) descending
|
// Sort by score (games * (winrate - 0.5)^2) descending
|
||||||
filteredMatchups.sort((a, b) => {
|
filteredMatchups.sort((a, b) => {
|
||||||
@@ -755,48 +743,6 @@ async function championList() {
|
|||||||
return list.slice(1)
|
return list.slice(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Compact matches collections to release memory back to the OS.
|
|
||||||
* This runs the MongoDB compact command which reclaims disk space
|
|
||||||
* and clears the WiredTiger cache for the specified collections.
|
|
||||||
*/
|
|
||||||
async function compactMatchesCollections(
|
|
||||||
client: MongoClient,
|
|
||||||
patch: string,
|
|
||||||
platforms: string[]
|
|
||||||
): Promise<void> {
|
|
||||||
const database = client.db('matches')
|
|
||||||
|
|
||||||
console.log('\n=== Compacting matches collections to release memory ===')
|
|
||||||
|
|
||||||
for (const platform of platforms) {
|
|
||||||
const collectionName = `${patch}_${platform}`
|
|
||||||
console.log(`Compacting collection: ${collectionName}...`)
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Run compact command to release memory and defragment
|
|
||||||
// This forces MongoDB to release WiredTiger cache for this collection
|
|
||||||
// Note: compact must be run on the database that contains the collection
|
|
||||||
const result = await database.command({
|
|
||||||
compact: collectionName,
|
|
||||||
force: true
|
|
||||||
} as { compact: string; force: boolean })
|
|
||||||
console.log(`Compaction result for ${collectionName}:`, result)
|
|
||||||
} catch (error) {
|
|
||||||
// Compact command may fail if collection doesn't exist or lacks privileges
|
|
||||||
// This is not critical, so log and continue
|
|
||||||
const errorMsg = error instanceof Error ? error.message : String(error)
|
|
||||||
if (errorMsg.includes('NamespaceNotFound')) {
|
|
||||||
console.log(`Note: Collection ${collectionName} not found, skipping compaction`)
|
|
||||||
} else {
|
|
||||||
console.log(`Note: Could not compact ${collectionName}:`, errorMsg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('Compaction complete.')
|
|
||||||
}
|
|
||||||
|
|
||||||
async function makeChampionsStats(client: MongoClient, patch: string, platforms: string[] = []) {
|
async function makeChampionsStats(client: MongoClient, patch: string, platforms: string[] = []) {
|
||||||
const globalItems = await itemList()
|
const globalItems = await itemList()
|
||||||
for (const item of globalItems) {
|
for (const item of globalItems) {
|
||||||
@@ -827,12 +773,6 @@ async function makeChampionsStats(client: MongoClient, patch: string, platforms:
|
|||||||
const platformMatches = await handleMatchList(client, patch, champions, platform)
|
const platformMatches = await handleMatchList(client, patch, champions, platform)
|
||||||
totalMatches += platformMatches
|
totalMatches += platformMatches
|
||||||
console.log(`Processed ${platformMatches} matches from ${platform}`)
|
console.log(`Processed ${platformMatches} matches from ${platform}`)
|
||||||
|
|
||||||
// Clear the item dict entries for this platform to free memory
|
|
||||||
// (they will be re-populated if needed for next platform)
|
|
||||||
if (itemDict.size > 0) {
|
|
||||||
console.log(`Clearing item cache to free memory...`)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`\n=== Total matches processed: ${totalMatches} ===`)
|
console.log(`\n=== Total matches processed: ${totalMatches} ===`)
|
||||||
@@ -848,9 +788,6 @@ async function makeChampionsStats(client: MongoClient, patch: string, platforms:
|
|||||||
// Create alias-index for better key-find
|
// Create alias-index for better key-find
|
||||||
await collection.createIndex({ alias: 1 })
|
await collection.createIndex({ alias: 1 })
|
||||||
console.log(`Stats saved to collection: ${patch}`)
|
console.log(`Stats saved to collection: ${patch}`)
|
||||||
|
|
||||||
// Compact matches collections to release memory back to the OS
|
|
||||||
await compactMatchesCollections(client, patch, platforms)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export default { makeChampionsStats }
|
export default { makeChampionsStats }
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ const sleep_minutes = 12
|
|||||||
import { MongoClient } from 'mongodb'
|
import { MongoClient } from 'mongodb'
|
||||||
|
|
||||||
import champion_stat from './champion_stat'
|
import champion_stat from './champion_stat'
|
||||||
import stats from './stats'
|
|
||||||
import { Match } from './api'
|
import { Match } from './api'
|
||||||
import { PLATFORMS, getPlatformBaseUrl, getRegionalBaseUrl, getRegionForPlatform } from './platform'
|
import { PLATFORMS, getPlatformBaseUrl, getRegionalBaseUrl, getRegionForPlatform } from './platform'
|
||||||
import { downloadCDragonAssets } from './cdragon_cache'
|
import { downloadCDragonAssets } from './cdragon_cache'
|
||||||
@@ -28,14 +27,6 @@ function extractPatchFromGameVersion(gameVersion: string): string {
|
|||||||
* Collections are named like "15.1_EUW1", "15.2_NA1", etc.
|
* Collections are named like "15.1_EUW1", "15.2_NA1", etc.
|
||||||
*/
|
*/
|
||||||
async function getLatestPatchFromCollections(client: MongoClient): Promise<string | null> {
|
async function getLatestPatchFromCollections(client: MongoClient): Promise<string | null> {
|
||||||
const patches = await getAllPatchesFromCollections(client)
|
|
||||||
return patches.length > 0 ? patches[0] : null
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all patches from existing match collections, sorted from latest to oldest.
|
|
||||||
*/
|
|
||||||
async function getAllPatchesFromCollections(client: MongoClient): Promise<string[]> {
|
|
||||||
const matchesDb = client.db('matches')
|
const matchesDb = client.db('matches')
|
||||||
const collections = await matchesDb.listCollections().toArray()
|
const collections = await matchesDb.listCollections().toArray()
|
||||||
const collectionNames = collections.map(c => c.name)
|
const collectionNames = collections.map(c => c.name)
|
||||||
@@ -51,10 +42,10 @@ async function getAllPatchesFromCollections(client: MongoClient): Promise<string
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (patches.size === 0) {
|
if (patches.size === 0) {
|
||||||
return []
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sort patches from latest to oldest (highest version number first)
|
// Sort patches and return the latest (highest version number)
|
||||||
const sortedPatches = Array.from(patches).sort((a, b) => {
|
const sortedPatches = Array.from(patches).sort((a, b) => {
|
||||||
const [aMajor, aMinor] = a.split('.').map(Number)
|
const [aMajor, aMinor] = a.split('.').map(Number)
|
||||||
const [bMajor, bMinor] = b.split('.').map(Number)
|
const [bMajor, bMinor] = b.split('.').map(Number)
|
||||||
@@ -62,47 +53,7 @@ async function getAllPatchesFromCollections(client: MongoClient): Promise<string
|
|||||||
return bMinor - aMinor
|
return bMinor - aMinor
|
||||||
})
|
})
|
||||||
|
|
||||||
return sortedPatches
|
return sortedPatches[0]
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Clean up match collections that are more than 2 patches old.
|
|
||||||
* This helps reduce database storage by removing outdated match data.
|
|
||||||
* @param client MongoDB client
|
|
||||||
* @param keepPatches Number of recent patches to keep (default: 2)
|
|
||||||
*/
|
|
||||||
async function cleanupOldPatches(client: MongoClient, keepPatches: number): Promise<void> {
|
|
||||||
const matchesDb = client.db('matches')
|
|
||||||
const allPatches = await getAllPatchesFromCollections(client)
|
|
||||||
|
|
||||||
if (allPatches.length <= keepPatches) {
|
|
||||||
console.log(`Cleanup: Only ${allPatches.length} patch(es) found, nothing to clean up.`)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get patches to remove (everything after the first `keepPatches` patches)
|
|
||||||
const patchesToRemove = allPatches.slice(keepPatches)
|
|
||||||
console.log(`Cleanup: Found ${allPatches.length} patches, keeping ${keepPatches} most recent.`)
|
|
||||||
console.log(`Cleanup: Patches to remove: ${patchesToRemove.join(', ')}`)
|
|
||||||
|
|
||||||
// Get all collections to find ones that match patches to remove
|
|
||||||
const collections = await matchesDb.listCollections().toArray()
|
|
||||||
const collectionNames = collections.map(c => c.name)
|
|
||||||
|
|
||||||
let droppedCount = 0
|
|
||||||
for (const collectionName of collectionNames) {
|
|
||||||
const patch = collectionName.split('_')[0]
|
|
||||||
if (patchesToRemove.includes(patch)) {
|
|
||||||
console.log(`Cleanup: Dropping collection '${collectionName}'...`)
|
|
||||||
await matchesDb.dropCollection(collectionName)
|
|
||||||
droppedCount++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete stats for removed patches
|
|
||||||
await stats.deleteStats(client, patchesToRemove)
|
|
||||||
|
|
||||||
console.log(`Cleanup: Dropped ${droppedCount} collection(s).`)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function main() {
|
async function main() {
|
||||||
@@ -167,10 +118,6 @@ async function main() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clean up old patches (keep only 2 most recent patches)
|
|
||||||
console.log('\n=== Cleaning up old patches ===')
|
|
||||||
await cleanupOldPatches(client, 2)
|
|
||||||
|
|
||||||
// Get the latest patch from collections and generate stats for it
|
// Get the latest patch from collections and generate stats for it
|
||||||
const latestPatch = await getLatestPatchFromCollections(client)
|
const latestPatch = await getLatestPatchFromCollections(client)
|
||||||
if (latestPatch) {
|
if (latestPatch) {
|
||||||
@@ -304,9 +251,6 @@ async function saveMatch(client: MongoClient, match: Match, patch: string, platf
|
|||||||
const collectionName = `${patch}_${platform}`
|
const collectionName = `${patch}_${platform}`
|
||||||
const matches = database.collection(collectionName)
|
const matches = database.collection(collectionName)
|
||||||
await matches.insertOne(match)
|
await matches.insertOne(match)
|
||||||
|
|
||||||
// Increment stats counter for this patch/platform
|
|
||||||
await stats.incrementMatchCount(client, patch, platform)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -355,11 +299,6 @@ async function runWithPreloadedData() {
|
|||||||
.filter(name => name.startsWith(`${latestPatch}_`))
|
.filter(name => name.startsWith(`${latestPatch}_`))
|
||||||
.map(name => name.replace(`${latestPatch}_`, ''))
|
.map(name => name.replace(`${latestPatch}_`, ''))
|
||||||
|
|
||||||
// Recalculate match count stats from existing collections
|
|
||||||
if (platforms.length > 0) {
|
|
||||||
await stats.recalculateStats(client, latestPatch, platforms)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate stats for each platform
|
// Generate stats for each platform
|
||||||
if (platforms.length > 0) {
|
if (platforms.length > 0) {
|
||||||
await champion_stat.makeChampionsStats(client, latestPatch, platforms)
|
await champion_stat.makeChampionsStats(client, latestPatch, platforms)
|
||||||
|
|||||||
@@ -1,152 +0,0 @@
|
|||||||
import { MongoClient } from 'mongodb'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Stats document structure stored in the 'stats' collection.
|
|
||||||
* One document per patch, containing total and per-platform match counts.
|
|
||||||
*/
|
|
||||||
interface StatsDocument {
|
|
||||||
patch: string
|
|
||||||
total: number
|
|
||||||
platforms: Record<string, number>
|
|
||||||
updatedAt: Date
|
|
||||||
}
|
|
||||||
|
|
||||||
const STATS_COLLECTION = 'stats'
|
|
||||||
const STATS_DATABASE = 'matches'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize stats document for a patch if it doesn't exist.
|
|
||||||
* This should be called before processing matches for a new patch.
|
|
||||||
*/
|
|
||||||
async function initStats(client: MongoClient, patch: string): Promise<void> {
|
|
||||||
const database = client.db(STATS_DATABASE)
|
|
||||||
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
|
|
||||||
|
|
||||||
await collection.updateOne(
|
|
||||||
{ patch },
|
|
||||||
{
|
|
||||||
$setOnInsert: {
|
|
||||||
patch,
|
|
||||||
total: 0,
|
|
||||||
platforms: {},
|
|
||||||
updatedAt: new Date()
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{ upsert: true }
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Increment match count for a specific patch and platform.
|
|
||||||
* This should be called each time a new match is saved.
|
|
||||||
*/
|
|
||||||
async function incrementMatchCount(
|
|
||||||
client: MongoClient,
|
|
||||||
patch: string,
|
|
||||||
platform: string
|
|
||||||
): Promise<void> {
|
|
||||||
const database = client.db(STATS_DATABASE)
|
|
||||||
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
|
|
||||||
|
|
||||||
await collection.updateOne(
|
|
||||||
{ patch },
|
|
||||||
{
|
|
||||||
$inc: { total: 1, [`platforms.${platform}`]: 1 },
|
|
||||||
$set: { updatedAt: new Date() }
|
|
||||||
},
|
|
||||||
{ upsert: true }
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get stats for a specific patch.
|
|
||||||
* Returns null if no stats exist for the patch.
|
|
||||||
*/
|
|
||||||
async function getStats(client: MongoClient, patch: string): Promise<StatsDocument | null> {
|
|
||||||
const database = client.db(STATS_DATABASE)
|
|
||||||
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
|
|
||||||
|
|
||||||
return await collection.findOne({ patch })
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get stats for all patches, sorted from latest to oldest.
|
|
||||||
*/
|
|
||||||
async function getAllStats(client: MongoClient): Promise<StatsDocument[]> {
|
|
||||||
const database = client.db(STATS_DATABASE)
|
|
||||||
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
|
|
||||||
|
|
||||||
const stats = await collection.find({}).toArray()
|
|
||||||
|
|
||||||
// Sort patches from latest to oldest
|
|
||||||
return stats.sort((a, b) => {
|
|
||||||
const [aMajor, aMinor] = a.patch.split('.').map(Number)
|
|
||||||
const [bMajor, bMinor] = b.patch.split('.').map(Number)
|
|
||||||
if (aMajor !== bMajor) return bMajor - aMajor
|
|
||||||
return bMinor - aMinor
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete stats for patches that are being cleaned up.
|
|
||||||
* This should be called when old patch collections are dropped.
|
|
||||||
*/
|
|
||||||
async function deleteStats(client: MongoClient, patches: string[]): Promise<void> {
|
|
||||||
if (patches.length === 0) return
|
|
||||||
|
|
||||||
const database = client.db(STATS_DATABASE)
|
|
||||||
const collection = database.collection<StatsDocument>(STATS_COLLECTION)
|
|
||||||
|
|
||||||
await collection.deleteMany({ patch: { $in: patches } })
|
|
||||||
console.log(`Stats: Deleted stats for patches: ${patches.join(', ')}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Recalculate stats from existing match collections.
|
|
||||||
* This is useful for migration or fixing inconsistent stats.
|
|
||||||
*/
|
|
||||||
async function recalculateStats(
|
|
||||||
client: MongoClient,
|
|
||||||
patch: string,
|
|
||||||
platforms: string[]
|
|
||||||
): Promise<void> {
|
|
||||||
const database = client.db(STATS_DATABASE)
|
|
||||||
|
|
||||||
let total = 0
|
|
||||||
const platformCounts: Record<string, number> = {}
|
|
||||||
|
|
||||||
for (const platform of platforms) {
|
|
||||||
const collectionName = `${patch}_${platform}`
|
|
||||||
const collection = database.collection(collectionName)
|
|
||||||
const count = await collection.countDocuments()
|
|
||||||
platformCounts[platform] = count
|
|
||||||
total += count
|
|
||||||
}
|
|
||||||
|
|
||||||
const statsCollection = database.collection<StatsDocument>(STATS_COLLECTION)
|
|
||||||
await statsCollection.updateOne(
|
|
||||||
{ patch },
|
|
||||||
{
|
|
||||||
$set: {
|
|
||||||
patch,
|
|
||||||
total,
|
|
||||||
platforms: platformCounts,
|
|
||||||
updatedAt: new Date()
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{ upsert: true }
|
|
||||||
)
|
|
||||||
|
|
||||||
console.log(`Stats: Recalculated stats for patch ${patch}: ${total} total matches`)
|
|
||||||
}
|
|
||||||
|
|
||||||
export default {
|
|
||||||
initStats,
|
|
||||||
incrementMatchCount,
|
|
||||||
getStats,
|
|
||||||
getAllStats,
|
|
||||||
deleteStats,
|
|
||||||
recalculateStats
|
|
||||||
}
|
|
||||||
|
|
||||||
export type { StatsDocument }
|
|
||||||
Reference in New Issue
Block a user