371 lines
13 KiB
TypeScript
371 lines
13 KiB
TypeScript
const api_key = process.env.RIOT_API_KEY
|
|
const sleep_minutes = 12
|
|
|
|
import { MongoClient } from 'mongodb'
|
|
|
|
import champion_stat from './champion_stat'
|
|
import { Match } from './api'
|
|
import { PLATFORMS, getPlatformBaseUrl, getRegionalBaseUrl, getRegionForPlatform } from './platform'
|
|
import { downloadCDragonAssets } from './cdragon_cache'
|
|
|
|
main()
|
|
|
|
/**
|
|
* Extract patch version from gameVersion string.
|
|
* gameVersion format is like "15.1.123.4567" -> we want "15.1"
|
|
*/
|
|
function extractPatchFromGameVersion(gameVersion: string): string {
|
|
const parts = gameVersion.split('.')
|
|
if (parts.length >= 2) {
|
|
return `${parts[0]}.${parts[1]}`
|
|
}
|
|
return gameVersion
|
|
}
|
|
|
|
/**
|
|
* Get the latest patch from existing match collections in the database.
|
|
* Collections are named like "15.1_EUW1", "15.2_NA1", etc.
|
|
*/
|
|
async function getLatestPatchFromCollections(client: MongoClient): Promise<string | null> {
|
|
const patches = await getAllPatchesFromCollections(client)
|
|
return patches.length > 0 ? patches[0] : null
|
|
}
|
|
|
|
/**
|
|
* Get all patches from existing match collections, sorted from latest to oldest.
|
|
*/
|
|
async function getAllPatchesFromCollections(client: MongoClient): Promise<string[]> {
|
|
const matchesDb = client.db('matches')
|
|
const collections = await matchesDb.listCollections().toArray()
|
|
const collectionNames = collections.map(c => c.name)
|
|
|
|
// Extract unique patch versions from collection names
|
|
const patches = new Set<string>()
|
|
for (const name of collectionNames) {
|
|
// Collection names are either "patch_platform" or just "patch"
|
|
const patch = name.split('_')[0]
|
|
if (patch && /^\d+\.\d+$/.test(patch)) {
|
|
patches.add(patch)
|
|
}
|
|
}
|
|
|
|
if (patches.size === 0) {
|
|
return []
|
|
}
|
|
|
|
// Sort patches from latest to oldest (highest version number first)
|
|
const sortedPatches = Array.from(patches).sort((a, b) => {
|
|
const [aMajor, aMinor] = a.split('.').map(Number)
|
|
const [bMajor, bMinor] = b.split('.').map(Number)
|
|
if (aMajor !== bMajor) return bMajor - aMajor
|
|
return bMinor - aMinor
|
|
})
|
|
|
|
return sortedPatches
|
|
}
|
|
|
|
/**
|
|
* Clean up match collections that are more than 2 patches old.
|
|
* This helps reduce database storage by removing outdated match data.
|
|
* @param client MongoDB client
|
|
* @param keepPatches Number of recent patches to keep (default: 2)
|
|
*/
|
|
async function cleanupOldPatches(client: MongoClient, keepPatches: number): Promise<void> {
|
|
const matchesDb = client.db('matches')
|
|
const allPatches = await getAllPatchesFromCollections(client)
|
|
|
|
if (allPatches.length <= keepPatches) {
|
|
console.log(`Cleanup: Only ${allPatches.length} patch(es) found, nothing to clean up.`)
|
|
return
|
|
}
|
|
|
|
// Get patches to remove (everything after the first `keepPatches` patches)
|
|
const patchesToRemove = allPatches.slice(keepPatches)
|
|
console.log(`Cleanup: Found ${allPatches.length} patches, keeping ${keepPatches} most recent.`)
|
|
console.log(`Cleanup: Patches to remove: ${patchesToRemove.join(', ')}`)
|
|
|
|
// Get all collections to find ones that match patches to remove
|
|
const collections = await matchesDb.listCollections().toArray()
|
|
const collectionNames = collections.map(c => c.name)
|
|
|
|
let droppedCount = 0
|
|
for (const collectionName of collectionNames) {
|
|
const patch = collectionName.split('_')[0]
|
|
if (patchesToRemove.includes(patch)) {
|
|
console.log(`Cleanup: Dropping collection '${collectionName}'...`)
|
|
await matchesDb.dropCollection(collectionName)
|
|
droppedCount++
|
|
}
|
|
}
|
|
|
|
console.log(`Cleanup: Dropped ${droppedCount} collection(s).`)
|
|
}
|
|
|
|
async function main() {
|
|
// Check if we're in development mode with pre-loaded data
|
|
if (process.env.NODE_ENV === 'development' && process.env.USE_IMPORTED_DATA === 'true') {
|
|
console.log('MatchCollector - Development mode with pre-loaded data')
|
|
await runWithPreloadedData()
|
|
return
|
|
}
|
|
|
|
// Production mode: collect matches and organize by their gameVersion
|
|
console.log('MatchCollector - Hello !')
|
|
const client = await connectToDatabase()
|
|
console.log('Connected to database')
|
|
|
|
console.log('Using RIOT_API_KEY: ' + api_key)
|
|
if (api_key != null && api_key != undefined && api_key != '') {
|
|
// Iterate through all platforms
|
|
for (const [platform, region] of Object.entries(PLATFORMS)) {
|
|
console.log(`\n=== Processing platform: ${platform} (region: ${region}) ===`)
|
|
|
|
// Get already seen games for all patches (we'll check by gameVersion when saving)
|
|
const alreadySeenGameList = await alreadySeenGamesAllPatches(client, platform)
|
|
console.log('We already have ' + alreadySeenGameList.length + ' matches for this platform !')
|
|
|
|
const challengerLeague = await fetchChallengerLeague(platform)
|
|
console.log('ChallengerLeague: got ' + challengerLeague.entries.length + ' entries')
|
|
|
|
// Use 30 days ago as start time for collecting matches
|
|
const startTime = Math.floor(Date.now() / 1000) - 30 * 24 * 60 * 60
|
|
|
|
const gameList: string[] = []
|
|
let i = 0
|
|
for (const challenger of challengerLeague.entries) {
|
|
console.log('Entry ' + i + '/' + challengerLeague.entries.length + ' ...')
|
|
const puuid = challenger.puuid
|
|
const challengerGameList = await summonerGameList(puuid, startTime, region)
|
|
for (const game of challengerGameList) {
|
|
if (!gameList.includes(game) && !alreadySeenGameList.includes(game)) {
|
|
gameList.push(game)
|
|
}
|
|
}
|
|
i++
|
|
}
|
|
|
|
console.log('Games: got ' + gameList.length + ' entries for ' + platform)
|
|
i = 0
|
|
for (const game of gameList) {
|
|
console.log('Entry ' + i + '/' + gameList.length + ' ...')
|
|
// Determine region from matchId (format: PLATFORM_matchId)
|
|
// Map platform prefix to regional routing value for match API
|
|
const matchPlatformPrefix = game.split('_')[0]
|
|
const matchRegion = getRegionForPlatform(matchPlatformPrefix) || region
|
|
const gameMatch = await match(game, matchRegion)
|
|
const gameTimeline = await matchTimeline(game, matchRegion)
|
|
gameMatch.timeline = gameTimeline
|
|
// Extract patch from gameVersion and save to appropriate collection
|
|
const patch = extractPatchFromGameVersion(gameMatch.info.gameVersion)
|
|
await saveMatch(client, gameMatch, patch, platform)
|
|
i++
|
|
}
|
|
}
|
|
}
|
|
|
|
// Clean up old patches (keep only 2 most recent patches)
|
|
console.log('\n=== Cleaning up old patches ===')
|
|
await cleanupOldPatches(client, 2)
|
|
|
|
// Get the latest patch from collections and generate stats for it
|
|
const latestPatch = await getLatestPatchFromCollections(client)
|
|
if (latestPatch) {
|
|
console.log(`Generating stats for latest patch: ${latestPatch}...`)
|
|
await champion_stat.makeChampionsStats(client, latestPatch, Object.keys(PLATFORMS))
|
|
|
|
// Download CDragon assets for the latest patch
|
|
await downloadCDragonAssets(latestPatch)
|
|
} else {
|
|
console.log('No matches found in database, skipping stat generation')
|
|
}
|
|
|
|
console.log('All done. Closing client.')
|
|
await client.close()
|
|
}
|
|
|
|
async function handleRateLimit(url: URL): Promise<Response> {
|
|
let response = await fetch(url)
|
|
if (response.status == 429) {
|
|
await new Promise(resolve => setTimeout(resolve, sleep_minutes * 60 * 1000))
|
|
response = await handleRateLimit(url)
|
|
}
|
|
|
|
return response
|
|
}
|
|
|
|
function handleError(response: Response) {
|
|
if (!response.ok) {
|
|
console.log(
|
|
'Error during fetch(' +
|
|
response.url +
|
|
'): STATUS ' +
|
|
response.status +
|
|
' (' +
|
|
response.statusText +
|
|
')'
|
|
)
|
|
process.exit(1)
|
|
}
|
|
}
|
|
|
|
async function connectToDatabase() {
|
|
// Create a MongoClient with a MongoClientOptions object to set the Stable API version
|
|
let uri = `mongodb://${process.env.MONGO_USER}:${process.env.MONGO_PASS}@${process.env.MONGO_HOST}`
|
|
if (
|
|
process.env.MONGO_URI != undefined &&
|
|
process.env.MONGO_URI != null &&
|
|
process.env.MONGO_URI != ''
|
|
) {
|
|
uri = process.env.MONGO_URI
|
|
}
|
|
const client = new MongoClient(uri)
|
|
await client.connect()
|
|
return client
|
|
}
|
|
|
|
async function fetchChallengerLeague(platform: string) {
|
|
const queue = 'RANKED_SOLO_5x5'
|
|
const endpoint = `/lol/league/v4/challengerleagues/by-queue/${queue}`
|
|
const baseUrl = getPlatformBaseUrl(platform)
|
|
const url = `${baseUrl}${endpoint}?api_key=${api_key}`
|
|
|
|
const challengerLeagueResponse = await handleRateLimit(new URL(url))
|
|
|
|
handleError(challengerLeagueResponse)
|
|
|
|
const challengerLeague = await challengerLeagueResponse.json()
|
|
return challengerLeague
|
|
}
|
|
|
|
async function summonerGameList(puuid: string, startTime: number, region: string) {
|
|
const baseUrl = getRegionalBaseUrl(region)
|
|
const endpoint = `/lol/match/v5/matches/by-puuid/${puuid}/ids`
|
|
const url = `${baseUrl}${endpoint}?queue=420&type=ranked&startTime=${startTime}&api_key=${api_key}`
|
|
|
|
const gameListResponse = await handleRateLimit(new URL(url))
|
|
handleError(gameListResponse)
|
|
const gameList = await gameListResponse.json()
|
|
|
|
return gameList
|
|
}
|
|
|
|
async function match(matchId: string, region: string) {
|
|
const baseUrl = getRegionalBaseUrl(region)
|
|
const endpoint = `/lol/match/v5/matches/${matchId}`
|
|
const url = `${baseUrl}${endpoint}?api_key=${api_key}`
|
|
|
|
const matchResponse = await handleRateLimit(new URL(url))
|
|
handleError(matchResponse)
|
|
const match = await matchResponse.json()
|
|
|
|
return match
|
|
}
|
|
|
|
async function matchTimeline(matchId: string, region: string) {
|
|
const baseUrl = getRegionalBaseUrl(region)
|
|
const endpoint = `/lol/match/v5/matches/${matchId}/timeline`
|
|
const url = `${baseUrl}${endpoint}?api_key=${api_key}`
|
|
|
|
const timelineResponse = await handleRateLimit(new URL(url))
|
|
handleError(timelineResponse)
|
|
const timeline = await timelineResponse.json()
|
|
|
|
return timeline
|
|
}
|
|
|
|
/**
|
|
* Get already seen games across all patches for a specific platform.
|
|
* This is used when we don't know the patch beforehand (we get it from gameVersion).
|
|
*/
|
|
async function alreadySeenGamesAllPatches(client: MongoClient, platform: string) {
|
|
const matchesDb = client.db('matches')
|
|
const collections = await matchesDb.listCollections().toArray()
|
|
const collectionNames = collections.map(c => c.name)
|
|
|
|
// Find all collections for this platform (format: "patch_platform")
|
|
const platformCollections = collectionNames.filter(name => name.endsWith(`_${platform}`))
|
|
|
|
const allSeen: string[] = []
|
|
for (const collectionName of platformCollections) {
|
|
const matches = matchesDb.collection(collectionName)
|
|
const seen = await matches.distinct('metadata.matchId')
|
|
allSeen.push(...seen)
|
|
}
|
|
|
|
return allSeen
|
|
}
|
|
|
|
async function saveMatch(client: MongoClient, match: Match, patch: string, platform: string) {
|
|
const database = client.db('matches')
|
|
const collectionName = `${patch}_${platform}`
|
|
const matches = database.collection(collectionName)
|
|
await matches.insertOne(match)
|
|
}
|
|
|
|
/**
|
|
* Development mode function that generates stats from pre-loaded data
|
|
*/
|
|
async function runWithPreloadedData() {
|
|
console.log('Using pre-loaded match data for development')
|
|
|
|
const client = await connectToDatabase()
|
|
try {
|
|
// Get the latest patch from collections instead of patches database
|
|
const latestPatch = await getLatestPatchFromCollections(client)
|
|
|
|
if (!latestPatch) {
|
|
console.error('❌ No match data found in database')
|
|
console.log('💡 Please run the data import script first:')
|
|
console.log(' node dev/scripts/setup-db.js')
|
|
return
|
|
}
|
|
|
|
console.log(`Latest patch: ${latestPatch}`)
|
|
|
|
// Check if we have matches for this patch (including platform-specific collections)
|
|
const matchesDb = client.db('matches')
|
|
const collections = await matchesDb.listCollections().toArray()
|
|
const collectionNames = collections.map(c => c.name)
|
|
|
|
// Find collections for this patch (both global and platform-specific)
|
|
const patchCollections = collectionNames.filter(
|
|
name => name === latestPatch || name.startsWith(`${latestPatch}_`)
|
|
)
|
|
|
|
if (patchCollections.length === 0) {
|
|
console.error(`❌ No match data found for patch ${latestPatch}`)
|
|
console.log('💡 Please run the data import script first:')
|
|
console.log(' node dev/scripts/setup-db.js')
|
|
return
|
|
}
|
|
|
|
console.log(
|
|
`Found ${patchCollections.length} match collection(s): ${patchCollections.join(', ')}`
|
|
)
|
|
|
|
// Extract platforms from collection names (e.g., "15.1_EUW1" -> "EUW1")
|
|
const platforms = patchCollections
|
|
.filter(name => name.startsWith(`${latestPatch}_`))
|
|
.map(name => name.replace(`${latestPatch}_`, ''))
|
|
|
|
// Generate stats for each platform
|
|
if (platforms.length > 0) {
|
|
await champion_stat.makeChampionsStats(client, latestPatch, platforms)
|
|
} else {
|
|
// Fallback for old-style collections without platform suffix
|
|
await champion_stat.makeChampionsStats(client, latestPatch)
|
|
}
|
|
|
|
// Download CDragon assets for the latest patch
|
|
await downloadCDragonAssets(latestPatch)
|
|
|
|
console.log('🎉 All stats generated successfully!')
|
|
console.log('🚀 Your development database is ready for frontend testing!')
|
|
} catch (error) {
|
|
console.error('❌ Error in development mode:', error)
|
|
throw error
|
|
} finally {
|
|
await client.close()
|
|
}
|
|
}
|