Merge pull request #106 from wafflestudio/fix/events

feat: sync events with large batch file

Author: aystoe

hangsha/batch/src/main/kotlin/com/team1/hangsha/batch/crawler/ExtraSnuCrawler.kt

@@ -10,6 +10,7 @@ import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 import java.util.concurrent.TimeUnit
 import com.team1.hangsha.common.upload.OciUploadService
@@ -81,19 +82,25 @@ class ExtraSnuCrawler(
             if (!shouldFetch(e)) return@map e
 
             val html1 = fetchDetailPageByPlaywright(dataSeq) ?: return@map e
-            var sessions = parseDetailSessions(html1)
-            var mainHtml = parseMainContentHtml(html1, ociUploadService)
+            var parsed = parseDetailData(
+                html = html1,
+                ociUploadService = ociUploadService,
+                cookieHeader = buildCookieHeader()
+            )
 
-            if (sessions.isEmpty()) { // fallback once
+            if (parsed.sessions.isEmpty()) { // fallback once
                 val html2 = fetchDetailPageByPlaywright(dataSeq)
                 if (html2 != null) {
-                    sessions = parseDetailSessions(html2)
-                    mainHtml = parseMainContentHtml(html2, ociUploadService)
+                    parsed = parseDetailData(
+                        html = html2,
+                        ociUploadService = ociUploadService,
+                        cookieHeader = buildCookieHeader()
+                    )
                 }
             }
 
             if (delayMsBetweenDetails > 0) Thread.sleep(delayMsBetweenDetails)
-            e.copy(detailSessions = sessions, mainContentHtml = mainHtml)
+            e.copy(detailSessions = parsed.sessions, mainContentHtml = parsed.mainContentHtml)
         }
     }
 
@@ -329,9 +336,7 @@ class ExtraSnuCrawler(
     // ---------------------------
     // ✅ 상세 페이지 -> DetailSession (start/end/time 평탄화, n회차, 우측 날짜 생략 처리)
     // ---------------------------
-    private fun parseDetailSessions(html: String): List<DetailSession> {
-        val doc = Jsoup.parse(html, baseUrl)
-
+    private fun parseDetailSessions(doc: Document): List<DetailSession> {
         val table = doc.select("table.table.t_view.add_tr")
             .firstOrNull { it.text().normalize().contains("교육(활동)기간") }
             ?: return emptyList()
@@ -381,18 +386,18 @@ class ExtraSnuCrawler(
      * 반환값은 td_box의 innerHTML.
      * 없으면 null.
      */
-    private fun parseMainContentHtml(html: String, ociUploadService: OciUploadService): String? {
-        val doc = Jsoup.parse(html, baseUrl)
-
+    private fun parseMainContentHtml(
+        doc: Document,
+        ociUploadService: OciUploadService,
+        cookieHeader: String,
+    ): String? {
         val box = doc.select("div.cont_box")
             .firstOrNull {
                 it.selectFirst("p.cont_tit")?.text()?.normalize() == "프로그램 주요내용"
             } ?: return null
 
         val tdBox = box.selectFirst("div.td_box") ?: return null
 
-        val cookieHeader = buildCookieHeader()
-
         tdBox.select("img").forEach { img ->
             val rawSrc = img.absUrl("src").ifBlank {
                 val decoded = Parser.unescapeEntities(img.attr("src"), false)
@@ -449,6 +454,17 @@ class ExtraSnuCrawler(
         return out.ifBlank { null }
     }
 
+    private fun parseDetailData(
+        html: String,
+        ociUploadService: OciUploadService,
+        cookieHeader: String,
+    ): ParsedDetailData {
+        val doc = Jsoup.parse(html, baseUrl)
+        val sessions = parseDetailSessions(doc)
+        val mainContentHtml = parseMainContentHtml(doc, ociUploadService, cookieHeader)
+        return ParsedDetailData(sessions = sessions, mainContentHtml = mainContentHtml)
+    }
+
     // ---------------------------
     // helpers
     // ---------------------------
@@ -659,4 +675,9 @@ class ExtraSnuCrawler(
         val contentType: String,
         val extension: String,
     )
+
+    private data class ParsedDetailData(
+        val sessions: List<DetailSession>,
+        val mainContentHtml: String?,
+    )
 }

hangsha/batch/src/main/kotlin/com/team1/hangsha/batch/job/ExtraSnuSyncRunner.kt

@@ -1,5 +1,6 @@
 package com.team1.hangsha.batch.job
 
+import com.fasterxml.jackson.databind.ObjectMapper
 import com.team1.hangsha.batch.crawler.DetailSession
 import com.team1.hangsha.batch.crawler.ExtraSnuCrawler
 import com.team1.hangsha.batch.crawler.ProgramEvent
@@ -10,12 +11,15 @@ import com.team1.hangsha.event.service.EventSyncService
 import org.springframework.boot.ApplicationArguments
 import org.springframework.boot.ApplicationRunner
 import org.springframework.stereotype.Component
+import java.nio.file.Files
+import java.nio.file.Path
 import kotlin.system.exitProcess
 
 @Component
 class ExtraSnuSyncRunner(
     private val eventSyncService: EventSyncService,
     private val ociUploadService: OciUploadService,
+    private val objectMapper: ObjectMapper,
 ) : ApplicationRunner {
 
     override fun run(args: ApplicationArguments) {
@@ -26,6 +30,7 @@ class ExtraSnuSyncRunner(
         var totalUpserted = 0
         var totalCrawled = 0
         var totalSkipped = 0
+        val dumpBuffer = mutableListOf<CrawledProgramEvent>()
 
         ExtraSnuCrawler(
             delayMsBetweenPages = opt.delayMs,
@@ -48,25 +53,46 @@ class ExtraSnuSyncRunner(
                     crawler.enrichDetails(baseEvents, ociUploadService) // { e -> e.status != "모집마감" } // @TODO: 위의 0001, 0002, ... 와 같이 매직 넘버라, ENUM화?
                 }
 
-                val eventsWithUploadedImages = if (!opt.withDetails) {
-                    events
-                } else {
-                    crawler.uploadEventImages(events, ociUploadService)
+                // dumpOnly 여부와 상관없이 이미지 업로드는 항상 수행한다.
+                val eventsWithUploadedImages = crawler.uploadEventImages(events, ociUploadService)
+
+                val crawledEvents = eventsWithUploadedImages.map { it.toCrawledProgramEvent() }
+                if (opt.outFile != null) {
+                    dumpBuffer += crawledEvents
                 }
 
-                val result = eventSyncService.sync(eventsWithUploadedImages.map { it.toCrawledProgramEvent() })
+                totalCrawled += crawledEvents.size
+                if (opt.dumpOnly) {
+                    println("Page $page crawled: total=${crawledEvents.size}")
+                    continue
+                }
 
+                val result = eventSyncService.sync(crawledEvents)
                 totalUpserted += result.upserted
-                totalCrawled += result.total
                 totalSkipped += result.skipped
 
                 println("Page $page synced: upserted=${result.upserted}, total=${result.total}, skipped=${result.skipped}")
             }
         }
 
-        println("Synced $totalUpserted rows from $totalCrawled crawled events (skipped=$totalSkipped)")
+        if (opt.outFile != null) {
+            writeDumpFile(opt.outFile, dumpBuffer)
+            println("Saved crawled events to ${opt.outFile} (count=${dumpBuffer.size})")
+        }
+
+        if (opt.dumpOnly) {
+            println("Crawled $totalCrawled rows (dump-only mode)")
+        } else {
+            println("Synced $totalUpserted rows from $totalCrawled crawled events (skipped=$totalSkipped)")
+        }
         exitProcess(0)
     }
+
+    private fun writeDumpFile(outFile: String, rows: List<CrawledProgramEvent>) {
+        val path = Path.of(outFile).toAbsolutePath().normalize()
+        path.parent?.let { Files.createDirectories(it) }
+        objectMapper.writerWithDefaultPrettyPrinter().writeValue(path.toFile(), rows)
+    }
 }
 
 private data class BatchArgs(
@@ -75,6 +101,8 @@ private data class BatchArgs(
     val delayMs: Long = 200,
     val withDetails: Boolean = true,
     val detailDelayMs: Long = 100,
+    val outFile: String? = null,
+    val dumpOnly: Boolean = false,
 ) {
     companion object {
         fun from(args: ApplicationArguments): BatchArgs {
@@ -92,6 +120,8 @@ private data class BatchArgs(
                 delayMs = single("delayMs")?.toLong() ?: 200L,
                 withDetails = withDetails,
                 detailDelayMs = single("detailDelayMs")?.toLong() ?: 100L,
+                outFile = single("outFile"),
+                dumpOnly = args.containsOption("dumpOnly"),
             )
         }
     }
@@ -124,4 +154,4 @@ private fun DetailSession.toCrawledDetailSession(): CrawledDetailSession =
         endDate = endDate,
         startTime = startTime,
         endTime = endTime
-    )
+    )