From 0465d19a6b85a75855ba726798f65788dea3babe Mon Sep 17 00:00:00 2001 From: epikaigle444 Date: Fri, 13 Feb 2026 12:15:33 +0100 Subject: [PATCH 1/3] MangaMoins: migrate parser to API-based source --- .../doki/parsers/site/fr/MangaMoins.kt | 507 +++++++++++------- 1 file changed, 302 insertions(+), 205 deletions(-) diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoins.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoins.kt index da0768c..209f239 100644 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoins.kt +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoins.kt @@ -1,7 +1,13 @@ package org.dokiteam.doki.parsers.site.fr +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import org.dokiteam.doki.parsers.MangaLoaderContext import org.dokiteam.doki.parsers.MangaSourceParser +import org.dokiteam.doki.parsers.config.ConfigKey +import org.dokiteam.doki.parsers.core.PagedMangaParser +import org.dokiteam.doki.parsers.exception.ParseException import org.dokiteam.doki.parsers.model.Manga import org.dokiteam.doki.parsers.model.MangaChapter import org.dokiteam.doki.parsers.model.MangaListFilter @@ -9,220 +15,311 @@ import org.dokiteam.doki.parsers.model.MangaListFilterCapabilities import org.dokiteam.doki.parsers.model.MangaListFilterOptions import org.dokiteam.doki.parsers.model.MangaPage import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.MangaSource import org.dokiteam.doki.parsers.model.MangaState import org.dokiteam.doki.parsers.model.RATING_UNKNOWN import org.dokiteam.doki.parsers.model.SortOrder import org.dokiteam.doki.parsers.util.generateUid import org.dokiteam.doki.parsers.util.parseHtml +import org.dokiteam.doki.parsers.util.parseJson import org.dokiteam.doki.parsers.util.toAbsoluteUrl -import org.dokiteam.doki.parsers.config.ConfigKey -import org.dokiteam.doki.parsers.core.SinglePageMangaParser +import org.json.JSONObject +import org.jsoup.nodes.Document +import java.net.URLDecoder +import java.nio.charset.StandardCharsets import java.util.EnumSet import java.util.Locale @MangaSourceParser("MANGAMOINS", "MangaMoins", "fr") internal class MangaMoins(context: MangaLoaderContext) : - SinglePageMangaParser(context, MangaParserSource.MANGAMOINS) { - - override val configKeyDomain = ConfigKey.Domain("mangamoins.com") - override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED) - override val filterCapabilities = MangaListFilterCapabilities() - - override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions() - - override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { - return listOf( - Manga( - id = generateUid("OP"), - title = "One Piece", - altTitles = emptySet(), - url = "OP", - publicUrl = "https://mangamoins.com/", - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = "https://mangamoins.com/files/scans/OP1161/thumbnail.png", - tags = emptySet(), - state = MangaState.ONGOING, - authors = setOf("Eiichiro Oda"), - source = source, - ), - Manga( - id = generateUid("LCDL"), - title = "Les Carnets de l'Apothicaire", - altTitles = emptySet(), - url = "LCDL", - publicUrl = "https://mangamoins.com/", - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = "https://mangamoins.com/files/scans/LCDL76.2/thumbnail.png", - tags = emptySet(), - state = MangaState.ONGOING, - authors = setOf("Itsuki Nanao", "Nekokurage"), - source = source, - ), - Manga( - id = generateUid("JKM"), - title = "Jujutsu Kaisen Modulo", - altTitles = emptySet(), - url = "JKM", - publicUrl = "https://mangamoins.com/", - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = "https://mangamoins.com/files/scans/JKM1/thumbnail.png", - tags = emptySet(), - state = MangaState.ONGOING, - authors = setOf("Gege Akutami", "Yuji Iwasaki"), - source = source, - ), - Manga( - id = generateUid("OPC"), - title = "One Piece Colo", - altTitles = emptySet(), - url = "OPC", - publicUrl = "https://mangamoins.com/", - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = "https://mangamoins.com/files/scans/OPC1160/thumbnail.png", - tags = emptySet(), - state = MangaState.ONGOING, - authors = setOf("Eiichiro Oda"), - source = source, - ), - Manga( - id = generateUid("LDS"), - title = "L'Atelier des Sorciers", - altTitles = emptySet(), - url = "LDS", - publicUrl = "https://mangamoins.com/", - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = "https://sceneario.com/wp-content/uploads/2023/05/9782811641344-1.jpg", - tags = emptySet(), - state = MangaState.ONGOING, - authors = setOf("Kamome Shirahama"), - source = source, - ) - ) - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet("https://$domain").parseHtml() - val prefix = manga.url - val latestSiteChapterNumber = doc.select("div.sortie a").mapNotNull { a -> - val href = a.attr("href") - if (href.startsWith("?scan=$prefix")) { - href.substringAfter(prefix).toFloatOrNull() - } else { - null - } - }.maxOrNull() - - if (latestSiteChapterNumber == null) return manga - - val cachedChapters = manga.chapters - val lastKnownChapterNumber = cachedChapters?.maxByOrNull { it.number }?.number - - if (lastKnownChapterNumber != null && cachedChapters.isNotEmpty()) { - // INCREMENTAL SCAN (CACHE EXISTS) - if (latestSiteChapterNumber <= lastKnownChapterNumber) { - return manga.copy(chapters = cachedChapters.sortedBy { it.number }) - } - - val newChapters = mutableListOf() - var currentChapterInt = latestSiteChapterNumber.toInt() - - while (currentChapterInt > lastKnownChapterNumber.toInt()) { - val chaptersForGroup = doChecks(prefix, currentChapterInt) - newChapters.addAll(chaptersForGroup) - currentChapterInt-- - } - - val combinedChapters = (cachedChapters + newChapters).distinctBy { it.number } - return manga.copy(chapters = combinedChapters.sortedBy { it.number }) - - } else { - // FULL SCAN (NO CACHE) - val allChapters = mutableListOf() - var currentChapterInt = latestSiteChapterNumber.toInt() - var misses = 0 - - while (currentChapterInt >= 1 && misses < 4) { - val chaptersForGroup = doChecks(prefix, currentChapterInt) - if (chaptersForGroup.isNotEmpty()) { - misses = 0 - allChapters.addAll(chaptersForGroup) - } else { - misses++ - } - currentChapterInt-- - } - return manga.copy(chapters = allChapters.sortedBy { it.number }) - } - } - - private suspend fun doChecks(prefix: String, chapterInt: Int): List { - val foundChapters = mutableListOf() - - // Check for integer chapter - val integerChapter = checkChapter(prefix, chapterInt.toString(), chapterInt.toFloat()) - if (integerChapter != null) { - foundChapters.add(integerChapter) - } - - // Conditionally and sequentially check for decimal chapters - if (prefix == "LCDL") { - // Start from 2 because .1 never exists - for (i in 2..9) { - val decimalNum = chapterInt + (i / 10.0f) - val decimalNumStr = String.format(Locale.US, "%.1f", decimalNum) - val decimalChapter = checkChapter(prefix, decimalNumStr, decimalNum) - if (decimalChapter != null) { - foundChapters.add(decimalChapter) - } else { - break // Stop if there's a gap - } - } - } - return foundChapters - } - - private suspend fun checkChapter(prefix: String, chapterNumStr: String, chapterNumFloat: Float): MangaChapter? { - val thumbUrl = "https://mangamoins.com/files/scans/$prefix$chapterNumStr/thumbnail.png" - return try { - val response = webClient.httpHead(thumbUrl) - if (response.isSuccessful) { - response.close() - val chapterUrl = "https://mangamoins.com/?scan=$prefix$chapterNumStr" - MangaChapter( - id = generateUid(chapterUrl), - title = null, - number = chapterNumFloat, - volume = 0, - url = chapterUrl, - scanlator = null, - uploadDate = 0, - branch = null, - source = source - ) - } else { - response.close() - null - } - } catch (_: Exception) { - null - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url).parseHtml() - return doc.select("link[rel=preload][as=image]").map { element -> - val url = element.attr("href").toAbsoluteUrl(domain) - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source - ) - } - } + PagedMangaParser(context, MangaParserSource.MANGAMOINS, API_LIMIT) { + + override val configKeyDomain = ConfigKey.Domain("mangamoins.com") + override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED) + override val filterCapabilities = MangaListFilterCapabilities(isSearchSupported = true) + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .build() + + override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = HttpUrl.Builder() + .scheme("https") + .host(domain) + .addPathSegments("api/v1/mangas") + .addQueryParameter("page", page.toString()) + .addQueryParameter("limit", API_LIMIT.toString()) + .apply { + filter.query?.trim()?.takeIf { it.isNotEmpty() }?.let { q -> + addQueryParameter("q", q) + } + } + .build() + val json = webClient.httpGet(url).parseJson() + if (json.optString("status").equals("error", ignoreCase = true)) { + throw ParseException(json.optString("message"), url.toString()) + } + val data = json.optJSONArray("data") ?: return emptyList() + val list = ArrayList(data.length()) + for (i in 0 until data.length()) { + val jo = data.optJSONObject(i) ?: continue + val title = jo.optString("title").trim() + if (title.isEmpty()) continue + val slug = title.toMangaSlug() + val coverFolder = jo.optString("cover_folder").trim() + val coverUrl = coverFolder.takeIf { it.isNotEmpty() }?.let { + "https://$domain/files/scans/$it/thumbnail.webp" + } + list += Manga( + id = generateUid(slug), + title = title, + altTitles = emptySet(), + url = "/manga/$slug", + publicUrl = "https://$domain/manga/$slug", + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = coverUrl, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + return list + } + + override suspend fun getDetails(manga: Manga): Manga { + val json = fetchMangaJson(manga) + val info = json.optJSONObject("info") + val detailsTitle = info?.optString("title").orEmpty().ifBlank { manga.title } + val chapters = parseChapters(json).ifEmpty { manga.chapters.orEmpty() } + val description = info?.optString("description").orEmpty().trim().ifEmpty { null } + val authors = parseAuthors(info?.optString("author")).ifEmpty { manga.authors } + val coverUrl = info?.optString("cover").orEmpty().trim().ifEmpty { null }?.toAbsoluteUrl(domain) + val canonicalSlug = extractSlugFromMangaUrl(manga.url) + ?: extractSlugFromMangaUrl(manga.publicUrl) + ?: detailsTitle.toMangaSlug() + return manga.copy( + title = detailsTitle, + publicUrl = "https://$domain/manga/$canonicalSlug", + coverUrl = coverUrl ?: manga.coverUrl, + description = description ?: manga.description, + state = parseState(info?.optString("status")) ?: manga.state, + authors = authors, + chapters = chapters.takeIf { it.isNotEmpty() } ?: manga.chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val preloaded = doc.select("link[rel=preload][as=image]") + .map { it.attr("href").trim() } + .filter { it.isNotEmpty() } + .map { it.toAbsoluteUrl(domain) } + .distinct() + if (preloaded.isNotEmpty()) { + return preloaded.map { pageUrl -> + MangaPage( + id = generateUid(pageUrl), + url = pageUrl, + preview = null, + source = chapter.source, + ) + } + } + val folder = extractScanFolder(chapter.url) ?: return emptyList() + return parsePagesFromScript(doc, folder, chapter.source) + } + + private suspend fun fetchMangaJson(manga: Manga): JSONObject { + val candidates = buildDetailsCandidates(manga) + if (candidates.isEmpty()) { + throw ParseException("Cannot build manga lookup candidates", manga.publicUrl) + } + var bestScore = Int.MIN_VALUE + var bestMatch: JSONObject? = null + for (title in candidates) { + val url = HttpUrl.Builder() + .scheme("https") + .host(domain) + .addPathSegments("api/v1/manga") + .addQueryParameter("manga", title) + .build() + val json = webClient.httpGet(url).parseJson() + val score = scoreDetailsResponse(json) + if (score > bestScore) { + bestScore = score + bestMatch = json + } + if (score >= DETAILS_SCORE_WITH_CHAPTERS) { + return json + } + } + return bestMatch ?: throw ParseException("Cannot load manga details", manga.publicUrl) + } + + private fun parseChapters(json: JSONObject): List { + val ja = json.optJSONArray("chapters") ?: return emptyList() + val result = ArrayList(ja.length()) + for (i in ja.length() - 1 downTo 0) { + val jo = ja.optJSONObject(i) ?: continue + val folder = jo.optString("folder").trim() + if (folder.isEmpty()) continue + val chapterTitle = jo.optString("title").trim().ifEmpty { null } + val chapterNumber = parseChapterNumber(jo.optString("num"), folder) + result += MangaChapter( + id = generateUid(folder), + title = chapterTitle, + number = chapterNumber, + volume = 0, + url = "/scan/$folder", + scanlator = null, + uploadDate = jo.optLong("time", 0L) * 1000L, + branch = null, + source = source, + ) + } + return result.distinctBy { it.url } + } + + private fun parsePagesFromScript(doc: Document, folder: String, source: MangaSource): List { + val scriptData = doc.select("script") + .firstOrNull { it.data().contains("imageMtimes") } + ?.data() + .orEmpty() + val payload = IMAGE_MTIMES_BLOCK.find(scriptData)?.groupValues?.getOrNull(1).orEmpty() + if (payload.isEmpty()) return emptyList() + val pairs = IMAGE_MTIMES_ENTRY.findAll(payload) + .mapNotNull { m -> + val page = m.groupValues.getOrNull(1)?.toIntOrNull() ?: return@mapNotNull null + val mtime = m.groupValues.getOrNull(2)?.takeIf { it.isNotEmpty() } ?: return@mapNotNull null + page to mtime + } + .sortedBy { it.first } + .map { (page, mtime) -> + val pageName = page.toString().padStart(2, '0') + "/files/scans/$folder/$pageName.png?v=$mtime" + } + .distinct() + return pairs.map { pageUrl -> + MangaPage( + id = generateUid(pageUrl), + url = pageUrl, + preview = null, + source = source, + ) + }.toList() + } + + private fun parseState(status: String?): MangaState? { + val value = status?.lowercase(Locale.ROOT)?.trim() ?: return null + return when { + "cours" in value -> MangaState.ONGOING + "term" in value || "fini" in value -> MangaState.FINISHED + else -> null + } + } + + private fun parseAuthors(raw: String?): Set { + return raw.orEmpty() + .split(AUTHORS_SPLIT_PATTERN) + .map { it.trim() } + .filter { it.isNotEmpty() && !it.equals("Auteur Inconnu", ignoreCase = true) } + .toSet() + } + + private fun parseChapterNumber(raw: String, folder: String): Float { + val fromLabel = raw.replace("#", "").trim().toFloatOrNull() + if (fromLabel != null) return fromLabel + val fromFolder = CHAPTER_NUMBER_REGEX.find(folder)?.groupValues?.getOrNull(1)?.toFloatOrNull() + return fromFolder ?: 0f + } + + private fun buildDetailsCandidates(manga: Manga): List { + val ordered = listOfNotNull( + extractSlugFromMangaUrl(manga.url)?.toMangaTitle(), + extractSlugFromMangaUrl(manga.publicUrl)?.toMangaTitle(), + legacyMangaTitle(manga.url), + manga.title.trim().takeIf { it.isNotEmpty() }, + ).distinctBy { it.lowercase(Locale.ROOT) } + + return when { + ordered.size <= 1 -> ordered + else -> ordered.filterNot { it.equals(UNKNOWN_MANGA_TITLE, ignoreCase = true) } + } + } + + private fun scoreDetailsResponse(json: JSONObject): Int { + val info = json.optJSONObject("info") ?: return DETAILS_SCORE_EMPTY + val chaptersCount = json.optJSONArray("chapters")?.length() ?: 0 + if (chaptersCount > 0) { + return DETAILS_SCORE_WITH_CHAPTERS + } + val author = info.optString("author").trim() + val description = info.optString("description").trim() + val cover = info.optString("cover").trim().lowercase(Locale.ROOT) + if (author.isNotEmpty() && !author.equals("Auteur Inconnu", ignoreCase = true)) { + return DETAILS_SCORE_WITH_METADATA + } + if (description.isNotEmpty()) { + return DETAILS_SCORE_WITH_METADATA + } + if (cover.isNotEmpty() && "logo-luffy" !in cover) { + return DETAILS_SCORE_WITH_METADATA + } + return DETAILS_SCORE_EMPTY + } + + private fun extractScanFolder(url: String): String? { + val httpUrl = url.toAbsoluteUrl(domain).toHttpUrlOrNull() ?: return null + httpUrl.queryParameter("scan")?.takeIf { it.isNotBlank() }?.let { return it } + val scanIndex = httpUrl.pathSegments.indexOf("scan") + if (scanIndex != -1 && scanIndex + 1 < httpUrl.pathSegments.size) { + return httpUrl.pathSegments[scanIndex + 1].takeIf { it.isNotBlank() } + } + return null + } + + private fun extractSlugFromMangaUrl(url: String): String? { + if (url.startsWith("/manga/")) { + return url.substringAfter("/manga/").substringBefore('/').takeIf { it.isNotBlank() } + } + val httpUrl = url.toHttpUrlOrNull() ?: return null + val mangaIndex = httpUrl.pathSegments.indexOf("manga") + if (mangaIndex != -1 && mangaIndex + 1 < httpUrl.pathSegments.size) { + return httpUrl.pathSegments[mangaIndex + 1].takeIf { it.isNotBlank() } + } + return null + } + + private fun String.toMangaTitle(): String = URLDecoder.decode(this, StandardCharsets.UTF_8.name()).replace('+', ' ').trim() + + private fun String.toMangaSlug(): String = lowercase(Locale.ROOT) + .trim() + .replace(WHITESPACES_REGEX, "+") + + private fun legacyMangaTitle(url: String): String? = when (url.uppercase(Locale.ROOT)) { + "OP" -> "One Piece" + "LCDL" -> "Les Carnets de l'apothicaire" + "JKM" -> "Jujutsu Kaisen Modulo" + "OPC" -> "One Piece Colo" + "LDS" -> "L'Atelier des Sorciers" + else -> null + } + + private companion object { + + private const val API_LIMIT = 12 + private const val UNKNOWN_MANGA_TITLE = "Unknown manga" + private const val DETAILS_SCORE_EMPTY = 0 + private const val DETAILS_SCORE_WITH_METADATA = 1 + private const val DETAILS_SCORE_WITH_CHAPTERS = 2 + private val AUTHORS_SPLIT_PATTERN = Regex("[,&/]") + private val WHITESPACES_REGEX = Regex("\\s+") + private val CHAPTER_NUMBER_REGEX = Regex("(\\d+(?:\\.\\d+)?)$") + private val IMAGE_MTIMES_BLOCK = Regex("imageMtimes\\s*=\\s*\\{([^}]*)}") + private val IMAGE_MTIMES_ENTRY = Regex("[\"']?([0-9]+)[\"']?\\s*:\\s*([0-9]+)") + } } From c5b183e692dfd584ba40e7eb3c8220a8427d2a74 Mon Sep 17 00:00:00 2001 From: epikaigle444 Date: Fri, 13 Feb 2026 17:08:59 +0100 Subject: [PATCH 2/3] fr parsers: update MangaMoins/MangasOrigines/FmTeam and migrate BlueSolo to PizzaReader --- .../doki/parsers/site/fr/MangaMoins.kt | 35 ++- .../doki/parsers/site/madara/fr/BlueSolo.kt | 14 - .../parsers/site/madara/fr/MangasOrigines.kt | 138 ++++++++- .../site/pizzareader/PizzaReaderParser.kt | 290 ++++++++++++------ .../parsers/site/pizzareader/fr/BlueSolo.kt | 24 ++ .../parsers/site/pizzareader/fr/FmTeam.kt | 7 + 6 files changed, 400 insertions(+), 108 deletions(-) delete mode 100644 src/main/kotlin/org/dokiteam/doki/parsers/site/madara/fr/BlueSolo.kt create mode 100644 src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/fr/BlueSolo.kt diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoins.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoins.kt index 209f239..b6f10ad 100644 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoins.kt +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoins.kt @@ -167,8 +167,8 @@ internal class MangaMoins(context: MangaLoaderContext) : val jo = ja.optJSONObject(i) ?: continue val folder = jo.optString("folder").trim() if (folder.isEmpty()) continue - val chapterTitle = jo.optString("title").trim().ifEmpty { null } val chapterNumber = parseChapterNumber(jo.optString("num"), folder) + val chapterTitle = buildChapterTitle(chapterNumber, jo.optString("title")) result += MangaChapter( id = generateUid(folder), title = chapterTitle, @@ -237,6 +237,37 @@ internal class MangaMoins(context: MangaLoaderContext) : return fromFolder ?: 0f } + private fun buildChapterTitle(number: Float, rawTitle: String?): String? { + val title = rawTitle?.trim()?.takeIf { it.isNotEmpty() } + if (number <= 0f) return title + + val formattedNumber = formatChapterNumber(number) + if (title == null) { + return "Chapitre $formattedNumber" + } + + val normalizedTitle = title.lowercase(Locale.ROOT) + if ( + normalizedTitle.startsWith("chapitre") || + normalizedTitle.startsWith("chapter") || + normalizedTitle.startsWith("ch.") + ) { + return title + } + if (normalizedTitle == formattedNumber || normalizedTitle == "#$formattedNumber") { + return "Chapitre $formattedNumber" + } + return "Chapitre $formattedNumber - $title" + } + + private fun formatChapterNumber(number: Float): String { + return if (number % 1f == 0f) { + number.toInt().toString() + } else { + number.toString() + } + } + private fun buildDetailsCandidates(manga: Manga): List { val ordered = listOfNotNull( extractSlugFromMangaUrl(manga.url)?.toMangaTitle(), @@ -319,7 +350,7 @@ internal class MangaMoins(context: MangaLoaderContext) : private val AUTHORS_SPLIT_PATTERN = Regex("[,&/]") private val WHITESPACES_REGEX = Regex("\\s+") private val CHAPTER_NUMBER_REGEX = Regex("(\\d+(?:\\.\\d+)?)$") - private val IMAGE_MTIMES_BLOCK = Regex("imageMtimes\\s*=\\s*\\{([^}]*)}") + private val IMAGE_MTIMES_BLOCK = Regex("imageMtimes\\s*=\\s*\\{([^}]*)\\}") private val IMAGE_MTIMES_ENTRY = Regex("[\"']?([0-9]+)[\"']?\\s*:\\s*([0-9]+)") } } diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/madara/fr/BlueSolo.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/madara/fr/BlueSolo.kt deleted file mode 100644 index 84e4413..0000000 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/madara/fr/BlueSolo.kt +++ /dev/null @@ -1,14 +0,0 @@ -package org.dokiteam.doki.parsers.site.madara.fr - -import org.dokiteam.doki.parsers.Broken -import org.dokiteam.doki.parsers.MangaLoaderContext -import org.dokiteam.doki.parsers.MangaSourceParser -import org.dokiteam.doki.parsers.model.MangaParserSource -import org.dokiteam.doki.parsers.site.madara.MadaraParser - -@Broken ( "Need refactor") -@MangaSourceParser("BLUESOLO", "BlueSolo", "fr") -internal class BlueSolo(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.BLUESOLO, "www1.bluesolo.org", 10) { - override val datePattern = "d MMMM yyyy" -} diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/madara/fr/MangasOrigines.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/madara/fr/MangasOrigines.kt index 36abc8c..b82cae2 100644 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/madara/fr/MangasOrigines.kt +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/madara/fr/MangasOrigines.kt @@ -1,14 +1,148 @@ package org.dokiteam.doki.parsers.site.madara.fr +import org.jsoup.nodes.Document import org.dokiteam.doki.parsers.MangaLoaderContext import org.dokiteam.doki.parsers.MangaSourceParser +import org.dokiteam.doki.parsers.model.Manga +import org.dokiteam.doki.parsers.model.MangaChapter import org.dokiteam.doki.parsers.model.MangaParserSource import org.dokiteam.doki.parsers.site.madara.MadaraParser +import org.dokiteam.doki.parsers.util.attrAsRelativeUrl +import org.dokiteam.doki.parsers.util.generateUid +import org.dokiteam.doki.parsers.util.mapChapters +import org.dokiteam.doki.parsers.util.parseHtml +import org.dokiteam.doki.parsers.util.parseSafe +import org.dokiteam.doki.parsers.util.selectFirstOrThrow +import org.dokiteam.doki.parsers.util.toAbsoluteUrl +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.Locale @MangaSourceParser("MANGASORIGINES", "MangasOrigines.fr", "fr") internal class MangasOrigines(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.MANGASORIGINES, "mangas-origines.fr") { - override val datePattern = "dd/MM/yyyy" + override val datePattern = "MMMM d, yyyy" override val tagPrefix = "manga-genres/" - override val listUrl = "oeuvre/" + override val listUrl = "catalogues/" + + override suspend fun getChapters(manga: Manga, doc: Document): List { + return parseChapterList(doc.body().select(selectChapter), sourceOrderFallback = true) + } + + override suspend fun loadChapters(mangaUrl: String, document: Document): List { + val doc = if (postReq) { + val mangaId = document.select("div#manga-chapters-holder").attr("data-id") + val url = "https://$domain/wp-admin/admin-ajax.php" + val postData = postDataReq + mangaId + webClient.httpPost(url, postData).parseHtml() + } else { + val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix("/") + "/ajax/chapters/" + webClient.httpPost(url, emptyMap()).parseHtml() + } + return parseChapterList(doc.select(selectChapter), sourceOrderFallback = true) + } + + private fun parseChapterList(items: List, sourceOrderFallback: Boolean): List { + val absoluteFrDate = SimpleDateFormat(datePattern, sourceLocale) + val absoluteEnDate = SimpleDateFormat(datePattern, Locale.ENGLISH) + return items.mapChapters(reversed = true) { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val chapterTitle = (a.selectFirst("p")?.text() ?: a.ownText()).trim().ifEmpty { null } + val dateText = li.selectFirst("a.c-new-tag")?.attr("title") + ?: li.selectFirst(selectDate)?.text() + + MangaChapter( + id = generateUid(href), + title = chapterTitle, + number = parseChapterNumber(chapterTitle, href, fallback = if (sourceOrderFallback) i + 1f else 0f), + volume = 0, + url = href + stylePage, + uploadDate = parseUploadDate(dateText, absoluteFrDate, absoluteEnDate), + source = source, + scanlator = null, + branch = null, + ) + } + } + + private fun parseUploadDate(raw: String?, frDate: DateFormat, enDate: DateFormat): Long { + val normalizedRaw = raw?.trim()?.replace('\u00a0', ' ')?.ifEmpty { return 0L } ?: return 0L + val date = normalizedRaw.lowercase(Locale.ROOT) + val number = DATE_NUMBER.find(date)?.groupValues?.getOrNull(1)?.toIntOrNull() + if (number != null) { + val cal = Calendar.getInstance() + when { + DATE_SECONDS.containsMatchIn(date) -> return cal.apply { add(Calendar.SECOND, -number) }.timeInMillis + DATE_MINUTES.containsMatchIn(date) -> return cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis + DATE_HOURS.containsMatchIn(date) -> return cal.apply { add(Calendar.HOUR, -number) }.timeInMillis + DATE_DAYS.containsMatchIn(date) -> return cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + DATE_WEEKS.containsMatchIn(date) -> return cal.apply { add(Calendar.WEEK_OF_YEAR, -number) }.timeInMillis + DATE_MONTHS.containsMatchIn(date) -> return cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + DATE_YEARS.containsMatchIn(date) -> return cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + } + } + if ("hier" in date) { + return Calendar.getInstance().apply { + add(Calendar.DAY_OF_MONTH, -1) + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + if ("aujourd" in date || "today" == date) { + return Calendar.getInstance().apply { + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + val parsedFr = frDate.parseSafe(normalizedRaw) + return if (parsedFr != 0L) parsedFr else enDate.parseSafe(normalizedRaw) + } + + private fun parseChapterNumber(title: String?, href: String, fallback: Float): Float { + title?.let { + CHAPTER_TITLE_NUMBER.find(it)?.groupValues?.getOrNull(1)?.normalizeChapterNumber()?.let { n -> + return n + } + } + CHAPTER_URL_NUMBER.find(href)?.groupValues?.getOrNull(1)?.normalizeChapterNumberFromSlug()?.let { n -> + return n + } + return fallback + } + + private fun String.normalizeChapterNumber(): Float? { + val cleaned = trim().replace(',', '.') + return cleaned.toFloatOrNull() + } + + private fun String.normalizeChapterNumberFromSlug(): Float? { + val token = trim().lowercase(Locale.ROOT) + val decimalToken = token.substringBefore("-va").substringBefore("-vf") + if (DECIMAL_SLUG.matches(decimalToken)) { + return decimalToken.replace('-', '.').toFloatOrNull() + } + return decimalToken.toFloatOrNull() + } + + private companion object { + + private val CHAPTER_TITLE_NUMBER = Regex("(?i)\\bchap(?:itre|ter)?\\.?\\s*([0-9]+(?:[.,][0-9]+)?)") + private val CHAPTER_URL_NUMBER = Regex("(?i)/(?:chapitre|chapter)-([0-9]+(?:-[0-9]+)?)(?:-|/|$)") + private val DECIMAL_SLUG = Regex("^[0-9]+-[0-9]+$") + + private val DATE_NUMBER = Regex("(\\d+)") + private val DATE_SECONDS = Regex("\\b(seconde|secondes|sec)\\b") + private val DATE_MINUTES = Regex("\\b(minute|minutes|min)\\b") + private val DATE_HOURS = Regex("\\b(heure|heures|h)\\b") + private val DATE_DAYS = Regex("\\b(jour|jours|j)\\b") + private val DATE_WEEKS = Regex("\\b(semaine|semaines|sem)\\b") + private val DATE_MONTHS = Regex("\\b(mois)\\b") + private val DATE_YEARS = Regex("\\b(an|ans|année|années|year|years)\\b") + } } diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/PizzaReaderParser.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/PizzaReaderParser.kt index f64337d..5e6fe36 100644 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/PizzaReaderParser.kt +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/PizzaReaderParser.kt @@ -9,9 +9,11 @@ import org.dokiteam.doki.parsers.core.SinglePageMangaParser import org.dokiteam.doki.parsers.model.* import org.dokiteam.doki.parsers.util.* import org.dokiteam.doki.parsers.util.json.asTypedList +import org.dokiteam.doki.parsers.util.json.getIntOrDefault import org.dokiteam.doki.parsers.util.json.getStringOrNull import org.dokiteam.doki.parsers.util.json.mapJSONIndexed import org.dokiteam.doki.parsers.util.json.mapJSONToSet +import org.dokiteam.doki.parsers.util.json.toStringSet import java.text.SimpleDateFormat import java.util.* @@ -28,7 +30,7 @@ internal abstract class PizzaReaderParser( keys.add(userAgentKey) } - override val availableSortOrders: Set = EnumSet.of(SortOrder.ALPHABETICAL) + override open val availableSortOrders: Set = EnumSet.of(SortOrder.ALPHABETICAL) override val filterCapabilities: MangaListFilterCapabilities get() = MangaListFilterCapabilities( @@ -45,19 +47,24 @@ internal abstract class PizzaReaderParser( @JvmField protected val ongoing: Set = hashSetOf( "en cours", + "ongoing", + "on going", "in corso", "in corso (cadenza irregolare)", "in corso (irregolare)", "in corso (mensile)", "in corso (quindicinale)", "in corso (settimanale)", - "In corso (bisettimanale)", + "in corso (bisettimanale)", ) @JvmField protected val finished: Set = hashSetOf( "terminé", + "finished", + "completed", + "complete", "concluso", "completato", ) @@ -65,12 +72,20 @@ internal abstract class PizzaReaderParser( @JvmField protected val paused: Set = hashSetOf( "in pausa", + "hiatus", + "paused", + "on hold", + "sospeso", "in corso (in pausa)", ) @JvmField protected val abandoned: Set = hashSetOf( "droppato", + "dropped", + "abandoned", + "cancelled", + "canceled", ) @@ -80,12 +95,11 @@ internal abstract class PizzaReaderParser( protected open val abandonedFilter = "droppato" override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { - var foundTag = true - var foundTagExclude = true - var foundState = true - var foundContentRating = true - - val manga = ArrayList() + val manga = ArrayList() + val selectedState = filter.states.oneOrThrowIfMany() + val selectedRating = filter.contentRating.oneOrThrowIfMany() + val includeTags = filter.tags.map { it.key.lowercase(Locale.ROOT) } + val excludeTags = filter.tagsExclude.map { it.key.lowercase(Locale.ROOT) } when { !filter.query.isNullOrEmpty() -> { @@ -94,7 +108,12 @@ internal abstract class PizzaReaderParser( for (i in 0 until jsonManga.length()) { val j = jsonManga.getJSONObject(i) val href = "/api" + j.getString("url") - manga.add(addManga(href, j)) + manga.add( + MangaWithDate( + manga = addManga(href, j), + updateDate = parseMangaUpdateDate(j), + ), + ) } } @@ -104,100 +123,110 @@ internal abstract class PizzaReaderParser( val j = jsonManga.getJSONObject(i) val href = "/api" + j.getString("url") - - if (filter.tags.isNotEmpty()) { - val a = j.getJSONArray("genres").toString() - foundTag = false - filter.tags.forEach { - if (a.contains(it.key, ignoreCase = true)) { - foundTag = true - } + if (includeTags.isNotEmpty() || excludeTags.isNotEmpty()) { + val genres = parseGenreKeys(j) + if (includeTags.isNotEmpty() && includeTags.none { it in genres }) { + continue } - } - - if (filter.tagsExclude.isNotEmpty()) { - val a = j.getJSONArray("genres").toString() - foundTagExclude = false - filter.tagsExclude.forEach { - if (!a.contains(it.key, ignoreCase = true)) { - foundTagExclude = true - } + if (excludeTags.isNotEmpty() && excludeTags.any { it in genres }) { + continue } } - - if (filter.states.isNotEmpty()) { - val a = j.getString("status") - foundState = false - filter.states.oneOrThrowIfMany()?.let { - if (a.lowercase().contains( - when (it) { - MangaState.PAUSED -> hiatusFilter - MangaState.ONGOING -> ongoingFilter - MangaState.FINISHED -> completedFilter - MangaState.ABANDONED -> abandonedFilter - else -> "" - }, - ignoreCase = true, - ) - ) { - foundState = true - } + selectedState?.let { state -> + if (!isMatchingState(j.getStringOrNull("status"), state)) { + continue } - } - - if (filter.contentRating.isNotEmpty()) { - val a = j.getInt("adult") - foundContentRating = false - filter.contentRating.oneOrThrowIfMany()?.let { - if (a == ( - when (it) { - ContentRating.SAFE -> 0 - ContentRating.ADULT -> 1 - else -> 0 - } - ) - ) { - foundContentRating = true - } + selectedRating?.let { rating -> + val expected = when (rating) { + ContentRating.SAFE -> 0 + ContentRating.ADULT -> 1 + else -> 0 + } + if (j.getIntOrDefault("adult", 0) != expected) { + continue } - - } - - if (foundState && foundTag && foundTagExclude && foundContentRating) { - manga.add(addManga(href, j)) } + manga.add( + MangaWithDate( + manga = addManga(href, j), + updateDate = parseMangaUpdateDate(j), + ), + ) } } } - return manga + return when (order) { + SortOrder.UPDATED -> manga.sortedByDescending { it.updateDate } + SortOrder.UPDATED_ASC -> manga.sortedBy { it.updateDate } + SortOrder.ALPHABETICAL_DESC -> manga.sortedByDescending { it.manga.title.lowercase(Locale.ROOT) } + else -> manga.sortedBy { it.manga.title.lowercase(Locale.ROOT) } + }.map { it.manga } + } + + private fun parseMangaUpdateDate(json: JSONObject): Long { + val lastChapter = json.optJSONObject("last_chapter") ?: return 0L + lastChapter.getStringOrNull("published_on")?.let { date -> + parseDate(date)?.let { return it } + } + lastChapter.getStringOrNull("updated_at")?.let { date -> + parseDate(date)?.let { return it } + } + return 0L + } + + private fun isMatchingState(rawStatus: String?, state: MangaState): Boolean { + val status = rawStatus?.trim()?.lowercase(Locale.ROOT) ?: return false + return when (state) { + MangaState.PAUSED -> status in paused || status.contains(hiatusFilter, ignoreCase = true) + MangaState.ONGOING -> status in ongoing || status.contains(ongoingFilter, ignoreCase = true) + MangaState.FINISHED -> status in finished || status.contains(completedFilter, ignoreCase = true) + MangaState.ABANDONED -> status in abandoned || status.contains(abandonedFilter, ignoreCase = true) + else -> false + } + } + + private fun parseGenreKeys(json: JSONObject): Set { + val genres = json.optJSONArray("genres") ?: return emptySet() + if (genres.length() == 0) return emptySet() + val result = HashSet(genres.length() * 2) + for (i in 0 until genres.length()) { + val genreObject = genres.optJSONObject(i) + if (genreObject != null) { + genreObject.getStringOrNull("slug")?.lowercase(Locale.ROOT)?.let(result::add) + genreObject.getStringOrNull("name")?.lowercase(Locale.ROOT)?.let(result::add) + continue + } + genres.optString(i).takeIf { it.isNotBlank() && !it.equals("null", ignoreCase = true) } + ?.lowercase(Locale.ROOT) + ?.let(result::add) + } + return result } private fun addManga(href: String, j: JSONObject): Manga { - val isNsfwSource = when (j.getString("adult").toInt()) { + val isNsfwSource = when (j.getIntOrDefault("adult", 0)) { 0 -> false 1 -> true else -> true } - val author = j.getString("author") + val author = j.getStringOrNull("author")?.takeIf { it.isNotBlank() } + val comicPath = j.getStringOrNull("url") + val altTitles = j.optJSONArray("alt_titles")?.toStringSet().orEmpty() return Manga( id = generateUid(href), url = href, - publicUrl = href.toAbsoluteUrl(domain), + publicUrl = comicPath?.toAbsoluteUrl(domain) ?: href.toAbsoluteUrl(domain), coverUrl = j.getString("thumbnail"), title = j.getString("title"), description = j.getString("description"), - altTitles = j.getJSONArray("alt_titles").toString() - .replace("[\"", "") - .replace("\"]", "") - .split("\",\"") - .toSet(), + altTitles = altTitles, rating = j.getString("rating").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, tags = emptySet(), authors = setOfNotNull(author), - state = when (j.getString("status").lowercase()) { + state = when (j.getStringOrNull("status")?.trim()?.lowercase(Locale.ROOT)) { in ongoing -> MangaState.ONGOING in finished -> MangaState.FINISHED in paused -> MangaState.PAUSED @@ -212,7 +241,6 @@ internal abstract class PizzaReaderParser( override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val fullUrl = manga.url.toAbsoluteUrl(domain) val json = webClient.httpGet(fullUrl).parseJson().getJSONObject("comic") - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US) val chapters = JSONArray(json.getJSONArray("chapters").asTypedList().reversed()) manga.copy( @@ -225,16 +253,19 @@ internal abstract class PizzaReaderParser( }, chapters = chapters.mapJSONIndexed { i, j -> val url = "/api" + j.getString("url").toRelativeUrl(domain) - val name = j.getString("full_title") - val date = j.getStringOrNull("updated_at") + val fallbackNumber = i + 1f + val number = parseChapterNumber(j, fallbackNumber) + val name = j.getStringOrNull("full_title") + ?: buildChapterTitle(number, j.getStringOrNull("title")) + val date = parseChapterDate(j) MangaChapter( id = generateUid(url), title = name, - number = i + 1f, - volume = 0, + number = number, + volume = j.optInt("volume", 0), url = url, scanlator = null, - uploadDate = dateFormat.parseSafe(date), + uploadDate = date, branch = null, source = source, ) @@ -244,16 +275,95 @@ internal abstract class PizzaReaderParser( override suspend fun getPages(chapter: MangaChapter): List { val fullUrl = chapter.url.toAbsoluteUrl(domain) - val jsonPages = webClient.httpGet(fullUrl).parseJson().getJSONObject("chapter").getJSONArray("pages").toString() - val pages = jsonPages.replace("[", "").replace("]", "") - .replace("\\", "").split("\",\"").drop(1) - return pages.map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, + val pages = webClient.httpGet(fullUrl) + .parseJson() + .getJSONObject("chapter") + .optJSONArray("pages") + ?: return emptyList() + val result = ArrayList(pages.length()) + for (i in 0 until pages.length()) { + val url = pages.optString(i).trim() + if (url.isEmpty()) { + continue + } + val absoluteUrl = url.toAbsoluteUrl(domain) + result.add( + MangaPage( + id = generateUid(absoluteUrl), + url = absoluteUrl, + preview = null, + source = source, + ), ) } + return result + } + + private fun parseChapterNumber(chapter: JSONObject, fallback: Float): Float { + val rawChapter = chapter.getStringOrNull("chapter") + val rawSubchapter = chapter.getStringOrNull("subchapter") + if (!rawChapter.isNullOrBlank()) { + if ('.' in rawChapter) { + rawChapter.toFloatOrNull()?.let { return it } + } + if (!rawSubchapter.isNullOrBlank()) { + "$rawChapter.$rawSubchapter".toFloatOrNull()?.let { return it } + } + rawChapter.toFloatOrNull()?.let { return it } + } + val fullChapter = chapter.getStringOrNull("full_chapter") + if (!fullChapter.isNullOrBlank()) { + CHAPTER_NUMBER_FROM_LABEL_REGEX.find(fullChapter)?.groupValues?.getOrNull(1)?.toFloatOrNull()?.let { + return it + } + CHAPTER_NUMBER_LAST_REGEX.find(fullChapter)?.groupValues?.getOrNull(1)?.toFloatOrNull()?.let { return it } + } + return fallback + } + + private fun buildChapterTitle(number: Float, title: String?): String? { + val cleanTitle = title?.trim().takeUnless { it.isNullOrEmpty() } + val formatted = if (number % 1f == 0f) number.toInt().toString() else number.toString() + return when { + formatted == "0" -> cleanTitle + cleanTitle == null -> "Chapter $formatted" + else -> "Chapter $formatted : $cleanTitle" + } + } + + private fun parseChapterDate(chapter: JSONObject): Long { + chapter.getStringOrNull("published_on")?.let { date -> + parseDate(date)?.let { return it } + } + chapter.getStringOrNull("updated_at")?.let { date -> + parseDate(date)?.let { return it } + } + return 0L + } + + private fun parseDate(rawDate: String): Long? { + for (pattern in CHAPTER_DATE_PATTERNS) { + val dateFormat = SimpleDateFormat(pattern, Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + dateFormat.parseSafe(rawDate).takeIf { it != 0L }?.let { return it } + } + return null + } + + private companion object { + private data class MangaWithDate( + val manga: Manga, + val updateDate: Long, + ) + + private val CHAPTER_DATE_PATTERNS = arrayOf( + "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", + "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", + "yyyy-MM-dd'T'HH:mm:ss'Z'", + "yyyy-MM-dd HH:mm:ss", + ) + private val CHAPTER_NUMBER_FROM_LABEL_REGEX = Regex("ch(?:apter)?\\.?\\s*(\\d+(?:\\.\\d+)?)", RegexOption.IGNORE_CASE) + private val CHAPTER_NUMBER_LAST_REGEX = Regex("(\\d+(?:\\.\\d+)?)(?!.*\\d)") } } diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/fr/BlueSolo.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/fr/BlueSolo.kt new file mode 100644 index 0000000..34bd094 --- /dev/null +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/fr/BlueSolo.kt @@ -0,0 +1,24 @@ +package org.dokiteam.doki.parsers.site.pizzareader.fr + +import org.dokiteam.doki.parsers.MangaLoaderContext +import org.dokiteam.doki.parsers.MangaSourceParser +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.SortOrder +import org.dokiteam.doki.parsers.site.pizzareader.PizzaReaderParser +import java.util.EnumSet + +@MangaSourceParser("BLUESOLO", "BlueSolo", "fr") +internal class BlueSolo(context: MangaLoaderContext) : + PizzaReaderParser(context, MangaParserSource.BLUESOLO, "bluesolo.org") { + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + ) + + override val ongoingFilter = "en cours" + override val completedFilter = "terminé" + override val hiatusFilter = "hiatus" + override val abandonedFilter = "cancel" +} diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/fr/FmTeam.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/fr/FmTeam.kt index eb18c06..1ac448e 100644 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/fr/FmTeam.kt +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/fr/FmTeam.kt @@ -3,11 +3,18 @@ package org.dokiteam.doki.parsers.site.pizzareader.fr import org.dokiteam.doki.parsers.MangaLoaderContext import org.dokiteam.doki.parsers.MangaSourceParser import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.SortOrder import org.dokiteam.doki.parsers.site.pizzareader.PizzaReaderParser +import java.util.EnumSet @MangaSourceParser("FMTEAM", "FmTeam", "fr") internal class FmTeam(context: MangaLoaderContext) : PizzaReaderParser(context, MangaParserSource.FMTEAM, "fmteam.fr") { + override val availableSortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + ) override val ongoingFilter = "en cours" override val completedFilter = "terminé" } From 2fadcbbfe796cd1b75bcd95406d01d9c9dc471bb Mon Sep 17 00:00:00 2001 From: epikaigle444 Date: Sat, 14 Feb 2026 16:56:06 +0100 Subject: [PATCH 3/3] feat(fr): optimize parsers, add full verifications, remove ScanManga --- .github/summary.yaml | 2 +- .../doki/parsers/site/all/MangaPlusParser.kt | 35 +- .../dokiteam/doki/parsers/site/fr/BigSolo.kt | 727 ++++++++++++++++++ .../doki/parsers/site/fr/MangaMoins.kt | 87 ++- .../doki/parsers/site/fr/PoseidonScans.kt | 685 +++++++++++++++-- .../doki/parsers/site/fr/PunkRecordz.kt | 654 ++++++++++++++++ .../parsers/site/madara/fr/MangasOrigines.kt | 20 + .../site/mangareader/fr/SushiScanFR.kt | 25 + .../doki/parsers/site/mmrcms/fr/ScanManga.kt | 16 - .../site/pizzareader/PizzaReaderParser.kt | 58 +- .../MangaPlusFrenchFullVerificationTest.kt | 94 +++ .../site/fr/BigSoloFullVerificationTest.kt | 82 ++ .../site/fr/MangaMoinsFullVerificationTest.kt | 89 +++ .../fr/PoseidonScansChapterPagesSmokeTest.kt | 49 ++ .../fr/PoseidonScansFullVerificationTest.kt | 183 +++++ .../PoseidonScansPremiumVerificationTest.kt | 204 +++++ .../fr/PunkRecordzFullVerificationTest.kt | 166 ++++ .../fr/MangasOriginesFullVerificationTest.kt | 85 ++ .../fr/SushiScanFRFullVerificationTest.kt | 85 ++ .../fr/PizzaReaderFrFullVerificationTest.kt | 107 +++ 20 files changed, 3338 insertions(+), 115 deletions(-) create mode 100644 src/main/kotlin/org/dokiteam/doki/parsers/site/fr/BigSolo.kt create mode 100644 src/main/kotlin/org/dokiteam/doki/parsers/site/fr/PunkRecordz.kt delete mode 100644 src/main/kotlin/org/dokiteam/doki/parsers/site/mmrcms/fr/ScanManga.kt create mode 100644 src/test/kotlin/org/dokiteam/doki/parsers/site/all/MangaPlusFrenchFullVerificationTest.kt create mode 100644 src/test/kotlin/org/dokiteam/doki/parsers/site/fr/BigSoloFullVerificationTest.kt create mode 100644 src/test/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoinsFullVerificationTest.kt create mode 100644 src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScansChapterPagesSmokeTest.kt create mode 100644 src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScansFullVerificationTest.kt create mode 100644 src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScansPremiumVerificationTest.kt create mode 100644 src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PunkRecordzFullVerificationTest.kt create mode 100644 src/test/kotlin/org/dokiteam/doki/parsers/site/madara/fr/MangasOriginesFullVerificationTest.kt create mode 100644 src/test/kotlin/org/dokiteam/doki/parsers/site/mangareader/fr/SushiScanFRFullVerificationTest.kt create mode 100644 src/test/kotlin/org/dokiteam/doki/parsers/site/pizzareader/fr/PizzaReaderFrFullVerificationTest.kt diff --git a/.github/summary.yaml b/.github/summary.yaml index 1a3f338..06fcf93 100644 --- a/.github/summary.yaml +++ b/.github/summary.yaml @@ -1 +1 @@ -total: 1267 \ No newline at end of file +total: 1268 \ No newline at end of file diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/all/MangaPlusParser.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/all/MangaPlusParser.kt index 406411f..b40c65e 100644 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/all/MangaPlusParser.kt +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/all/MangaPlusParser.kt @@ -29,6 +29,16 @@ internal abstract class MangaPlusParser( private val apiUrl = "https://jumpg-webapi.tokyo-cdn.com/api" override val configKeyDomain = ConfigKey.Domain("mangaplus.shueisha.co.jp") + private val detailsCache = object : LinkedHashMap(64, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { + return size > DETAILS_CACHE_SIZE + } + } + private val pagesCache = object : LinkedHashMap>(128, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry>?): Boolean { + return size > PAGES_CACHE_SIZE + } + } override fun onCreateConfig(keys: MutableCollection>) { super.onCreateConfig(keys) @@ -133,6 +143,9 @@ internal abstract class MangaPlusParser( } override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCache) { + detailsCache[manga.url]?.let { return it } + } val json = apiCall("/title_detailV3?title_id=${manga.url}") .getJSONObject("titleDetailView") val title = json.getJSONObject("title") @@ -146,7 +159,7 @@ internal abstract class MangaPlusParser( val author = title.getString("author") .split("/").joinToString(transform = String::trim) - return manga.copy( + val details = manga.copy( title = title.getString("name"), publicUrl = "/titles/${title.getInt("titleId")}".toAbsoluteUrl(domain), coverUrl = title.getString("portraitImageUrl"), @@ -167,6 +180,10 @@ internal abstract class MangaPlusParser( else -> MangaState.ONGOING }, ) + synchronized(detailsCache) { + detailsCache[manga.url] = details + } + return details } private fun parseChapters(chapterListGroup: JSONArray, language: String): List { @@ -201,11 +218,14 @@ internal abstract class MangaPlusParser( } override suspend fun getPages(chapter: MangaChapter): List { + synchronized(pagesCache) { + pagesCache[chapter.url]?.let { return it } + } val pages = apiCall("/manga_viewer?chapter_id=${chapter.url}&split=yes&img_quality=super_high") .getJSONObject("mangaViewer") .getJSONArray("pages") - return pages.mapJSONNotNull { + val parsedPages = pages.mapJSONNotNull { val mangaPage = it.optJSONObject("mangaPage") ?: return@mapJSONNotNull null val url = mangaPage.getString("imageUrl") @@ -217,6 +237,12 @@ internal abstract class MangaPlusParser( source = source, ) } + if (parsedPages.isNotEmpty()) { + synchronized(pagesCache) { + pagesCache[chapter.url] = parsedPages + } + } + return parsedPages } // image descrambling @@ -329,4 +355,9 @@ internal abstract class MangaPlusParser( MangaParserSource.MANGAPLUSPARSER_DE, "GERMAN", ) + + private companion object { + private const val DETAILS_CACHE_SIZE = 200 + private const val PAGES_CACHE_SIZE = 400 + } } diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/BigSolo.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/BigSolo.kt new file mode 100644 index 0000000..22aef70 --- /dev/null +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/BigSolo.kt @@ -0,0 +1,727 @@ +package org.dokiteam.doki.parsers.site.fr + +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.json.JSONArray +import org.json.JSONObject +import org.dokiteam.doki.parsers.MangaLoaderContext +import org.dokiteam.doki.parsers.MangaSourceParser +import org.dokiteam.doki.parsers.config.ConfigKey +import org.dokiteam.doki.parsers.core.SinglePageMangaParser +import org.dokiteam.doki.parsers.model.ContentType +import org.dokiteam.doki.parsers.model.Demographic +import org.dokiteam.doki.parsers.model.Manga +import org.dokiteam.doki.parsers.model.MangaChapter +import org.dokiteam.doki.parsers.model.MangaListFilter +import org.dokiteam.doki.parsers.model.MangaListFilterCapabilities +import org.dokiteam.doki.parsers.model.MangaListFilterOptions +import org.dokiteam.doki.parsers.model.MangaPage +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.MangaState +import org.dokiteam.doki.parsers.model.MangaTag +import org.dokiteam.doki.parsers.model.RATING_UNKNOWN +import org.dokiteam.doki.parsers.model.SortOrder +import org.dokiteam.doki.parsers.model.YEAR_UNKNOWN +import org.dokiteam.doki.parsers.util.generateUid +import org.dokiteam.doki.parsers.util.parseJson +import org.dokiteam.doki.parsers.util.parseJsonArray +import org.dokiteam.doki.parsers.util.toAbsoluteUrl +import org.dokiteam.doki.parsers.util.urlEncoded +import org.dokiteam.doki.parsers.util.json.getIntOrDefault +import org.dokiteam.doki.parsers.util.json.getLongOrDefault +import org.dokiteam.doki.parsers.util.json.getStringOrNull +import java.text.Normalizer +import java.util.EnumSet +import java.util.LinkedHashSet +import java.util.Locale + +@MangaSourceParser("BIGSOLO", "BigSolo", "fr") +internal class BigSolo(context: MangaLoaderContext) : + SinglePageMangaParser(context, MangaParserSource.BIGSOLO) { + + override val configKeyDomain = ConfigKey.Domain("bigsolo.org") + + override val defaultSortOrder: SortOrder = SortOrder.UPDATED + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + ) + + override val filterCapabilities: MangaListFilterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearSupported = true, + isYearRangeSupported = true, + isAuthorSearchSupported = true, + ) + + @Volatile + private var catalogCache: CatalogCache? = null + private val pagesCache = HashMap() + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val catalog = getCatalog() + return MangaListFilterOptions( + availableTags = catalog.availableTags, + availableStates = catalog.availableStates, + availableContentTypes = catalog.availableContentTypes, + availableDemographics = catalog.availableDemographics, + ) + } + + override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { + val catalog = getCatalog() + val query = normalizeSearchText(filter.query) + val includeTags = filter.tags.mapTo(hashSetOf()) { normalizeTagKey(it.key) } + val excludeTags = filter.tagsExclude.mapTo(hashSetOf()) { normalizeTagKey(it.key) } + val stateFilter = filter.states + val typesFilter = filter.types + val demographicFilter = filter.demographics + val authorFilter = normalizeSearchText(filter.author) + val filtered = catalog.entries.filter { entry -> + if (query.isNotEmpty() && query !in entry.searchText) { + return@filter false + } + if (authorFilter.isNotEmpty() && authorFilter !in entry.authorsSearchText) { + return@filter false + } + if (includeTags.isNotEmpty() && !entry.tagKeys.containsAll(includeTags)) { + return@filter false + } + if (excludeTags.isNotEmpty() && entry.tagKeys.any { it in excludeTags }) { + return@filter false + } + if (stateFilter.isNotEmpty() && entry.state !in stateFilter) { + return@filter false + } + if (typesFilter.isNotEmpty() && entry.contentType !in typesFilter) { + return@filter false + } + if (demographicFilter.isNotEmpty() && entry.demographic !in demographicFilter) { + return@filter false + } + if (filter.year != YEAR_UNKNOWN && entry.releaseYear != filter.year) { + return@filter false + } + if (filter.yearFrom != YEAR_UNKNOWN && (entry.releaseYear == YEAR_UNKNOWN || entry.releaseYear < filter.yearFrom)) { + return@filter false + } + if (filter.yearTo != YEAR_UNKNOWN && (entry.releaseYear == YEAR_UNKNOWN || entry.releaseYear > filter.yearTo)) { + return@filter false + } + true + } + return sort(filtered, order).map { entry -> + Manga( + id = generateUid(entry.url), + title = entry.title, + altTitles = entry.altTitles, + url = entry.url, + publicUrl = entry.publicUrl, + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = entry.coverUrl, + tags = entry.tags, + state = entry.state, + authors = entry.authors, + largeCoverUrl = entry.largeCoverUrl, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = extractSlugFromUrl(manga.url) ?: extractSlugFromUrl(manga.publicUrl) ?: return manga + val catalog = getCatalog() + val entry = catalog.bySlug[slug] ?: getCatalog(forceRefresh = true).bySlug[slug] ?: return manga + return manga.copy( + title = entry.title, + altTitles = entry.altTitles, + url = "/${entry.slug}", + publicUrl = "https://$domain/${entry.slug}", + coverUrl = entry.coverUrl ?: manga.coverUrl, + largeCoverUrl = entry.largeCoverUrl ?: manga.largeCoverUrl, + description = entry.description ?: manga.description, + tags = entry.tags, + state = entry.state, + authors = entry.authors, + chapters = entry.chapters.map { chapter -> + val chapterUrl = buildChapterUrl(entry.slug, chapter.key, chapter.sourceService, chapter.sourceId) + MangaChapter( + id = generateUid(chapterUrl), + title = buildChapterTitle(chapter.numberLabel, chapter.title), + number = chapter.number, + volume = chapter.volume, + url = chapterUrl, + scanlator = chapter.scanlator, + uploadDate = chapter.uploadDate, + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fallbackChapter = resolveChapterFromCatalog(chapter.url) + val sourceId = chapter.url.queryParameter(SOURCE_ID_PARAM) + ?: fallbackChapter?.sourceId + ?: return emptyList() + val sourceService = chapter.url.queryParameter(SOURCE_SERVICE_PARAM) + ?: fallbackChapter?.sourceService + ?: SOURCE_IMG_CHEST + if (!sourceService.equals(SOURCE_IMG_CHEST, ignoreCase = true)) { + return emptyList() + } + val now = System.currentTimeMillis() + synchronized(pagesCache) { + val cached = pagesCache[sourceId] + if (cached != null && now - cached.fetchedAt <= PAGES_TTL_MS) { + return cached.pages + } + } + val apiUrl = HttpUrl.Builder() + .scheme("https") + .host(domain) + .addPathSegments("api/imgchest-chapter-pages") + .addQueryParameter("id", sourceId) + .build() + val pagesArray = runCatching { + webClient.httpGet(apiUrl).parseJsonArray() + }.getOrElse { + return emptyList() + } + val pages = parsePages(sourceId, pagesArray) + if (pages.isNotEmpty()) { + synchronized(pagesCache) { + pagesCache[sourceId] = CachedPages( + fetchedAt = now, + pages = pages, + ) + } + } + return pages + } + + private suspend fun getCatalog(forceRefresh: Boolean = false): CatalogCache { + val now = System.currentTimeMillis() + val cached = catalogCache + if (!forceRefresh && cached != null && now - cached.fetchedAt <= CATALOG_TTL_MS) { + return cached + } + val fresh = fetchCatalog(now) + catalogCache = fresh + return fresh + } + + private suspend fun fetchCatalog(fetchedAt: Long): CatalogCache { + val payload = webClient.httpGet("https://$domain/data/series").parseJson() + val teamNames = fetchTeamNames() + val entries = ArrayList() + parseSeriesArray(payload.optJSONArray("series"), isOneShot = false, teamNames = teamNames, destination = entries) + parseSeriesArray(payload.optJSONArray("os"), isOneShot = true, teamNames = teamNames, destination = entries) + val availableTags = entries + .flatMap { it.tags } + .distinctBy { it.key } + .sortedBy { it.title.lowercase(Locale.ROOT) } + .toCollection(LinkedHashSet()) + val availableStates = EnumSet.noneOf(MangaState::class.java).apply { + entries.forEach { e -> + e.state?.let { add(it) } + } + } + val availableContentTypes = EnumSet.noneOf(ContentType::class.java).apply { + entries.forEach { add(it.contentType) } + } + val availableDemographics = EnumSet.noneOf(Demographic::class.java).apply { + entries.forEach { e -> + e.demographic?.let { add(it) } + } + } + return CatalogCache( + fetchedAt = fetchedAt, + entries = entries, + bySlug = entries.associateBy { it.slug.lowercase(Locale.ROOT) }, + availableTags = availableTags, + availableStates = availableStates, + availableContentTypes = availableContentTypes, + availableDemographics = availableDemographics, + ) + } + + private fun parseSeriesArray( + array: JSONArray?, + isOneShot: Boolean, + teamNames: Map, + destination: MutableList, + ) { + if (array == null) return + for (i in 0 until array.length()) { + val jo = array.optJSONObject(i) ?: continue + val entry = parseSeriesEntry(jo, isOneShot, teamNames) ?: continue + destination.add(entry) + } + } + + private fun parseSeriesEntry( + json: JSONObject, + isOneShotFromList: Boolean, + teamNames: Map, + ): BigSoloEntry? { + val rawSlug = json.getStringOrNull("slug")?.trim().orEmpty() + if (rawSlug.isEmpty()) return null + val title = json.getStringOrNull("title")?.trim().orEmpty() + if (title.isEmpty()) return null + val coverUrls = parseCoverUrls(json) + val coverUrl = coverUrls.first + val largeCoverUrl = coverUrls.second + + val tagsRaw = parseStringArray(json.optJSONArray("tags")) + val tags = tagsRaw.mapTo(LinkedHashSet()) { value -> + createTag(value) + } + val tagKeys = tags.mapTo(hashSetOf()) { it.key } + val isOneShot = isOneShotFromList || json.optBoolean("os", false) + val contentType = detectContentType(isOneShot, tagKeys) + val chapters = parseChapters(json.optJSONObject("chapters"), teamNames) + val lastUpdate = normalizeTimestamp(json.optJSONObject("last_chapter")?.getLongOrDefault("timestamp", 0L) ?: 0L) + .takeIf { it > 0L } + ?: chapters.maxOfOrNull { it.uploadDate } + ?: 0L + val altTitles = parseAlternativeTitles(json, title) + val authors = parseAuthors(json) + val releaseYear = json.getIntOrDefault("release_year", YEAR_UNKNOWN) + return BigSoloEntry( + slug = rawSlug, + title = title, + altTitles = altTitles, + url = "/$rawSlug", + publicUrl = "https://$domain/$rawSlug", + coverUrl = coverUrl, + largeCoverUrl = largeCoverUrl, + description = json.getStringOrNull("description"), + tags = tags, + tagKeys = tagKeys, + state = parseState(json.getStringOrNull("status")), + contentType = contentType, + demographic = parseDemographic(json.getStringOrNull("demographic")), + releaseYear = releaseYear, + authors = authors, + lastUpdate = lastUpdate, + chapters = chapters, + chaptersByKey = chapters.associateBy { it.key }, + searchText = buildSearchText(title, altTitles, authors, tags), + authorsSearchText = normalizeSearchText(authors.joinToString(" ")), + ) + } + + private fun parseChapters( + chaptersObject: JSONObject?, + teamNames: Map, + ): List { + if (chaptersObject == null) return emptyList() + val chapters = ArrayList(chaptersObject.length()) + val keys = chaptersObject.keys() + while (keys.hasNext()) { + val key = keys.next() + val chapterJson = chaptersObject.optJSONObject(key) ?: continue + val numberLabel = normalizeChapterLabel(key) + val number = numberLabel.toFloatOrNull() ?: continue + val sourceJson = chapterJson.optJSONObject("source") ?: continue + val sourceId = sourceJson.getStringOrNull("id")?.trim()?.takeIf { it.isNotEmpty() } ?: continue + val sourceService = sourceJson.getStringOrNull("service") + ?.trim() + ?.lowercase(Locale.ROOT) + ?.takeIf { it.isNotEmpty() } + ?: SOURCE_IMG_CHEST + val title = chapterJson.getStringOrNull("title") + val volume = chapterJson.getStringOrNull("volume") + ?.trim() + ?.toIntOrNull() + ?: 0 + val teamIds = parseStringList(chapterJson.optJSONArray("teams")) + val scanlator = teamIds.asSequence() + .map { id -> teamNames[id] ?: id.replace('_', ' ') } + .map { it.trim() } + .filter { it.isNotEmpty() } + .distinct() + .joinToString(", ") + .ifBlank { null } + chapters.add( + BigSoloChapterEntry( + key = key, + number = number, + numberLabel = numberLabel, + title = title, + volume = volume, + uploadDate = normalizeTimestamp(chapterJson.getLongOrDefault("timestamp", 0L)), + sourceService = sourceService, + sourceId = sourceId, + scanlator = scanlator, + ), + ) + } + return chapters.sortedWith( + compareBy { it.number } + .thenBy { it.uploadDate } + .thenBy { it.key }, + ) + } + + private fun parsePages(sourceId: String, pagesArray: JSONArray): List { + val pages = ArrayList(pagesArray.length()) + for (i in 0 until pagesArray.length()) { + val jo = pagesArray.optJSONObject(i) ?: continue + val pageUrl = jo.getStringOrNull("link")?.toAbsoluteUrl(domain) ?: continue + val previewUrl = jo.getStringOrNull("thumbnail")?.toAbsoluteUrl(domain) + val position = jo.getIntOrDefault("position", i + 1) + val pageId = jo.getStringOrNull("id") ?: "$sourceId-$position" + pages += IndexedPage( + position = position, + page = MangaPage( + id = generateUid("$sourceId/$pageId"), + url = pageUrl, + preview = previewUrl, + source = source, + ), + ) + } + return pages.sortedBy { it.position }.map { it.page } + } + + private fun sort(entries: List, order: SortOrder): List { + return when (order) { + SortOrder.ALPHABETICAL -> entries.sortedBy { it.title.lowercase(Locale.ROOT) } + SortOrder.ALPHABETICAL_DESC -> entries.sortedByDescending { it.title.lowercase(Locale.ROOT) } + SortOrder.UPDATED -> entries.sortedWith( + compareByDescending { it.lastUpdate }.thenBy { it.title.lowercase(Locale.ROOT) }, + ) + SortOrder.UPDATED_ASC -> entries.sortedWith( + compareBy { it.lastUpdate }.thenBy { it.title.lowercase(Locale.ROOT) }, + ) + SortOrder.NEWEST -> entries.sortedWith( + compareByDescending { it.releaseYear } + .thenByDescending { it.lastUpdate } + .thenBy { it.title.lowercase(Locale.ROOT) }, + ) + SortOrder.NEWEST_ASC -> entries.sortedWith( + compareBy { it.releaseYear } + .thenBy { it.lastUpdate } + .thenBy { it.title.lowercase(Locale.ROOT) }, + ) + else -> entries.sortedWith( + compareByDescending { it.lastUpdate }.thenBy { it.title.lowercase(Locale.ROOT) }, + ) + } + } + + private suspend fun fetchTeamNames(): Map { + return runCatching { + val teamsJson = webClient.httpGet("https://$domain/data/teams").parseJson() + val names = HashMap(teamsJson.length()) + val keys = teamsJson.keys() + while (keys.hasNext()) { + val key = keys.next() + val team = teamsJson.optJSONObject(key) + val name = team?.getStringOrNull("name")?.trim()?.takeIf { it.isNotEmpty() } + ?: key.replace('_', ' ') + names[key] = name + } + names + }.getOrDefault(emptyMap()) + } + + private fun parseCoverUrls(json: JSONObject): Pair { + val cover = json.optJSONObject("cover") + var coverUrl = cover?.getStringOrNull("url_lq")?.toAbsoluteUrl(domain) + ?: cover?.getStringOrNull("url_hq")?.toAbsoluteUrl(domain) + var largeCoverUrl = cover?.getStringOrNull("url_hq")?.toAbsoluteUrl(domain) + ?: cover?.getStringOrNull("url_lq")?.toAbsoluteUrl(domain) + if (coverUrl != null && largeCoverUrl != null) { + return coverUrl to largeCoverUrl + } + val covers = json.optJSONArray("covers") ?: return coverUrl to largeCoverUrl + for (i in 0 until covers.length()) { + val item = covers.optJSONObject(i) ?: continue + if (coverUrl == null) { + coverUrl = item.getStringOrNull("url_lq")?.toAbsoluteUrl(domain) + ?: item.getStringOrNull("url_hq")?.toAbsoluteUrl(domain) + } + if (largeCoverUrl == null) { + largeCoverUrl = item.getStringOrNull("url_hq")?.toAbsoluteUrl(domain) + ?: item.getStringOrNull("url_lq")?.toAbsoluteUrl(domain) + } + if (coverUrl != null && largeCoverUrl != null) { + break + } + } + return coverUrl to largeCoverUrl + } + + private fun parseAlternativeTitles(json: JSONObject, title: String): Set { + val altTitles = LinkedHashSet() + json.getStringOrNull("ja_title")?.trim()?.takeIf { it.isNotEmpty() }?.let { altTitles.add(it) } + val altArray = json.optJSONArray("alternative_titles") + if (altArray != null) { + for (i in 0 until altArray.length()) { + altArray.optString(i).trim().takeIf { it.isNotEmpty() }?.let { altTitles.add(it) } + } + } + altTitles.remove(title) + return altTitles + } + + private fun parseAuthors(json: JSONObject): Set { + val authors = LinkedHashSet() + json.getStringOrNull("author")?.trim()?.takeIf { it.isNotEmpty() }?.let { authors.add(it) } + json.getStringOrNull("artist")?.trim()?.takeIf { it.isNotEmpty() }?.let { authors.add(it) } + return authors + } + + private fun createTag(value: String): MangaTag { + return MangaTag( + key = normalizeTagKey(value), + title = value, + source = source, + ) + } + + private fun buildSearchText( + title: String, + altTitles: Set, + authors: Set, + tags: Set, + ): String { + return normalizeSearchText( + buildString { + append(title) + altTitles.forEach { + append(' ') + append(it) + } + authors.forEach { + append(' ') + append(it) + } + tags.forEach { + append(' ') + append(it.title) + } + }, + ) + } + + private fun detectContentType(isOneShot: Boolean, tagKeys: Set): ContentType { + if (isOneShot) return ContentType.ONE_SHOT + return when { + tagKeys.any { it.contains("manhua") } -> ContentType.MANHUA + tagKeys.any { it.contains("manhwa") || it.contains("webtoon") } -> ContentType.MANHWA + else -> ContentType.MANGA + } + } + + private fun parseDemographic(raw: String?): Demographic? { + val value = raw?.trim()?.lowercase(Locale.ROOT) ?: return null + return when { + value.startsWith("shonen") || value.startsWith("shounen") -> Demographic.SHOUNEN + value.startsWith("shojo") || value.startsWith("shoujo") -> Demographic.SHOUJO + value.startsWith("seinen") -> Demographic.SEINEN + value.startsWith("josei") -> Demographic.JOSEI + value.startsWith("kodomo") -> Demographic.KODOMO + else -> null + } + } + + private fun parseState(raw: String?): MangaState? { + val value = raw?.trim()?.lowercase(Locale.ROOT) ?: return null + return when { + "cours" in value || "ongoing" in value -> MangaState.ONGOING + "fini" in value || "termin" in value || "finished" in value || "complete" in value -> MangaState.FINISHED + "pause" in value || "hiatus" in value -> MangaState.PAUSED + "annul" in value || "cancel" in value || "aband" in value -> MangaState.ABANDONED + else -> null + } + } + + private fun normalizeChapterLabel(raw: String): String { + val value = raw.trim() + if (value.isEmpty()) return value + return value.toBigDecimalOrNull()?.stripTrailingZeros()?.toPlainString() ?: value + } + + private fun buildChapterTitle(numberLabel: String, rawTitle: String?): String { + val title = rawTitle?.trim()?.takeIf { it.isNotEmpty() } + if (title == null) { + return "Chapitre $numberLabel" + } + val normalizedTitle = title.lowercase(Locale.ROOT) + if ( + normalizedTitle.startsWith("chapitre") || + normalizedTitle.startsWith("chapter") || + normalizedTitle.startsWith("ch.") + ) { + return title + } + if (normalizedTitle == numberLabel || normalizedTitle == "#$numberLabel") { + return "Chapitre $numberLabel" + } + return "Chapitre $numberLabel - $title" + } + + private suspend fun resolveChapterFromCatalog(chapterUrl: String): BigSoloChapterEntry? { + val slug = extractSlugFromUrl(chapterUrl) ?: return null + val chapterKey = extractChapterKey(chapterUrl) ?: return null + return getCatalog().bySlug[slug]?.chaptersByKey?.get(chapterKey) + } + + private fun buildChapterUrl(slug: String, chapterKey: String, sourceService: String, sourceId: String): String { + return "/$slug/$chapterKey?$SOURCE_ID_PARAM=${sourceId.urlEncoded()}&$SOURCE_SERVICE_PARAM=${sourceService.urlEncoded()}" + } + + private fun extractSlugFromUrl(url: String): String? { + val absoluteUrl = if (url.startsWith("http://") || url.startsWith("https://")) { + url + } else { + "https://$domain/${url.trimStart('/')}" + } + val path = absoluteUrl.toHttpUrlOrNull()?.encodedPath ?: url.substringBefore('?').substringBefore('#') + val slug = path.trim('/').substringBefore('/').trim() + return slug.takeIf { it.isNotEmpty() }?.lowercase(Locale.ROOT) + } + + private fun extractChapterKey(url: String): String? { + val absoluteUrl = if (url.startsWith("http://") || url.startsWith("https://")) { + url + } else { + "https://$domain/${url.trimStart('/')}" + } + val path = absoluteUrl.toHttpUrlOrNull()?.encodedPath ?: url.substringBefore('?').substringBefore('#') + val parts = path.trim('/').split('/') + return parts.getOrNull(1)?.takeIf { it.isNotEmpty() } + } + + private fun String.queryParameter(name: String): String? { + val absoluteUrl = if (startsWith("http://") || startsWith("https://")) { + this + } else { + "https://$domain/${trimStart('/')}" + } + return absoluteUrl.toHttpUrlOrNull()?.queryParameter(name)?.takeIf { it.isNotEmpty() } + } + + private fun normalizeTimestamp(raw: Long): Long { + if (raw <= 0L) return 0L + return if (raw < TIMESTAMP_SECONDS_THRESHOLD) raw * 1000L else raw + } + + private fun normalizeTagKey(value: String): String { + return value.trim().lowercase(Locale.ROOT).replace(WHITESPACES, " ") + } + + private fun normalizeSearchText(value: String?): String { + val raw = value?.trim().orEmpty() + if (raw.isEmpty()) return "" + val decomposed = Normalizer.normalize(raw, Normalizer.Form.NFD) + return DIACRITICS.replace(decomposed, "") + .lowercase(Locale.ROOT) + .replace(WHITESPACES, " ") + .trim() + } + + private fun parseStringArray(array: JSONArray?): Set { + if (array == null) return emptySet() + val result = LinkedHashSet(array.length()) + for (i in 0 until array.length()) { + val value = array.optString(i).trim() + if (value.isNotEmpty()) { + result.add(value) + } + } + return result + } + + private fun parseStringList(array: JSONArray?): List { + if (array == null) return emptyList() + val result = ArrayList(array.length()) + for (i in 0 until array.length()) { + val value = array.optString(i).trim() + if (value.isNotEmpty()) { + result += value + } + } + return result + } + + private data class CatalogCache( + val fetchedAt: Long, + val entries: List, + val bySlug: Map, + val availableTags: Set, + val availableStates: Set, + val availableContentTypes: Set, + val availableDemographics: Set, + ) + + private data class BigSoloEntry( + val slug: String, + val title: String, + val altTitles: Set, + val url: String, + val publicUrl: String, + val coverUrl: String?, + val largeCoverUrl: String?, + val description: String?, + val tags: Set, + val tagKeys: Set, + val state: MangaState?, + val contentType: ContentType, + val demographic: Demographic?, + val releaseYear: Int, + val authors: Set, + val lastUpdate: Long, + val chapters: List, + val chaptersByKey: Map, + val searchText: String, + val authorsSearchText: String, + ) + + private data class BigSoloChapterEntry( + val key: String, + val number: Float, + val numberLabel: String, + val title: String?, + val volume: Int, + val uploadDate: Long, + val sourceService: String, + val sourceId: String, + val scanlator: String?, + ) + + private data class IndexedPage( + val position: Int, + val page: MangaPage, + ) + + private data class CachedPages( + val fetchedAt: Long, + val pages: List, + ) + + private companion object { + private const val SOURCE_ID_PARAM = "sid" + private const val SOURCE_SERVICE_PARAM = "svc" + private const val SOURCE_IMG_CHEST = "imgchest" + private const val CATALOG_TTL_MS = 10 * 60 * 1000L + private const val PAGES_TTL_MS = 30 * 60 * 1000L + private const val TIMESTAMP_SECONDS_THRESHOLD = 10_000_000_000L + private val WHITESPACES = Regex("\\s+") + private val DIACRITICS = Regex("\\p{Mn}+") + } +} diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoins.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoins.kt index b6f10ad..5a966bf 100644 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoins.kt +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoins.kt @@ -28,6 +28,7 @@ import org.jsoup.nodes.Document import java.net.URLDecoder import java.nio.charset.StandardCharsets import java.util.EnumSet +import java.util.LinkedHashMap import java.util.Locale @MangaSourceParser("MANGAMOINS", "MangaMoins", "fr") @@ -37,6 +38,16 @@ internal class MangaMoins(context: MangaLoaderContext) : override val configKeyDomain = ConfigKey.Domain("mangamoins.com") override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED) override val filterCapabilities = MangaListFilterCapabilities(isSearchSupported = true) + private val detailsCache = object : LinkedHashMap(64, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { + return size > DETAILS_CACHE_SIZE + } + } + private val pagesCache = object : LinkedHashMap>(128, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry>?): Boolean { + return size > PAGES_CACHE_SIZE + } + } override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() .add("Referer", "https://$domain/") @@ -91,6 +102,10 @@ internal class MangaMoins(context: MangaLoaderContext) : } override suspend fun getDetails(manga: Manga): Manga { + val cacheKey = buildDetailsCacheKey(manga) + synchronized(detailsCache) { + detailsCache[cacheKey]?.let { return it } + } val json = fetchMangaJson(manga) val info = json.optJSONObject("info") val detailsTitle = info?.optString("title").orEmpty().ifBlank { manga.title } @@ -101,7 +116,7 @@ internal class MangaMoins(context: MangaLoaderContext) : val canonicalSlug = extractSlugFromMangaUrl(manga.url) ?: extractSlugFromMangaUrl(manga.publicUrl) ?: detailsTitle.toMangaSlug() - return manga.copy( + val details = manga.copy( title = detailsTitle, publicUrl = "https://$domain/manga/$canonicalSlug", coverUrl = coverUrl ?: manga.coverUrl, @@ -110,17 +125,24 @@ internal class MangaMoins(context: MangaLoaderContext) : authors = authors, chapters = chapters.takeIf { it.isNotEmpty() } ?: manga.chapters, ) + synchronized(detailsCache) { + detailsCache[cacheKey] = details + detailsCache[canonicalSlug.lowercase(Locale.ROOT)] = details + } + return details } override suspend fun getPages(chapter: MangaChapter): List { + val cacheKey = extractScanFolder(chapter.url) ?: chapter.url.lowercase(Locale.ROOT) + synchronized(pagesCache) { + pagesCache[cacheKey]?.let { return it } + } val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() val preloaded = doc.select("link[rel=preload][as=image]") - .map { it.attr("href").trim() } - .filter { it.isNotEmpty() } - .map { it.toAbsoluteUrl(domain) } + .mapNotNull { sanitizePageUrl(it.attr("href")) } .distinct() if (preloaded.isNotEmpty()) { - return preloaded.map { pageUrl -> + val pages = preloaded.map { pageUrl -> MangaPage( id = generateUid(pageUrl), url = pageUrl, @@ -128,9 +150,19 @@ internal class MangaMoins(context: MangaLoaderContext) : source = chapter.source, ) } + synchronized(pagesCache) { + pagesCache[cacheKey] = pages + } + return pages } val folder = extractScanFolder(chapter.url) ?: return emptyList() - return parsePagesFromScript(doc, folder, chapter.source) + val pages = parsePagesFromScript(doc, folder, chapter.source) + if (pages.isNotEmpty()) { + synchronized(pagesCache) { + pagesCache[cacheKey] = pages + } + } + return pages } private suspend fun fetchMangaJson(manga: Manga): JSONObject { @@ -198,19 +230,29 @@ internal class MangaMoins(context: MangaLoaderContext) : page to mtime } .sortedBy { it.first } - .map { (page, mtime) -> - val pageName = page.toString().padStart(2, '0') - "/files/scans/$folder/$pageName.png?v=$mtime" - } - .distinct() - return pairs.map { pageUrl -> + .mapNotNull { (page, mtime) -> + val pageName = page.toString().padStart(2, '0') + sanitizePageUrl("/files/scans/$folder/$pageName.png?v=$mtime") + } + .distinct() + return pairs.map { pageUrl -> MangaPage( id = generateUid(pageUrl), url = pageUrl, preview = null, source = source, ) - }.toList() + }.toList() + } + + private fun sanitizePageUrl(raw: String?): String? { + val candidate = raw?.trim()?.takeIf { it.isNotEmpty() } ?: return null + val absolute = candidate.toAbsoluteUrl(domain) + val url = absolute.toHttpUrlOrNull() ?: return null + if (url.queryParameterNames.contains("v") && url.queryParameter("v").isNullOrBlank()) { + return null + } + return url.newBuilder().build().toString() } private fun parseState(status: String?): MangaState? { @@ -303,6 +345,15 @@ internal class MangaMoins(context: MangaLoaderContext) : return DETAILS_SCORE_EMPTY } + private fun buildDetailsCacheKey(manga: Manga): String { + return ( + extractSlugFromMangaUrl(manga.url) + ?: extractSlugFromMangaUrl(manga.publicUrl) + ?: manga.title.toMangaSlug() + ) + .lowercase(Locale.ROOT) + } + private fun extractScanFolder(url: String): String? { val httpUrl = url.toAbsoluteUrl(domain).toHttpUrlOrNull() ?: return null httpUrl.queryParameter("scan")?.takeIf { it.isNotBlank() }?.let { return it } @@ -344,10 +395,12 @@ internal class MangaMoins(context: MangaLoaderContext) : private const val API_LIMIT = 12 private const val UNKNOWN_MANGA_TITLE = "Unknown manga" - private const val DETAILS_SCORE_EMPTY = 0 - private const val DETAILS_SCORE_WITH_METADATA = 1 - private const val DETAILS_SCORE_WITH_CHAPTERS = 2 - private val AUTHORS_SPLIT_PATTERN = Regex("[,&/]") + private const val DETAILS_SCORE_EMPTY = 0 + private const val DETAILS_SCORE_WITH_METADATA = 1 + private const val DETAILS_SCORE_WITH_CHAPTERS = 2 + private const val DETAILS_CACHE_SIZE = 200 + private const val PAGES_CACHE_SIZE = 400 + private val AUTHORS_SPLIT_PATTERN = Regex("[,&/]") private val WHITESPACES_REGEX = Regex("\\s+") private val CHAPTER_NUMBER_REGEX = Regex("(\\d+(?:\\.\\d+)?)$") private val IMAGE_MTIMES_BLOCK = Regex("imageMtimes\\s*=\\s*\\{([^}]*)\\}") diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScans.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScans.kt index c5fc5bf..f163a08 100644 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScans.kt +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScans.kt @@ -1,12 +1,12 @@ package org.dokiteam.doki.parsers.site.fr +import org.json.JSONArray import org.json.JSONObject import org.jsoup.nodes.Document -import org.dokiteam.doki.parsers.Broken import org.dokiteam.doki.parsers.MangaLoaderContext import org.dokiteam.doki.parsers.MangaSourceParser import org.dokiteam.doki.parsers.config.ConfigKey -import org.dokiteam.doki.parsers.core.SinglePageMangaParser +import org.dokiteam.doki.parsers.core.PagedMangaParser import org.dokiteam.doki.parsers.model.ContentRating import org.dokiteam.doki.parsers.model.ContentType import org.dokiteam.doki.parsers.model.Manga @@ -25,6 +25,7 @@ import org.dokiteam.doki.parsers.util.json.getStringOrNull import org.dokiteam.doki.parsers.util.json.mapJSON import org.dokiteam.doki.parsers.util.json.mapJSONNotNull import org.dokiteam.doki.parsers.util.parseHtml +import org.dokiteam.doki.parsers.util.parseJson import org.dokiteam.doki.parsers.util.parseSafe import org.dokiteam.doki.parsers.util.suspendlazy.suspendLazy import org.dokiteam.doki.parsers.util.toAbsoluteUrl @@ -33,12 +34,11 @@ import java.util.EnumSet import java.util.Locale import java.util.TimeZone -@Broken("The source to change structure") @MangaSourceParser("POSEIDONSCANS", "Poseidon Scans", "fr") internal class PoseidonScans(context: MangaLoaderContext) : - SinglePageMangaParser(context, MangaParserSource.POSEIDONSCANS) { + PagedMangaParser(context, MangaParserSource.POSEIDONSCANS, 24) { - override val configKeyDomain = ConfigKey.Domain("poseidonscans.com") + override val configKeyDomain = ConfigKey.Domain("poseidon-scans.co") override val availableSortOrders: Set = EnumSet.of( SortOrder.UPDATED, @@ -59,22 +59,46 @@ internal class PoseidonScans(context: MangaLoaderContext) : timeZone = TimeZone.getTimeZone("UTC") } - private val nextFPushRegex = - Regex("""self\.__next_f\.push\(\s*\[\s*1\s*,\s*"(.*)"\s*]\s*\)""", RegexOption.DOT_MATCHES_ALL) + private val nextFPushRegex = Regex( + """self\.__next_f\.push\(\s*\[\s*1\s*,\s*"((?:\\.|[^"\\])*)"\s*]\s*\)""", + RegexOption.DOT_MATCHES_ALL, + ) + private val chapterRouteRegex = Regex("""(?:https?://[^/]+)?/serie/([^/]+)/chapter/([^/?#]+)""") - // Helper data class to hold extra info for sorting + // Helper data classes for sorting and chapter recovery. private data class MangaCache( val manga: Manga, val viewCount: Int, val type: ContentType, val latestChapterDate: Long, + val chaptersCount: Int, + ) + private data class ChapterSeed( + val raw: String, + val value: Double, + ) + private data class ApiChapterEntry( + val id: String?, + val numberRaw: String, + val numberValue: Double, + val title: String?, + val createdAt: String?, + ) + private data class ChapterRoute( + val slug: String, + val chapterNumberRaw: String, ) // Cache all manga for local search and sorting private val allMangaCache = suspendLazy { - val doc = webClient.httpGet("https://$domain/series").parseHtml() - extractFullMangaListFromDocument(doc) + runCatching { + fetchAllMangaFromApi() + }.getOrElse { + // Fallback if API route changes: parse homepage Next.js payload. + val doc = webClient.httpGet("https://$domain").parseHtml() + extractFullMangaListFromDocument(doc) + } } // Cache all available tags for filtering @@ -84,6 +108,23 @@ internal class PoseidonScans(context: MangaLoaderContext) : .toSet() } + // Cache fallback chapters from last updates API (used when series page is blocked by Cloudflare) + private val recentChaptersCache = suspendLazy { + fetchRecentChaptersBySlugFromApi() + } + + // Popularity endpoint exposes stable metrics (viewCount/favorites) unlike /api/manga/all. + private val popularityBySlugCache = suspendLazy { + fetchPopularityBySlugFromApi() + } + + // Latest endpoint gives the current chapter number per slug (seed for full chapter list API). + private val latestChapterSeedBySlugCache = suspendLazy { + fetchLatestChapterSeedBySlugFromApi() + } + private val apiChaptersBySlugCache = HashMap>() + private val chapterPagesCache = HashMap>() + override suspend fun getFilterOptions() = MangaListFilterOptions( availableTags = allTagsCache.get(), availableStates = EnumSet.of( @@ -99,7 +140,17 @@ internal class PoseidonScans(context: MangaLoaderContext) : ), ) - override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val all = getFilteredAndSortedList(order, filter) + val fromIndex = ((page - 1) * pageSize).coerceAtLeast(0) + if (fromIndex >= all.size) { + return emptyList() + } + val toIndex = (fromIndex + pageSize).coerceAtMost(all.size) + return all.subList(fromIndex, toIndex) + } + + private suspend fun getFilteredAndSortedList(order: SortOrder, filter: MangaListFilter): List { var mangaCacheList = allMangaCache.get() if (!filter.query.isNullOrEmpty()) { @@ -128,7 +179,9 @@ internal class PoseidonScans(context: MangaLoaderContext) : // Tag inclusion filter if (filter.tags.isNotEmpty()) { - mangaCacheList = mangaCacheList.filter { it.manga.tags.containsAll(filter.tags) } + mangaCacheList = mangaCacheList.filter { cachedManga -> + cachedManga.manga.tags.any { tag -> filter.tags.contains(tag) } + } } // Tag exclusion filter @@ -138,6 +191,9 @@ internal class PoseidonScans(context: MangaLoaderContext) : } } + // Hide entries without chapters to avoid unusable detail pages. + mangaCacheList = mangaCacheList.filter { it.chaptersCount > 0 } + // Sorting val sortedCachedMangaList = when (order) { SortOrder.UPDATED -> mangaCacheList.sortedByDescending { it.latestChapterDate } @@ -154,6 +210,240 @@ internal class PoseidonScans(context: MangaLoaderContext) : return sortedCachedMangaList.map { it.manga } } + private suspend fun fetchAllMangaFromApi(): List { + val payload = webClient.httpGet("https://$domain/api/manga/all").parseJson() + val mangasArray = payload.optJSONArray("data") ?: return emptyList() + val popularityBySlug = runCatching { popularityBySlugCache.get() }.getOrDefault(emptyMap()) + return mangasArray.mapJSON { mangaJson -> + val slug = mangaJson.getStringOrNull("slug").orEmpty() + parseMangaDetailsFromJson(mangaJson, popularityBySlug[slug]) + } + } + + private suspend fun fetchPopularityBySlugFromApi(): Map { + val payload = webClient.httpGet("https://$domain/api/manga/popular?limit=500").parseJson() + val mangasArray = payload.optJSONArray("data") ?: return emptyMap() + return mangasArray.mapJSONNotNull { mangaJson -> + val slug = mangaJson.getStringOrNull("slug")?.takeIf { it.isNotBlank() } ?: return@mapJSONNotNull null + val viewCount = mangaJson.optInt("viewCount", 0) + val favorites = mangaJson.optJSONObject("_count")?.optInt("favorites", 0) ?: 0 + slug to maxOf(viewCount, favorites) + }.toMap() + } + + private suspend fun fetchLatestChapterSeedBySlugFromApi(): Map { + val payload = webClient.httpGet("https://$domain/api/manga/latest?limit=500&page=1").parseJson() + val mangasArray = payload.optJSONArray("data") ?: return emptyMap() + return mangasArray.mapJSONNotNull { mangaJson -> + val slug = mangaJson.getStringOrNull("slug")?.takeIf { it.isNotBlank() } ?: return@mapJSONNotNull null + val chaptersArray = mangaJson.optJSONArray("chapters") ?: return@mapJSONNotNull null + val latestChapter = + chaptersArray.mapJSONNotNull { chapterJson -> + parseChapterSeed(chapterJson.opt("number")) + }.maxByOrNull { it.value } ?: return@mapJSONNotNull null + slug to latestChapter + }.toMap() + } + + private suspend fun fetchRecentChaptersBySlugFromApi(): Map> { + val payload = webClient.httpGet("https://$domain/api/manga/lastchapters?limit=500&page=1").parseJson() + val mangasArray = payload.optJSONArray("data") ?: return emptyMap() + return mangasArray.mapJSONNotNull { mangaJson -> + val slug = mangaJson.getStringOrNull("slug")?.takeIf { it.isNotBlank() } ?: return@mapJSONNotNull null + val chaptersArray = mangaJson.optJSONArray("chapters") ?: return@mapJSONNotNull null + val chapters = chaptersArray.mapJSONNotNull { chapterJson -> + if (chapterJson.optBoolean("isPremium", false)) { + return@mapJSONNotNull null + } + val chapterNumber = chapterJson.optDouble("number", 0.0).toFloat() + val chapterNumberRaw = chapterJson.opt("number")?.toString()?.trim() + val chapterNumberForUrl = + chapterNumberRaw?.takeIf { it.isNotEmpty() && it != "null" } ?: formatChapterNumber(chapterNumber) + val chapterUrl = "/serie/$slug/chapter/$chapterNumberForUrl" + val chapterTitleRaw = chapterJson.getStringOrNull("title")?.takeIf { it.isNotBlank() && it != "null" } + val chapterTitle = if (chapterTitleRaw != null) { + "Chapitre ${formatChapterNumber(chapterNumber)} - $chapterTitleRaw" + } else { + "Chapitre ${formatChapterNumber(chapterNumber)}" + } + MangaChapter( + id = generateUid("${chapterUrl}#${chapterJson.getStringOrNull("id") ?: chapterNumberForUrl}"), + title = chapterTitle, + number = chapterNumber, + volume = 0, + url = chapterUrl, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + }.reversed() + if (chapters.isEmpty()) { + null + } else { + slug to chapters + } + }.toMap() + } + + private suspend fun fetchAllChaptersBySlugFromApi(slug: String): List { + if (slug.isBlank()) return emptyList() + val chapterSeedCandidates = buildChapterSeedCandidates(slug) + for (seed in chapterSeedCandidates) { + val chapters = runCatching { + fetchChapterListBySeed(slug, seed) + }.getOrNull().orEmpty() + if (chapters.isNotEmpty()) { + return chapters + } + } + return emptyList() + } + + private suspend fun buildChapterSeedCandidates(slug: String): List { + val candidatesByRaw = LinkedHashMap() + + fun addCandidate(seed: ChapterSeed?) { + if (seed == null) return + if (seed.value <= 0.0) return + candidatesByRaw.putIfAbsent(seed.raw, seed) + } + + addCandidate(latestChapterSeedBySlugCache.get()[slug]) + addCandidate( + recentChaptersCache.get()[slug].orEmpty().maxByOrNull { it.number }?.let { + ChapterSeed(formatChapterNumber(it.number), it.number.toDouble()) + }, + ) + addCandidate( + allMangaCache.get().firstOrNull { it.manga.url == "/serie/$slug" }?.chaptersCount + ?.takeIf { it > 0 }?.let { ChapterSeed(it.toString(), it.toDouble()) }, + ) + addCandidate(ChapterSeed("1", 1.0)) + return candidatesByRaw.values.toList() + } + + private suspend fun fetchChapterListBySeed(slug: String, seed: ChapterSeed): List { + val payload = webClient.httpGet("https://$domain/api/manga/$slug/${seed.raw}").parseJson() + val data = payload.optJSONObject("data") ?: return emptyList() + val chapterListArray = data.optJSONArray("chapterList") ?: return emptyList() + val chapterEntries = chapterListArray.mapJSONNotNull { parseApiChapterEntry(it) }.sortedBy { it.numberValue } + if (chapterEntries.isEmpty()) { + return emptyList() + } + + val premiumStatusCache = mutableMapOf() + val chapterData = data.optJSONObject("chapterData") + if (chapterData != null) { + parseChapterSeed(chapterData.opt("number"))?.let { chapterSeed -> + premiumStatusCache[chapterSeed.raw] = chapterData.optBoolean("isPremium", false) + } + } + + val firstPremiumChapterIndex = findFirstPremiumChapterIndex(slug, chapterEntries, premiumStatusCache) + val readableChapters = if (firstPremiumChapterIndex != null) { + chapterEntries.subList(0, firstPremiumChapterIndex) + } else { + chapterEntries + } + + return readableChapters.map { chapterEntry -> + buildMangaChapterFromApiChapter(slug, chapterEntry) + } + } + + private fun parseApiChapterEntry(chapterJson: JSONObject): ApiChapterEntry? { + val chapterSeed = parseChapterSeed(chapterJson.opt("number")) ?: return null + return ApiChapterEntry( + id = chapterJson.getStringOrNull("id"), + numberRaw = chapterSeed.raw, + numberValue = chapterSeed.value, + title = chapterJson.getStringOrNull("title")?.takeIf { it.isNotBlank() && it != "null" }, + createdAt = chapterJson.getStringOrNull("createdAt"), + ) + } + + private suspend fun findFirstPremiumChapterIndex( + slug: String, + chapterEntries: List, + premiumStatusCache: MutableMap, + ): Int? { + if (chapterEntries.isEmpty()) return null + + val latestChapter = chapterEntries.last() + val latestChapterIsPremium = + getChapterPremiumStatus(slug, latestChapter.numberRaw, premiumStatusCache) ?: return null + if (!latestChapterIsPremium) { + return null + } + + var low = 0 + var high = chapterEntries.lastIndex + var firstPremiumIndex = chapterEntries.lastIndex + + while (low <= high) { + val mid = (low + high) ushr 1 + val chapterIsPremium = + getChapterPremiumStatus(slug, chapterEntries[mid].numberRaw, premiumStatusCache) ?: return null + if (chapterIsPremium) { + firstPremiumIndex = mid + high = mid - 1 + } else { + low = mid + 1 + } + } + return firstPremiumIndex + } + + private suspend fun getChapterPremiumStatus( + slug: String, + chapterNumberRaw: String, + premiumStatusCache: MutableMap, + ): Boolean? { + premiumStatusCache[chapterNumberRaw]?.let { return it } + + return runCatching { + val payload = webClient.httpGet("https://$domain/api/manga/$slug/$chapterNumberRaw").parseJson() + val isPremium = payload.optJSONObject("data") + ?.optJSONObject("chapterData") + ?.optBoolean("isPremium", false) ?: false + premiumStatusCache[chapterNumberRaw] = isPremium + isPremium + }.getOrNull() + } + + private fun buildMangaChapterFromApiChapter(slug: String, chapterEntry: ApiChapterEntry): MangaChapter { + val chapterNumber = chapterEntry.numberValue.toFloat() + val chapterTitle = if (!chapterEntry.title.isNullOrBlank()) { + "Chapitre ${formatChapterNumber(chapterNumber)} - ${chapterEntry.title}" + } else { + "Chapitre ${formatChapterNumber(chapterNumber)}" + } + val chapterUrl = "/serie/$slug/chapter/${chapterEntry.numberRaw}" + return MangaChapter( + id = generateUid("${chapterUrl}#${chapterEntry.id ?: chapterEntry.numberRaw}"), + title = chapterTitle, + number = chapterNumber, + volume = 0, + url = chapterUrl, + uploadDate = parseDate(chapterEntry.createdAt), + source = source, + scanlator = null, + branch = null, + ) + } + + private fun parseChapterSeed(rawNumber: Any?): ChapterSeed? { + val rawText = rawNumber?.toString()?.trim()?.takeIf { it.isNotBlank() && it != "null" } ?: return null + val value = rawText.toDoubleOrNull() ?: return null + val normalizedRaw = if (value % 1.0 == 0.0) { + value.toInt().toString() + } else { + rawText + } + return ChapterSeed(raw = normalizedRaw, value = value) + } + private fun extractFullMangaListFromDocument(doc: Document): List { val pageData = extractNextJsPageData(doc) @@ -162,14 +452,20 @@ internal class PoseidonScans(context: MangaLoaderContext) : ?.optJSONArray("mangas") ?: it.optJSONObject("initialData")?.optJSONArray("series") } ?: return emptyList() - return mangasArray.mapJSON { parseMangaDetailsFromJson(it) } + return mangasArray.mapJSON { parseMangaDetailsFromJson(it, popularityScore = null) } } - private fun parseMangaDetailsFromJson(mangaJson: JSONObject): MangaCache { + private fun parseMangaDetailsFromJson(mangaJson: JSONObject, popularityScore: Int?): MangaCache { val slug = mangaJson.getString("slug") val url = "/serie/$slug" - val coverUrl = "https://$domain/api/covers/$slug.webp" + val coverImageRaw = mangaJson.getStringOrNull("coverImage") + val coverUrl = when { + coverImageRaw.isNullOrBlank() || coverImageRaw == "null" -> "https://$domain/api/covers/$slug.webp" + coverImageRaw.startsWith("http://") || coverImageRaw.startsWith("https://") -> coverImageRaw + coverImageRaw.startsWith("storage/") || coverImageRaw.startsWith("/storage/") -> "https://$domain/api/covers/$slug.webp" + else -> coverImageRaw.toAbsoluteUrl(domain) + } val authors = mangaJson.getStringOrNull("author")?.takeIf { it.isNotBlank() && it != "null" }?.split(',') ?.map(String::trim)?.toSet() ?: emptySet() @@ -212,8 +508,13 @@ internal class PoseidonScans(context: MangaLoaderContext) : ?.takeIf { it.isNotBlank() && it != "null" && it != "Aucune description." }, source = source, ) - val viewCount = mangaJson.optInt("viewCount", 0) + val viewCount = when { + popularityScore != null && popularityScore > 0 -> popularityScore + mangaJson.has("viewCount") -> mangaJson.optInt("viewCount", 0) + else -> mangaJson.optJSONObject("_count")?.optInt("favorites", 0) ?: 0 + } val latestChapterDate = parseDate(mangaJson.optString("latestChapterCreatedAt")) + val chaptersCount = mangaJson.optJSONObject("_count")?.optInt("chapters", 1) ?: 1 val type = when (mangaJson.optString("type", "").lowercase(sourceLocale)) { "manhwa" -> ContentType.MANHWA "manhua" -> ContentType.MANHUA @@ -226,6 +527,7 @@ internal class PoseidonScans(context: MangaLoaderContext) : viewCount = viewCount, latestChapterDate = latestChapterDate, type = type, + chaptersCount = chaptersCount, ) } @@ -234,19 +536,51 @@ internal class PoseidonScans(context: MangaLoaderContext) : return manga } - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val pageData = extractNextJsPageData(doc) ?: throw Exception("Could not extract Next.js data for manga details") - - val mangaDetailsJson = - pageData.optJSONObject("manga") ?: pageData.optJSONObject("initialData")?.optJSONObject("manga") - ?: pageData.takeIf { it.has("slug") && it.has("title") } - ?: throw Exception("JSON 'manga' structure not found") + // Preferred path: parse series page (full chapters list). + val pageChapters = runCatching { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val pageData = extractNextJsPageData(doc) ?: throw Exception("Could not extract Next.js data for manga details") + val mangaDetailsJson = + pageData.optJSONObject("manga") ?: pageData.optJSONObject("initialData")?.optJSONObject("manga") + ?: pageData.takeIf { it.has("slug") && it.has("title") } + ?: throw Exception("JSON 'manga' structure not found") + extractChaptersFromMangaData(mangaDetailsJson, manga) + }.getOrNull().orEmpty() + if (pageChapters.isNotEmpty()) { + return manga.copy( + description = manga.description ?: "", + chapters = pageChapters, + ) + } - // All details are already in the manga object. We only need to fetch the chapters. - val chapters = extractChaptersFromMangaData(mangaDetailsJson, manga) + // Fallback path: use API chapter list endpoint (works without parsing the series HTML). + val slug = manga.url.substringAfter("/serie/").substringBefore("/").ifBlank { + manga.publicUrl.substringAfter("/serie/").substringBefore("/") + } + val cachedManga = if (slug.isBlank()) { + null + } else { + allMangaCache.get().firstOrNull { it.manga.url == "/serie/$slug" }?.manga + } + val base = cachedManga ?: manga + val apiChapters = if (slug.isBlank()) { + emptyList() + } else { + fetchAllChaptersBySlugFromApiCached(slug) + } + if (apiChapters.isNotEmpty()) { + return base.copy( + description = base.description ?: "", + chapters = apiChapters, + ) + } - return manga.copy(chapters = chapters) + // Last-resort fallback: lastchapters endpoint (usually only latest 2 chapters). + val fallbackChapters = if (slug.isBlank()) emptyList() else recentChaptersCache.get()[slug].orEmpty() + return base.copy( + description = base.description ?: "", + chapters = fallbackChapters, + ) } private fun extractChaptersFromMangaData(mangaJson: JSONObject, manga: Manga): List { @@ -262,7 +596,10 @@ internal class PoseidonScans(context: MangaLoaderContext) : val createdAt = chapterJson.optString("createdAt", "").takeIf(String::isNotEmpty) ?: chapterJson.optString("publishedAt", "") - val chapterUrl = "/serie/$slug/chapter/${chapterNumber.toInt()}" + val chapterNumberRaw = chapterJson.opt("number")?.toString()?.trim() + val chapterNumberForUrl = + chapterNumberRaw?.takeIf { it.isNotEmpty() && it != "null" } ?: formatChapterNumber(chapterNumber) + val chapterUrl = "/serie/$slug/chapter/$chapterNumberForUrl" val uploadDate = parseDate(createdAt) val chapterTitle = if (title.isNotBlank() && title != "null") { @@ -294,28 +631,159 @@ internal class PoseidonScans(context: MangaLoaderContext) : } override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + synchronized(chapterPagesCache) { + chapterPagesCache[chapter.url]?.let { return it } + } + val apiPages = fetchChapterPagesFromApi(chapter) + if (apiPages.isNotEmpty()) { + synchronized(chapterPagesCache) { + chapterPagesCache[chapter.url] = apiPages + } + return apiPages + } + val fallbackPages = runCatching { + val directDoc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + extractPagesFromDocument(chapter, directDoc) + }.getOrDefault(emptyList()) + if (fallbackPages.isNotEmpty()) { + synchronized(chapterPagesCache) { + chapterPagesCache[chapter.url] = fallbackPages + } + } + return fallbackPages + } - val pageData = extractNextJsPageData(doc) ?: throw Exception("Could not extract Next.js data for chapter pages") + private suspend fun fetchAllChaptersBySlugFromApiCached(slug: String): List { + synchronized(apiChaptersBySlugCache) { + apiChaptersBySlugCache[slug]?.let { return it } + } + val chapters = fetchAllChaptersBySlugFromApi(slug) + synchronized(apiChaptersBySlugCache) { + apiChaptersBySlugCache[slug] = chapters + } + return chapters + } - val imagesArray = - pageData.optJSONObject("initialData")?.optJSONArray("images") ?: pageData.optJSONArray("images") - ?: throw Exception("Could not find 'images' array in page data") + private suspend fun fetchChapterPagesFromApi(chapter: MangaChapter): List { + val chapterRoute = parseChapterRoute(chapter.url) ?: return emptyList() + val chapterData = runCatching { + webClient.httpGet("https://$domain/api/manga/${chapterRoute.slug}/${chapterRoute.chapterNumberRaw}") + .parseJson() + }.getOrNull() + ?.optJSONObject("data") + ?.optJSONObject("chapterData") ?: return emptyList() + if (chapterData.optBoolean("isPremium", false)) { + return emptyList() + } + val chapterId = chapterData.getStringOrNull("id")?.takeIf { it.isNotBlank() } ?: return emptyList() + val imagesArray = runCatching { + webClient.httpGet("https://$domain/api/chapters/${chapterRoute.slug}/$chapterId/images") + .parseJson() + }.getOrNull() + ?.let { payload -> + payload.optJSONArray("images") + ?: payload.optJSONObject("data")?.optJSONArray("images") + } ?: return emptyList() + + val indexedUrls = mutableListOf>() + for (index in 0 until imagesArray.length()) { + val imageEntry = imagesArray.opt(index) + val rawUrl = when (imageEntry) { + is JSONObject -> imageEntry.getStringOrNull("url") + ?: imageEntry.getStringOrNull("originalUrl") + ?: imageEntry.getStringOrNull("src") + is String -> imageEntry + else -> null + } ?: continue + val normalizedUrl = normalizePageImageUrl(rawUrl) ?: continue + val pageIndex = (imageEntry as? JSONObject)?.optInt("index", index) ?: index + indexedUrls.add(pageIndex to normalizedUrl) + } + if (indexedUrls.isEmpty()) return emptyList() - return imagesArray.mapJSON { imageJson -> - val imageUrl = imageJson.getString("originalUrl") + return indexedUrls.sortedBy { it.first }.map { it.second }.distinct().map { imageUrl -> MangaPage( id = generateUid(imageUrl), - url = imageUrl.toAbsoluteUrl(domain), + url = imageUrl, preview = null, source = chapter.source, ) } } + private fun parseChapterRoute(chapterUrl: String): ChapterRoute? { + val routeMatch = chapterRouteRegex.find(chapterUrl) ?: return null + val slug = routeMatch.groupValues.getOrNull(1)?.trim().orEmpty() + val chapterNumberRaw = routeMatch.groupValues.getOrNull(2)?.trim().orEmpty() + if (slug.isEmpty() || chapterNumberRaw.isEmpty()) { + return null + } + return ChapterRoute( + slug = slug, + chapterNumberRaw = chapterNumberRaw, + ) + } + + private fun extractPagesFromDocument(chapter: MangaChapter, document: Document): List { + val extractedUrls = LinkedHashSet() + + val pageData = extractNextJsPageData(document) + val imagesArray = pageData?.let { findBestImagesArray(it) } + if (imagesArray != null) { + for (index in 0 until imagesArray.length()) { + val imageUrl = extractImageUrlFromArrayItem(imagesArray.opt(index)) ?: continue + extractedUrls.add(imageUrl) + } + } + + if (extractedUrls.isEmpty()) { + extractedUrls += extractImageUrlsFromChapterHtml(document) + } + + if (extractedUrls.isEmpty()) return emptyList() + + val slug = chapter.url.substringAfter("/serie/").substringBefore("/chapter/").lowercase(Locale.ROOT) + val chapterScopedUrls = if (slug.isBlank()) { + emptyList() + } else { + extractedUrls.filter { url -> + val lower = url.lowercase(Locale.ROOT) + lower.contains("/$slug/") || lower.contains("/mangas/$slug/") || lower.contains("/api/chapters/$slug/") + } + } + val finalUrls = if (chapterScopedUrls.isNotEmpty()) chapterScopedUrls else extractedUrls + return finalUrls.map { imageUrl -> + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = chapter.source, + ) + } + } + + private fun extractImageUrlsFromChapterHtml(document: Document): List { + val urls = LinkedHashSet() + + fun extractFromSelector(selector: String, attribute: String) { + document.select(selector).forEach { element -> + val value = element.attr(attribute).trim() + val normalized = normalizePageImageUrl(value) + normalized?.let { urls.add(it) } + } + } + + extractFromSelector("a[href*=\"/api/chapters/\"]", "href") + extractFromSelector("img[src*=\"/api/chapters/\"]", "src") + extractFromSelector("link[href*=\"/api/chapters/\"]", "href") + + return urls.toList() + } + private fun extractNextJsPageData(document: Document): JSONObject? { try { - var foundRelevantObject: JSONObject? = null + var bestObject: JSONObject? = null + var bestImageCount = 0 for (script in document.select("script")) { val scriptContent = script.data() if (!scriptContent.contains("self.__next_f.push")) continue @@ -326,6 +794,7 @@ internal class PoseidonScans(context: MangaLoaderContext) : val rawDataString = matchResult.groupValues[1] val cleanedDataString = rawDataString.replace("\\\\", "\\").replace("\\\"", "\"") + val seenObjectStarts = HashSet() val patterns = listOf( "\"initialData\":{", @@ -334,6 +803,7 @@ internal class PoseidonScans(context: MangaLoaderContext) : "\"series\":[", "\"chapter\":{", "\"images\":[", + "\"pages\":[", ) for (pattern in patterns) { @@ -342,46 +812,125 @@ internal class PoseidonScans(context: MangaLoaderContext) : searchIdx = cleanedDataString.indexOf(pattern, startIndex = searchIdx + 1) if (searchIdx == -1) break - // Find the start of the JSON object - var objectStartIndex = -1 - var braceDepth = 0 - for (i in searchIdx downTo 0) { - when (cleanedDataString[i]) { - '}' -> braceDepth++ - '{' -> { - if (braceDepth == 0) { - objectStartIndex = i - break - } - braceDepth-- - } - } - } - - if (objectStartIndex != -1) { - val potentialJson = extractJsonObjectString(cleanedDataString, objectStartIndex) - if (potentialJson != null) { - try { - val parsedContainer = JSONObject(potentialJson) - foundRelevantObject = parsedContainer - break - } catch (_: Exception) { - // Continue searching + val objectStartIndex = findJsonObjectStart(cleanedDataString, searchIdx) + if (objectStartIndex == -1 || !seenObjectStarts.add(objectStartIndex)) continue + + val potentialJson = extractJsonObjectString(cleanedDataString, objectStartIndex) + if (potentialJson != null) { + try { + val parsedContainer = JSONObject(potentialJson) + val imageCount = findBestImagesArray(parsedContainer)?.let { countImageEntries(it) } ?: 0 + if ( + bestObject == null || + imageCount > bestImageCount + ) { + bestObject = parsedContainer + bestImageCount = imageCount } + } catch (_: Exception) { + // Continue searching } } } - if (foundRelevantObject != null) break } - if (foundRelevantObject != null) break } - if (foundRelevantObject != null) break } - return foundRelevantObject - } catch (e: Exception) { - error("General error in extractNextJsPageData: ${e.message}") + return bestObject + } catch (_: Exception) { + return null + } + } + + private fun findJsonObjectStart(data: String, fromIndex: Int): Int { + var braceDepth = 0 + for (i in fromIndex downTo 0) { + when (data[i]) { + '}' -> braceDepth++ + '{' -> { + if (braceDepth == 0) { + return i + } + braceDepth-- + } + } + } + return -1 + } + + private fun findBestImagesArray(node: Any?): JSONArray? { + var bestArray: JSONArray? = null + var bestCount = 0 + + fun visit(value: Any?) { + when (value) { + is JSONObject -> { + val keys = value.keys() + while (keys.hasNext()) { + visit(value.opt(keys.next())) + } + } + is JSONArray -> { + val count = countImageEntries(value) + if (count > bestCount) { + bestCount = count + bestArray = value + } + for (index in 0 until value.length()) { + visit(value.opt(index)) + } + } + } + } + + visit(node) + return if (bestCount > 0) bestArray else null + } + + private fun countImageEntries(array: JSONArray): Int { + var count = 0 + for (index in 0 until array.length()) { + if (extractImageUrlFromArrayItem(array.opt(index)) != null) { + count++ + } + } + return count + } + + private fun extractImageUrlFromArrayItem(item: Any?): String? { + val raw = when (item) { + is JSONObject -> { + item.getStringOrNull("originalUrl") + ?: item.getStringOrNull("url") + ?: item.getStringOrNull("src") + } + is String -> item + else -> null + } ?: return null + return normalizePageImageUrl(raw) + } + + private fun normalizePageImageUrl(rawUrl: String): String? { + val value = rawUrl.trim() + .replace("\\/", "/") + .takeIf { it.isNotEmpty() && it != "null" } ?: return null + val lowerValue = value.lowercase(Locale.ROOT) + if (lowerValue.contains("/api/covers/")) return null + if (lowerValue.contains("/preview.")) return null + if (!looksLikeImageUrl(lowerValue)) return null + return value.toAbsoluteUrl(domain) + } + + private fun looksLikeImageUrl(valueLowercase: String): Boolean { + if (valueLowercase.endsWith(".jpg") || valueLowercase.endsWith(".jpeg") || valueLowercase.endsWith(".png") || + valueLowercase.endsWith(".webp") || valueLowercase.endsWith(".avif") || valueLowercase.endsWith(".gif") + ) { + return true + } + if (valueLowercase.contains("/api/chapters/")) { + return true } + return valueLowercase.contains("/storage/") && valueLowercase.contains("/mangas/") } private fun extractJsonObjectString(data: String, startIndex: Int): String? { diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/PunkRecordz.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/PunkRecordz.kt new file mode 100644 index 0000000..ba7e99f --- /dev/null +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/fr/PunkRecordz.kt @@ -0,0 +1,654 @@ +package org.dokiteam.doki.parsers.site.fr + +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.json.JSONArray +import org.json.JSONObject +import org.dokiteam.doki.parsers.MangaLoaderContext +import org.dokiteam.doki.parsers.MangaSourceParser +import org.dokiteam.doki.parsers.config.ConfigKey +import org.dokiteam.doki.parsers.core.PagedMangaParser +import org.dokiteam.doki.parsers.model.ContentType +import org.dokiteam.doki.parsers.model.Manga +import org.dokiteam.doki.parsers.model.MangaChapter +import org.dokiteam.doki.parsers.model.MangaListFilter +import org.dokiteam.doki.parsers.model.MangaListFilterCapabilities +import org.dokiteam.doki.parsers.model.MangaListFilterOptions +import org.dokiteam.doki.parsers.model.MangaPage +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.MangaState +import org.dokiteam.doki.parsers.model.MangaTag +import org.dokiteam.doki.parsers.model.RATING_UNKNOWN +import org.dokiteam.doki.parsers.model.SortOrder +import org.dokiteam.doki.parsers.util.generateUid +import org.dokiteam.doki.parsers.util.json.getStringOrNull +import org.dokiteam.doki.parsers.util.json.mapJSONNotNull +import org.dokiteam.doki.parsers.util.parseJson +import org.dokiteam.doki.parsers.util.parseSafe +import org.dokiteam.doki.parsers.util.suspendlazy.suspendLazy +import org.dokiteam.doki.parsers.util.toAbsoluteUrl +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.LinkedHashSet +import java.util.Locale +import java.util.TimeZone +import kotlin.text.toBigDecimalOrNull + +@MangaSourceParser("PUNKRECORDZ", "PunkRecordz", "fr") +internal class PunkRecordz(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.PUNKRECORDZ, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("punkrecordz.com") + + override val defaultSortOrder: SortOrder = SortOrder.UPDATED + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.RELEVANCE, + ) + + override val filterCapabilities: MangaListFilterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isAuthorSearchSupported = true, + ) + + private val isoDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + private val allMangaCache = suspendLazy { fetchAllManga() } + private val mangaBySlugCache = HashMap() + private val chaptersBySlugCache = HashMap>() + private val pagesByCacheKey = HashMap>() + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val mangas = allMangaCache.get() + val tags = mangas + .flatMap { it.tags } + .sortedBy { it.title.lowercase(Locale.ROOT) } + .toCollection(LinkedHashSet()) + val states = EnumSet.noneOf(MangaState::class.java).apply { + mangas.forEach { item -> + item.state?.let(::add) + } + } + return MangaListFilterOptions( + availableTags = if (tags.isNotEmpty()) { + tags + } else { + setOf( + MangaTag( + key = ALL_TAG_KEY, + title = "General", + source = source, + ), + ) + }, + availableStates = states, + availableContentTypes = EnumSet.of(ContentType.MANGA), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val filtered = filterManga(allMangaCache.get(), filter) + val sorted = sortManga(filtered, order, filter.query) + val fromIndex = ((page - 1) * pageSize).coerceAtLeast(0) + if (fromIndex >= sorted.size) { + return emptyList() + } + val toIndex = (fromIndex + pageSize).coerceAtMost(sorted.size) + return sorted.subList(fromIndex, toIndex).map { it.toManga() } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = extractSlug(manga.url) ?: extractSlug(manga.publicUrl) ?: return manga + val mangaData = fetchMangaBySlugCached(slug) + val chapters = fetchChaptersCached(slug) + return manga.copy( + title = mangaData?.title ?: manga.title, + coverUrl = mangaData?.coverUrl ?: manga.coverUrl, + tags = mangaData?.tags ?: manga.tags, + state = mangaData?.state ?: manga.state, + description = mangaData?.description ?: manga.description ?: "", + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterId = chapter.url.toAbsoluteUrl(domain).toHttpUrlOrNull()?.queryParameter("id") + val cacheKey = if (chapterId.isNullOrBlank()) chapter.url else "id:$chapterId" + synchronized(pagesByCacheKey) { + pagesByCacheKey[cacheKey]?.let { return it } + } + val pagesJson = if (!chapterId.isNullOrBlank()) { + fetchPagesByChapterId(chapterId) + } else { + val slug = extractSlug(chapter.url) + val chapterNumber = extractChapterNumber(chapter.url) + if (slug.isNullOrBlank() || chapterNumber == null) { + JSONArray() + } else { + fetchPagesBySlugAndNumber(slug, chapterNumber) + } + } + + val pages = pagesJson.mapJSONNotNull { pageJson -> + val image = pageJson.getStringOrNull("colored") ?: pageJson.getStringOrNull("original") + val pageUrl = buildImageUrl(image) ?: return@mapJSONNotNull null + MangaPage( + id = generateUid(pageUrl), + url = pageUrl, + preview = null, + source = source, + ) + } + if (pages.isNotEmpty()) { + synchronized(pagesByCacheKey) { + pagesByCacheKey[cacheKey] = pages + } + } + return pages + } + + private suspend fun fetchAllManga(): List { + val result = ArrayList() + var skip = 0 + while (true) { + val data = graphQl( + query = MANGAS_QUERY, + variables = JSONObject().apply { + put("skip", skip) + put("limit", CATALOG_PAGE_LIMIT) + put("order", -1) + }, + ) + val mangasArray = data.optJSONArray("mangas") ?: break + if (mangasArray.length() == 0) { + break + } + result.addAll(mangasArray.mapJSONNotNull(::parseMangaJson)) + skip += mangasArray.length() + if (mangasArray.length() < CATALOG_PAGE_LIMIT) { + break + } + } + return result.distinctBy { it.slug.lowercase(Locale.ROOT) } + } + + private suspend fun fetchMangaBySlug(slug: String): CatalogManga? { + val data = graphQl( + query = MANGA_QUERY, + variables = JSONObject().apply { + put("slug", slug) + }, + ) + val manga = data.optJSONObject("manga") ?: return null + return parseMangaJson(manga) + } + + private suspend fun fetchMangaBySlugCached(slug: String): CatalogManga? { + synchronized(mangaBySlugCache) { + mangaBySlugCache[slug]?.let { return it } + } + val fetched = fetchMangaBySlug(slug) ?: return null + synchronized(mangaBySlugCache) { + mangaBySlugCache[slug] = fetched + } + return fetched + } + + private suspend fun fetchChapters(slug: String): List { + val result = ArrayList() + val seenNumbers = HashSet() + var skip = 0 + while (true) { + val data = graphQl( + query = CHAPTERS_QUERY, + variables = JSONObject().apply { + put("slug", slug) + put("limit", CHAPTERS_PAGE_LIMIT) + put("skip", skip) + put("order", -1) + }, + ) + val chaptersArray = data.optJSONArray("chapters") ?: break + if (chaptersArray.length() == 0) { + break + } + val pageChapters = chaptersArray.mapJSONNotNull { chapterJson -> + val chapterId = chapterJson.getStringOrNull("id")?.takeIf { it.isNotBlank() } ?: return@mapJSONNotNull null + val chapterNumber = parseChapterNumber(chapterJson.opt("number")) ?: return@mapJSONNotNull null + val chapterNumberLabel = chapterNumber.second + if (!seenNumbers.add(chapterNumberLabel)) { + return@mapJSONNotNull null + } + val uploadDate = parseDate( + chapterJson.getStringOrNull("updateTime") ?: chapterJson.getStringOrNull("insertTime"), + ) + val chapterUrl = "/mangas/$slug/$chapterNumberLabel?id=$chapterId" + MangaChapter( + id = generateUid("$slug#$chapterId"), + title = "Chapitre $chapterNumberLabel", + number = chapterNumber.first, + volume = 0, + url = chapterUrl, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + } + result.addAll(pageChapters) + skip += chaptersArray.length() + if (chaptersArray.length() < CHAPTERS_PAGE_LIMIT) { + break + } + } + return result + } + + private suspend fun fetchChaptersCached(slug: String): List { + synchronized(chaptersBySlugCache) { + chaptersBySlugCache[slug]?.let { return it } + } + val fetched = fetchChapters(slug) + synchronized(chaptersBySlugCache) { + chaptersBySlugCache[slug] = fetched + } + return fetched + } + + private suspend fun fetchPagesByChapterId(chapterId: String): JSONArray { + val data = graphQl( + query = CHAPTER_BY_ID_QUERY, + variables = JSONObject().apply { + put("id", chapterId) + }, + ) + val chapter = data.optJSONObject("chapter") ?: return JSONArray() + if (chapter.optBoolean("deleted", false) || !chapter.optBoolean("published", true)) { + return JSONArray() + } + return chapter.optJSONArray("pages") ?: JSONArray() + } + + private suspend fun fetchPagesBySlugAndNumber(slug: String, chapterNumber: Double): JSONArray { + val data = graphQl( + query = CHAPTER_PAGES_QUERY, + variables = JSONObject().apply { + put("slug", slug) + put("number", chapterNumber) + }, + ) + val chaptersArray = data.optJSONArray("chapters") ?: return JSONArray() + val chapter = chaptersArray.optJSONObject(0) ?: return JSONArray() + if (chapter.optBoolean("deleted", false) || !chapter.optBoolean("published", true)) { + return JSONArray() + } + return chapter.optJSONArray("pages") ?: JSONArray() + } + + private suspend fun graphQl(query: String, variables: JSONObject = JSONObject()): JSONObject { + val body = JSONObject().apply { + put("query", query) + put("variables", variables) + } + val response = webClient.httpPost(apiUrl, body).parseJson() + val errors = response.optJSONArray("errors") + if (errors != null && errors.length() > 0) { + val message = errors.optJSONObject(0)?.optString("message")?.takeIf { it.isNotBlank() } + error("PunkRecordz GraphQL error: ${message ?: "Unknown error"}") + } + return response.optJSONObject("data") ?: JSONObject() + } + + private fun parseMangaJson(mangaJson: JSONObject): CatalogManga? { + val slug = mangaJson.getStringOrNull("slug")?.trim()?.takeIf { it.isNotEmpty() } ?: return null + val title = mangaJson.getStringOrNull("name")?.trim()?.takeIf { it.isNotEmpty() } ?: return null + val tags = parseTags(mangaJson.getStringOrNull("keywords")) + val story = mangaJson.getStringOrNull("story")?.trim()?.takeIf { it.isNotEmpty() } + val status = mangaJson.getStringOrNull("status")?.trim()?.takeIf { it.isNotEmpty() } + return CatalogManga( + slug = slug, + title = title, + coverUrl = buildImageUrl(mangaJson.getStringOrNull("thumb")), + tags = tags, + tagKeys = tags.mapTo(HashSet(tags.size)) { it.key.lowercase(Locale.ROOT) }, + state = parseState(mangaJson.optInt("statusProgression", -1)), + description = story ?: status, + keywords = mangaJson.getStringOrNull("keywords"), + status = status, + updateTime = parseDate(mangaJson.getStringOrNull("updateTime")), + insertTime = parseDate(mangaJson.getStringOrNull("insertTime")), + ) + } + + private fun parseTags(rawKeywords: String?): Set { + if (rawKeywords.isNullOrBlank()) { + return emptySet() + } + return rawKeywords + .split(',') + .mapNotNull { raw -> + val title = raw.trim().replace(Regex("\\s+"), " ") + if (title.isEmpty()) { + return@mapNotNull null + } + val key = normalizeTagKey(title) + if (key.isEmpty()) { + return@mapNotNull null + } + MangaTag( + key = key, + title = title.replaceFirstChar { + if (it.isLowerCase()) { + it.titlecase(sourceLocale) + } else { + it.toString() + } + }, + source = source, + ) + } + .toSet() + } + + private fun filterManga(mangas: List, filter: MangaListFilter): List { + if (mangas.isEmpty()) { + return mangas + } + val query = filter.query?.trim()?.lowercase(sourceLocale).orEmpty() + val author = filter.author?.trim()?.lowercase(sourceLocale).orEmpty() + val includeTags = filter.tags.mapTo(HashSet(filter.tags.size)) { it.key.lowercase(Locale.ROOT) } + val excludeTags = filter.tagsExclude.mapTo(HashSet(filter.tagsExclude.size)) { it.key.lowercase(Locale.ROOT) } + + return mangas.filter { item -> + if (query.isNotEmpty()) { + val haystack = buildString { + append(item.title) + append('\n') + append(item.slug) + item.keywords?.let { + append('\n') + append(it) + } + item.status?.let { + append('\n') + append(it) + } + }.lowercase(sourceLocale) + if (!haystack.contains(query)) { + return@filter false + } + } + + if (author.isNotEmpty()) { + val haystack = listOfNotNull(item.keywords, item.status).joinToString("\n").lowercase(sourceLocale) + if (!haystack.contains(author)) { + return@filter false + } + } + + if (filter.states.isNotEmpty() && item.state !in filter.states) { + return@filter false + } + + if (filter.types.isNotEmpty() && ContentType.MANGA !in filter.types) { + return@filter false + } + + if (includeTags.isNotEmpty() && ALL_TAG_KEY !in includeTags && item.tagKeys.none { it in includeTags }) { + return@filter false + } + + if (excludeTags.isNotEmpty() && ALL_TAG_KEY !in excludeTags && item.tagKeys.any { it in excludeTags }) { + return@filter false + } + + true + } + } + + private fun sortManga(mangas: List, order: SortOrder, query: String?): List { + if (mangas.size < 2) { + return mangas + } + return when (order) { + SortOrder.UPDATED -> mangas.sortedByDescending { it.updateTime } + SortOrder.UPDATED_ASC -> mangas.sortedBy { it.updateTime } + SortOrder.NEWEST -> mangas.sortedByDescending { it.insertTime } + SortOrder.NEWEST_ASC -> mangas.sortedBy { it.insertTime } + SortOrder.ALPHABETICAL -> mangas.sortedBy { it.title.lowercase(sourceLocale) } + SortOrder.ALPHABETICAL_DESC -> mangas.sortedByDescending { it.title.lowercase(sourceLocale) } + SortOrder.RELEVANCE -> { + val normalizedQuery = query?.trim()?.lowercase(sourceLocale).orEmpty() + if (normalizedQuery.isEmpty()) { + mangas.sortedByDescending { it.updateTime } + } else { + mangas.sortedWith( + compareBy { relevanceScore(it, normalizedQuery) } + .thenBy { it.title.length } + .thenBy { it.title.lowercase(sourceLocale) }, + ) + } + } + else -> mangas + } + } + + private fun relevanceScore(manga: CatalogManga, query: String): Int { + val title = manga.title.lowercase(sourceLocale) + if (title == query) return 0 + if (title.startsWith(query)) return 1 + if (title.contains(query)) return 2 + return 3 + } + + private fun parseState(statusProgression: Int): MangaState? = when (statusProgression) { + 4 -> MangaState.FINISHED + 1, 2, 3 -> MangaState.ONGOING + else -> null + } + + private fun buildImageUrl(path: String?): String? { + val value = path?.trim().orEmpty() + if (value.isEmpty() || value == "null") { + return null + } + if (value.startsWith("http://") || value.startsWith("https://")) { + return value + } + val clean = value.removePrefix("/") + return when { + clean.startsWith("images/") -> "https://api.$domain/$clean" + clean.startsWith("webp/") -> "https://api.$domain/images/$clean" + else -> "https://api.$domain/images/webp/$clean.webp" + } + } + + private fun extractSlug(url: String): String? { + val path = url.substringBefore('?').substringBefore('#') + val after = path.substringAfter("/mangas/", "") + if (after.isEmpty()) { + return null + } + return after.substringBefore('/').ifBlank { null } + } + + private fun extractChapterNumber(url: String): Double? { + val path = url.substringBefore('?').substringBefore('#') + val raw = path.substringAfterLast('/', "") + if (raw.isEmpty()) { + return null + } + return raw.toDoubleOrNull() + } + + private fun parseChapterNumber(rawValue: Any?): Pair? { + val raw = rawValue?.toString()?.trim()?.takeIf { it.isNotEmpty() && it != "null" } ?: return null + val decimal = raw.toBigDecimalOrNull() ?: return null + val normalized = decimal.stripTrailingZeros().toPlainString() + return decimal.toFloat() to normalized + } + + private fun normalizeTagKey(tag: String): String { + val normalized = tag + .lowercase(Locale.ROOT) + .replace(Regex("[^\\p{Alnum}]+"), "-") + .trim('-') + return normalized + } + + private fun parseDate(date: String?): Long { + if (date.isNullOrBlank()) { + return 0L + } + return isoDateFormat.parseSafe(date) + } + + private val apiUrl: String + get() = "https://api.$domain/graphql" + + private fun CatalogManga.toManga(): Manga = Manga( + id = generateUid("/mangas/$slug"), + title = title, + altTitles = emptySet(), + url = "/mangas/$slug", + publicUrl = "/mangas/$slug".toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = coverUrl, + tags = tags, + state = state, + authors = emptySet(), + description = description, + source = source, + ) + + private data class CatalogManga( + val slug: String, + val title: String, + val coverUrl: String?, + val tags: Set, + val tagKeys: Set, + val state: MangaState?, + val description: String?, + val keywords: String?, + val status: String?, + val updateTime: Long, + val insertTime: Long, + ) + + private companion object { + const val CATALOG_PAGE_LIMIT = 200 + const val CHAPTERS_PAGE_LIMIT = 200 + const val ALL_TAG_KEY = "__all__" + + val MANGAS_QUERY: String = """ + query Mangas(${'$'}skip: Int!, ${'$'}limit: Int!, ${'$'}order: Float!) { + mangas( + skip: ${'$'}skip + limit: ${'$'}limit + where: { published: true, deleted: false } + order: [{ field: "updateTime", order: ${'$'}order }] + ) { + id + slug + name + thumb + keywords + story + status + statusProgression + updateTime + insertTime + } + } + """.trimIndent() + + val MANGA_QUERY: String = """ + query MangaBySlug(${'$'}slug: String!) { + manga(where: { slug: ${'$'}slug, published: true, deleted: false }) { + id + slug + name + thumb + keywords + story + status + statusProgression + updateTime + insertTime + } + } + """.trimIndent() + + val CHAPTERS_QUERY: String = """ + query Chapters(${'$'}slug: String!, ${'$'}limit: Int!, ${'$'}skip: Int!, ${'$'}order: Float!) { + chapters( + limit: ${'$'}limit + skip: ${'$'}skip + where: { + deleted: false + published: true + manga: { slug: ${'$'}slug, published: true, deleted: false } + } + order: [{ field: "number", order: ${'$'}order }] + ) { + id + number + updateTime + insertTime + } + } + """.trimIndent() + + val CHAPTER_BY_ID_QUERY: String = """ + query ChapterById(${'$'}id: String!) { + chapter(where: { id: ${'$'}id }) { + id + deleted + published + manga { + id + } + pages { + original + colored + } + } + } + """.trimIndent() + + val CHAPTER_PAGES_QUERY: String = """ + query ChapterPages(${'$'}slug: String!, ${'$'}number: Float!) { + chapters( + limit: 1 + skip: 0 + where: { + number: ${'$'}number + deleted: false + published: true + manga: { slug: ${'$'}slug, published: true, deleted: false } + } + order: [{ field: "number", order: -1 }] + ) { + id + deleted + published + pages { + original + colored + } + } + } + """.trimIndent() + } +} diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/madara/fr/MangasOrigines.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/madara/fr/MangasOrigines.kt index b82cae2..b04ee0e 100644 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/madara/fr/MangasOrigines.kt +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/madara/fr/MangasOrigines.kt @@ -5,6 +5,7 @@ import org.dokiteam.doki.parsers.MangaLoaderContext import org.dokiteam.doki.parsers.MangaSourceParser import org.dokiteam.doki.parsers.model.Manga import org.dokiteam.doki.parsers.model.MangaChapter +import org.dokiteam.doki.parsers.model.MangaPage import org.dokiteam.doki.parsers.model.MangaParserSource import org.dokiteam.doki.parsers.site.madara.MadaraParser import org.dokiteam.doki.parsers.util.attrAsRelativeUrl @@ -25,11 +26,29 @@ internal class MangasOrigines(context: MangaLoaderContext) : override val datePattern = "MMMM d, yyyy" override val tagPrefix = "manga-genres/" override val listUrl = "catalogues/" + private val pagesCache = object : LinkedHashMap>(64, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry>?): Boolean { + return size > PAGES_CACHE_SIZE + } + } override suspend fun getChapters(manga: Manga, doc: Document): List { return parseChapterList(doc.body().select(selectChapter), sourceOrderFallback = true) } + override suspend fun getPages(chapter: MangaChapter): List { + synchronized(pagesCache) { + pagesCache[chapter.url]?.let { return it } + } + val pages = super.getPages(chapter) + if (pages.isNotEmpty()) { + synchronized(pagesCache) { + pagesCache[chapter.url] = pages + } + } + return pages + } + override suspend fun loadChapters(mangaUrl: String, document: Document): List { val doc = if (postReq) { val mangaId = document.select("div#manga-chapters-holder").attr("data-id") @@ -131,6 +150,7 @@ internal class MangasOrigines(context: MangaLoaderContext) : } private companion object { + private const val PAGES_CACHE_SIZE = 200 private val CHAPTER_TITLE_NUMBER = Regex("(?i)\\bchap(?:itre|ter)?\\.?\\s*([0-9]+(?:[.,][0-9]+)?)") private val CHAPTER_URL_NUMBER = Regex("(?i)/(?:chapitre|chapter)-([0-9]+(?:-[0-9]+)?)(?:-|/|$)") diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/mangareader/fr/SushiScanFR.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/mangareader/fr/SushiScanFR.kt index f4c9780..7d4c1be 100644 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/mangareader/fr/SushiScanFR.kt +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/mangareader/fr/SushiScanFR.kt @@ -2,7 +2,9 @@ package org.dokiteam.doki.parsers.site.mangareader.fr import org.dokiteam.doki.parsers.MangaLoaderContext import org.dokiteam.doki.parsers.MangaSourceParser +import org.dokiteam.doki.parsers.model.MangaChapter import org.dokiteam.doki.parsers.model.MangaListFilterCapabilities +import org.dokiteam.doki.parsers.model.MangaPage import org.dokiteam.doki.parsers.model.MangaParserSource import org.dokiteam.doki.parsers.site.mangareader.MangaReaderParser @@ -14,4 +16,27 @@ internal class SushiScanFR(context: MangaLoaderContext) : get() = super.filterCapabilities.copy( isTagsExclusionSupported = false, ) + + private val pagesCache = object : LinkedHashMap>(64, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry>?): Boolean { + return size > PAGES_CACHE_SIZE + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + synchronized(pagesCache) { + pagesCache[chapter.url]?.let { return it } + } + val pages = super.getPages(chapter) + if (pages.isNotEmpty()) { + synchronized(pagesCache) { + pagesCache[chapter.url] = pages + } + } + return pages + } + + private companion object { + private const val PAGES_CACHE_SIZE = 200 + } } diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/mmrcms/fr/ScanManga.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/mmrcms/fr/ScanManga.kt deleted file mode 100644 index 4c66177..0000000 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/mmrcms/fr/ScanManga.kt +++ /dev/null @@ -1,16 +0,0 @@ -package org.dokiteam.doki.parsers.site.mmrcms.fr - -import org.dokiteam.doki.parsers.Broken -import org.dokiteam.doki.parsers.MangaLoaderContext -import org.dokiteam.doki.parsers.MangaSourceParser -import org.dokiteam.doki.parsers.model.MangaParserSource -import org.dokiteam.doki.parsers.site.mmrcms.MmrcmsParser -import java.util.* - -@Broken -@MangaSourceParser("SCANMANGA", "ScanManga", "fr") -internal class ScanManga(context: MangaLoaderContext) : - MmrcmsParser(context, MangaParserSource.SCANMANGA, "scan-manga.me") { - override val imgUpdated = ".jpg" - override val sourceLocale: Locale = Locale.ENGLISH -} diff --git a/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/PizzaReaderParser.kt b/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/PizzaReaderParser.kt index 5e6fe36..c8641cb 100644 --- a/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/PizzaReaderParser.kt +++ b/src/main/kotlin/org/dokiteam/doki/parsers/site/pizzareader/PizzaReaderParser.kt @@ -24,6 +24,16 @@ internal abstract class PizzaReaderParser( ) : SinglePageMangaParser(context, source) { override val configKeyDomain = ConfigKey.Domain(domain) + private val detailsCache = object : LinkedHashMap(64, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { + return size > DETAILS_CACHE_SIZE + } + } + private val pagesCache = object : LinkedHashMap>(128, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry>?): Boolean { + return size > PAGES_CACHE_SIZE + } + } override fun onCreateConfig(keys: MutableCollection>) { super.onCreateConfig(keys) @@ -222,8 +232,7 @@ internal abstract class PizzaReaderParser( title = j.getString("title"), description = j.getString("description"), altTitles = altTitles, - rating = j.getString("rating").toFloatOrNull()?.div(10f) - ?: RATING_UNKNOWN, + rating = parseRating(j.opt("rating")), tags = emptySet(), authors = setOfNotNull(author), state = when (j.getStringOrNull("status")?.trim()?.lowercase(Locale.ROOT)) { @@ -238,12 +247,24 @@ internal abstract class PizzaReaderParser( ) } + private fun parseRating(raw: Any?): Float { + val numeric = when (raw) { + is Number -> raw.toFloat() + is String -> raw.toFloatOrNull() + else -> null + } ?: return RATING_UNKNOWN + return numeric.div(10f) + } + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + synchronized(detailsCache) { + detailsCache[manga.url]?.let { return@coroutineScope it } + } val fullUrl = manga.url.toAbsoluteUrl(domain) val json = webClient.httpGet(fullUrl).parseJson().getJSONObject("comic") val chapters = JSONArray(json.getJSONArray("chapters").asTypedList().reversed()) - manga.copy( + val details = manga.copy( tags = json.getJSONArray("genres").mapJSONToSet { MangaTag( key = it.getString("slug"), @@ -269,11 +290,19 @@ internal abstract class PizzaReaderParser( branch = null, source = source, ) - }, - ) + }, + ) + synchronized(detailsCache) { + detailsCache[manga.url] = details + detailsCache[details.url] = details + } + details } override suspend fun getPages(chapter: MangaChapter): List { + synchronized(pagesCache) { + pagesCache[chapter.url]?.let { return it } + } val fullUrl = chapter.url.toAbsoluteUrl(domain) val pages = webClient.httpGet(fullUrl) .parseJson() @@ -294,9 +323,14 @@ internal abstract class PizzaReaderParser( preview = null, source = source, ), - ) - } - return result + ) + } + if (result.isNotEmpty()) { + synchronized(pagesCache) { + pagesCache[chapter.url] = result + } + } + return result } private fun parseChapterNumber(chapter: JSONObject, fallback: Float): Float { @@ -363,7 +397,9 @@ internal abstract class PizzaReaderParser( "yyyy-MM-dd'T'HH:mm:ss'Z'", "yyyy-MM-dd HH:mm:ss", ) - private val CHAPTER_NUMBER_FROM_LABEL_REGEX = Regex("ch(?:apter)?\\.?\\s*(\\d+(?:\\.\\d+)?)", RegexOption.IGNORE_CASE) - private val CHAPTER_NUMBER_LAST_REGEX = Regex("(\\d+(?:\\.\\d+)?)(?!.*\\d)") - } + private val CHAPTER_NUMBER_FROM_LABEL_REGEX = Regex("ch(?:apter)?\\.?\\s*(\\d+(?:\\.\\d+)?)", RegexOption.IGNORE_CASE) + private val CHAPTER_NUMBER_LAST_REGEX = Regex("(\\d+(?:\\.\\d+)?)(?!.*\\d)") + private const val DETAILS_CACHE_SIZE = 200 + private const val PAGES_CACHE_SIZE = 400 + } } diff --git a/src/test/kotlin/org/dokiteam/doki/parsers/site/all/MangaPlusFrenchFullVerificationTest.kt b/src/test/kotlin/org/dokiteam/doki/parsers/site/all/MangaPlusFrenchFullVerificationTest.kt new file mode 100644 index 0000000..724f86b --- /dev/null +++ b/src/test/kotlin/org/dokiteam/doki/parsers/site/all/MangaPlusFrenchFullVerificationTest.kt @@ -0,0 +1,94 @@ +package org.dokiteam.doki.parsers.site.all + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.dokiteam.doki.parsers.MangaLoaderContextMock +import org.dokiteam.doki.parsers.model.MangaChapter +import org.dokiteam.doki.parsers.model.MangaListFilter +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.SortOrder +import kotlin.time.Duration.Companion.minutes + +internal class MangaPlusFrenchFullVerificationTest { + + @Test + fun verifyMangaPlusFrenchParserEndToEnd() = runTest(timeout = 6.minutes) { + val parser = MangaLoaderContextMock.newParserInstance(MangaParserSource.MANGAPLUSPARSER_FR) + + val popular = parser.getList(offset = 0, order = SortOrder.POPULARITY, filter = MangaListFilter.EMPTY) + check(popular.isNotEmpty()) { "Popularity list is empty" } + + val updated = parser.getList(offset = 0, order = SortOrder.UPDATED, filter = MangaListFilter.EMPTY) + check(updated.isNotEmpty()) { "Updated list is empty" } + + val alphabetical = parser.getList(offset = 0, order = SortOrder.ALPHABETICAL, filter = MangaListFilter.EMPTY) + check(alphabetical.isNotEmpty()) { "Alphabetical list is empty" } + + val query = popular.first().title + .split(Regex("\\s+")) + .firstOrNull { it.length >= 3 } + ?: popular.first().title.take(4) + val search = parser.getList( + offset = 0, + order = SortOrder.UPDATED, + filter = MangaListFilter(query = query), + ) + check(search.isNotEmpty()) { "Search returned empty list for query '$query'" } + + val detailsCandidates = (popular + updated + alphabetical) + .distinctBy { it.id } + .take(40) + .mapNotNull { manga -> runCatching { parser.getDetails(manga) }.getOrNull() } + .filter { !it.chapters.isNullOrEmpty() } + check(detailsCandidates.isNotEmpty()) { "Unable to load detailed manga with chapters" } + val detailed = detailsCandidates.maxByOrNull { it.chapters.orEmpty().size } + ?: error("No detailed manga selected") + + check(!detailed.coverUrl.isNullOrBlank()) { "Cover URL is blank for ${detailed.publicUrl}" } + checkImageResponse(detailed.coverUrl) + check(!detailed.description.isNullOrBlank()) { "Description is blank for ${detailed.publicUrl}" } + + val chapters = detailed.chapters.orEmpty() + check(chapters.size > 1) { "Not enough chapters for ${detailed.publicUrl}" } + check(chapters.distinctBy { it.id }.size == chapters.size) { "Duplicate chapter IDs for ${detailed.publicUrl}" } + + val readableSamples = mutableListOf>>() + for (chapter in chapters) { + if (readableSamples.size >= 3) break + val pages = runCatching { parser.getPages(chapter) }.getOrNull().orEmpty() + val pageUrls = pages.map { parser.getPageUrl(it) }.distinct() + if (pageUrls.isEmpty()) continue + readableSamples += chapter to pageUrls + } + check(readableSamples.isNotEmpty()) { "No readable chapters found for ${detailed.publicUrl}" } + + val sampleSummary = readableSamples.map { (chapter, pageUrls) -> + check(pageUrls.all { it.startsWith("http://") || it.startsWith("https://") }) { + "Non-absolute page URLs for ${chapter.url}: $pageUrls" + } + checkImageResponse(pageUrls.first()) + checkImageResponse(pageUrls.last()) + val cachedPagesCount = parser.getPages(chapter).size + check(cachedPagesCount == pageUrls.size) { + "Page count mismatch between repeated calls for ${chapter.url}: ${pageUrls.size} != $cachedPagesCount" + } + chapter.url to pageUrls.size + } + + println( + "MANGAPLUSFR_FULL_SMOKE|popular=${popular.size}|updated=${updated.size}|alpha=${alphabetical.size}|" + + "search=${search.size}|manga=${detailed.publicUrl}|chapters=${chapters.size}|samples=$sampleSummary", + ) + } + + private suspend fun checkImageResponse(url: String?) { + check(!url.isNullOrBlank()) { "Image URL is blank" } + MangaLoaderContextMock.doRequest(url, MangaParserSource.MANGAPLUSPARSER_FR).use { response -> + check(response.isSuccessful) { "Image request failed ${response.code}(${response.message}) for $url" } + val contentType = response.header("Content-Type").orEmpty() + check(contentType.startsWith("image/")) { + "Invalid image Content-Type '$contentType' for $url" + } + } + } +} diff --git a/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/BigSoloFullVerificationTest.kt b/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/BigSoloFullVerificationTest.kt new file mode 100644 index 0000000..ff9a4da --- /dev/null +++ b/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/BigSoloFullVerificationTest.kt @@ -0,0 +1,82 @@ +package org.dokiteam.doki.parsers.site.fr + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.dokiteam.doki.parsers.MangaLoaderContextMock +import org.dokiteam.doki.parsers.model.MangaChapter +import org.dokiteam.doki.parsers.model.MangaListFilter +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.SortOrder +import java.util.Locale +import kotlin.time.Duration.Companion.minutes + +internal class BigSoloFullVerificationTest { + + @Test + fun verifyBigSoloParserEndToEnd() = runTest(timeout = 6.minutes) { + val parser = MangaLoaderContextMock.newParserInstance(MangaParserSource.BIGSOLO) + + val updated = parser.getList(offset = 0, order = SortOrder.UPDATED, filter = MangaListFilter.EMPTY) + check(updated.isNotEmpty()) { "Updated list is empty" } + val alphabetical = parser.getList(offset = 0, order = SortOrder.ALPHABETICAL, filter = MangaListFilter.EMPTY) + check(alphabetical.isNotEmpty()) { "Alphabetical list is empty" } + val expectedAlpha = alphabetical.sortedBy { it.title.lowercase(Locale.ROOT) }.map { it.id } + check(alphabetical.map { it.id } == expectedAlpha) { "Alphabetical order is incorrect" } + + val options = parser.getFilterOptions() + check(options.availableTags.isNotEmpty()) { "No tags in filter options" } + + val detailedCandidates = updated + .take(20) + .mapNotNull { manga -> runCatching { parser.getDetails(manga) }.getOrNull() } + check(detailedCandidates.isNotEmpty()) { "Unable to load details from updated list" } + val detailed = detailedCandidates.maxByOrNull { it.chapters.orEmpty().size } + ?: error("No detailed manga candidate") + + check(!detailed.coverUrl.isNullOrBlank()) { "Cover URL is blank for ${detailed.publicUrl}" } + checkImageResponse(detailed.coverUrl) + + val chapters = detailed.chapters.orEmpty() + check(chapters.size > 1) { "Too few chapters for ${detailed.publicUrl}: ${chapters.size}" } + check(chapters.distinctBy { it.id }.size == chapters.size) { + "Duplicate chapter ids for ${detailed.publicUrl}" + } + check(chapters.zipWithNext().all { (left, right) -> left.number <= right.number }) { + "Chapter order is not ascending for ${detailed.publicUrl}" + } + + val sampled = buildList { + add(chapters.first()) + chapters.getOrNull(chapters.size / 2)?.let { add(it) } + add(chapters.last()) + }.distinctBy { it.id } + + val sampleSummary = sampled.map { chapter -> + val pages = parser.getPages(chapter) + val pageUrls = pages.map { parser.getPageUrl(it) }.distinct() + check(pageUrls.isNotEmpty()) { "No pages for ${chapter.url}" } + check(pageUrls.all { it.startsWith("http://") || it.startsWith("https://") }) { + "Non-absolute page URL for ${chapter.url}: $pageUrls" + } + checkImageResponse(pageUrls.first()) + checkImageResponse(pageUrls.last()) + chapter.url to pageUrls.size + } + + println( + "BIGSOLO_FULL_SMOKE|updated=${updated.size}|alpha=${alphabetical.size}|manga=${detailed.publicUrl}|" + + "chapters=${chapters.size}|samples=$sampleSummary", + ) + } + + private suspend fun checkImageResponse(url: String?) { + check(!url.isNullOrBlank()) { "Image URL is blank" } + MangaLoaderContextMock.doRequest(url, MangaParserSource.BIGSOLO).use { response -> + check(response.isSuccessful) { "Image request failed ${response.code}(${response.message}) for $url" } + val contentType = response.header("Content-Type").orEmpty() + check(contentType.startsWith("image/")) { + "Invalid image Content-Type '$contentType' for $url" + } + } + } +} diff --git a/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoinsFullVerificationTest.kt b/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoinsFullVerificationTest.kt new file mode 100644 index 0000000..1967aea --- /dev/null +++ b/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/MangaMoinsFullVerificationTest.kt @@ -0,0 +1,89 @@ +package org.dokiteam.doki.parsers.site.fr + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.dokiteam.doki.parsers.MangaLoaderContextMock +import org.dokiteam.doki.parsers.model.MangaChapter +import org.dokiteam.doki.parsers.model.MangaListFilter +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.SortOrder +import kotlin.time.Duration.Companion.minutes + +internal class MangaMoinsFullVerificationTest { + + @Test + fun verifyMangaMoinsParserEndToEnd() = runTest(timeout = 6.minutes) { + val source = MangaParserSource.MANGAMOINS + val parser = MangaLoaderContextMock.newParserInstance(source) + + val updated = parser.getList(offset = 0, order = SortOrder.UPDATED, filter = MangaListFilter.EMPTY) + check(updated.isNotEmpty()) { "Updated list is empty" } + + val query = updated.first().title + .split(Regex("\\s+")) + .firstOrNull { it.length >= 3 } + ?: updated.first().title.take(4) + val search = parser.getList( + offset = 0, + order = SortOrder.UPDATED, + filter = MangaListFilter(query = query), + ) + check(search.isNotEmpty()) { "Search returned empty list for query '$query'" } + + val detailsCandidates = updated + .take(20) + .mapNotNull { manga -> runCatching { parser.getDetails(manga) }.getOrNull() } + .filter { !it.chapters.isNullOrEmpty() } + check(detailsCandidates.isNotEmpty()) { "Unable to load detailed manga with chapters" } + val detailed = detailsCandidates.maxByOrNull { it.chapters.orEmpty().size } + ?: error("No detailed manga selected") + + check(!detailed.coverUrl.isNullOrBlank()) { "Cover URL is blank for ${detailed.publicUrl}" } + checkImageResponse(detailed.coverUrl, source) + check(!detailed.description.isNullOrBlank()) { "Description is blank for ${detailed.publicUrl}" } + + val chapters = detailed.chapters.orEmpty() + check(chapters.size > 1) { "Not enough chapters for ${detailed.publicUrl}" } + check(chapters.distinctBy { it.id }.size == chapters.size) { "Duplicate chapter IDs for ${detailed.publicUrl}" } + + val readableSamples = mutableListOf>>() + for (chapter in chapters) { + if (readableSamples.size >= 3) break + val pageUrls = runCatching { + parser.getPages(chapter).map { parser.getPageUrl(it) }.distinct() + }.getOrDefault(emptyList()) + if (pageUrls.isEmpty()) continue + readableSamples += chapter to pageUrls + } + check(readableSamples.isNotEmpty()) { "No readable chapter found for ${detailed.publicUrl}" } + + val sampleSummary = readableSamples.map { (chapter, pageUrls) -> + check(pageUrls.all { it.startsWith("http://") || it.startsWith("https://") }) { + "Non-absolute page URL for ${chapter.url}: $pageUrls" + } + checkImageResponse(pageUrls.first(), source) + checkImageResponse(pageUrls.last(), source) + val cachedPagesCount = parser.getPages(chapter).size + check(cachedPagesCount == pageUrls.size) { + "Page count mismatch between repeated calls for ${chapter.url}: ${pageUrls.size} != $cachedPagesCount" + } + chapter.url to pageUrls.size + } + + println( + "MANGAMOINS_FULL_SMOKE|updated=${updated.size}|search=${search.size}|manga=${detailed.publicUrl}|" + + "chapters=${chapters.size}|samples=$sampleSummary", + ) + } + + private suspend fun checkImageResponse(url: String?, source: MangaParserSource) { + check(!url.isNullOrBlank()) { "Image URL is blank" } + MangaLoaderContextMock.doRequest(url, source).use { response -> + check(response.isSuccessful) { "Image request failed ${response.code}(${response.message}) for $url" } + val contentType = response.header("Content-Type").orEmpty() + check(contentType.startsWith("image/")) { + "Invalid image Content-Type '$contentType' for $url" + } + } + } +} diff --git a/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScansChapterPagesSmokeTest.kt b/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScansChapterPagesSmokeTest.kt new file mode 100644 index 0000000..fa54b20 --- /dev/null +++ b/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScansChapterPagesSmokeTest.kt @@ -0,0 +1,49 @@ +package org.dokiteam.doki.parsers.site.fr + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.dokiteam.doki.parsers.MangaLoaderContextMock +import org.dokiteam.doki.parsers.model.MangaChapter +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.search.MangaSearchQuery +import kotlin.time.Duration.Companion.minutes + +internal class PoseidonScansChapterPagesSmokeTest { + + @Test + fun verifyChapterPagesRecovery() = runTest(timeout = 3.minutes) { + val parser = MangaLoaderContextMock.newParserInstance(MangaParserSource.POSEIDONSCANS) + val list = parser.getList(MangaSearchQuery.EMPTY) + check(list.isNotEmpty()) { "Poseidon list is empty" } + + val detailedManga = list.take(8).firstNotNullOfOrNull { manga -> + runCatching { + parser.getDetails(manga).takeIf { !it.chapters.isNullOrEmpty() } + }.getOrNull() + } ?: error("Unable to load a manga with chapters from Poseidon list") + + val chapters = detailedManga.chapters.orEmpty() + check(chapters.isNotEmpty()) { "No chapters found for ${detailedManga.publicUrl}" } + + val sampleChapters = buildList { + add(chapters.first()) + chapters.getOrNull(chapters.size / 2)?.let { add(it) } + add(chapters.last()) + }.distinctBy { it.id } + + val chapterPageCounts = sampleChapters.map { chapter -> + val pages = parser.getPages(chapter) + val pageUrls = pages.map { it.url }.distinct() + println("POSEIDON_SMOKE|chapter=${chapter.url}|pages=${pageUrls.size}|first=${pageUrls.firstOrNull()}") + check(pageUrls.size > 1) { "Expected more than one page for ${chapter.url}, got ${pageUrls.size}" } + check(pageUrls.none { it.contains("/api/covers/") }) { + "Chapter pages fallbacked to cover for ${chapter.url}: $pageUrls" + } + chapter.url to pageUrls.size + } + + println( + "POSEIDON_SMOKE_SUMMARY|manga=${detailedManga.publicUrl}|totalChapters=${chapters.size}|sample=$chapterPageCounts", + ) + } +} diff --git a/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScansFullVerificationTest.kt b/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScansFullVerificationTest.kt new file mode 100644 index 0000000..25cf203 --- /dev/null +++ b/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScansFullVerificationTest.kt @@ -0,0 +1,183 @@ +package org.dokiteam.doki.parsers.site.fr + +import kotlinx.coroutines.test.runTest +import org.json.JSONObject +import org.junit.jupiter.api.Test +import org.dokiteam.doki.parsers.MangaLoaderContextMock +import org.dokiteam.doki.parsers.model.MangaChapter +import org.dokiteam.doki.parsers.model.MangaListFilter +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.SortOrder +import kotlin.time.Duration.Companion.minutes + +internal class PoseidonScansFullVerificationTest { + + private val source = MangaParserSource.POSEIDONSCANS + private val mangaRouteRegex = Regex("""(?:https?://[^/]+)?/serie/([^/?#]+)""") + private val chapterRouteRegex = Regex("""(?:https?://[^/]+)?/serie/([^/]+)/chapter/([^/?#]+)""") + + @Test + fun verifyPoseidonScanParserEndToEnd() = runTest(timeout = 5.minutes) { + val parser = MangaLoaderContextMock.newParserInstance(source) + + val updated = parser.getList(offset = 0, order = SortOrder.UPDATED, filter = MangaListFilter.EMPTY) + check(updated.isNotEmpty()) { "Updated list is empty" } + val popularity = parser.getList(offset = 0, order = SortOrder.POPULARITY, filter = MangaListFilter.EMPTY) + check(popularity.isNotEmpty()) { "Popularity list is empty" } + val alphabetical = parser.getList(offset = 0, order = SortOrder.ALPHABETICAL, filter = MangaListFilter.EMPTY) + check(alphabetical.isNotEmpty()) { "Alphabetical list is empty" } + val rating = parser.getList(offset = 0, order = SortOrder.RATING, filter = MangaListFilter.EMPTY) + check(rating.isNotEmpty()) { "Rating list is empty" } + + val alphabeticalExpected = alphabetical.sortedBy { it.title.lowercase() }.map { it.id } + check(alphabetical.map { it.id } == alphabeticalExpected) { "Alphabetical order is incorrect" } + + val sampleForQuery = updated.first() + val query = sampleForQuery.title + .split(Regex("\\s+")) + .firstOrNull { it.length >= 3 } + ?: sampleForQuery.title.take(4) + val searchResults = parser.getList( + offset = 0, + order = SortOrder.UPDATED, + filter = MangaListFilter(query = query), + ) + check(searchResults.isNotEmpty()) { "Search by query '$query' is empty" } + + val filterOptions = parser.getFilterOptions() + check(filterOptions.availableTags.isNotEmpty()) { "No tags returned by getFilterOptions()" } + val (tag, tagResults) = filterOptions.availableTags + .take(12) + .firstNotNullOfOrNull { tag -> + val list = parser.getList( + offset = 0, + order = SortOrder.UPDATED, + filter = MangaListFilter(tags = setOf(tag)), + ) + if (list.isEmpty()) { + null + } else { + tag to list + } + } + ?: error("Failed to find a tag that returns mangas") + check(tagResults.all { manga -> manga.tags.contains(tag) }) { + "Tag filtering is inconsistent for '${tag.title}'" + } + + val detailsCandidates = updated + .take(20) + .mapNotNull { manga -> runCatching { parser.getDetails(manga) }.getOrNull() } + check(detailsCandidates.isNotEmpty()) { "Unable to load manga details from Poseidon" } + check(detailsCandidates.any { !it.description.isNullOrBlank() }) { + "All tested manga descriptions are blank in top 20 detailed entries" + } + val detailed = detailsCandidates + .filter { it.chapters.orEmpty().size > 2 } + .maxByOrNull { it.chapters.orEmpty().size } + ?: error("Unable to load manga details from Poseidon") + check(!detailed.coverUrl.isNullOrBlank()) { "Cover URL is blank for ${detailed.publicUrl}" } + check(!detailed.description.isNullOrBlank()) { "Description is blank for ${detailed.publicUrl}" } + checkImageResponse(detailed.coverUrl) + + val chapters = detailed.chapters.orEmpty() + check(chapters.size > 2) { "Only ${chapters.size} chapters parsed for ${detailed.publicUrl}" } + check(chapters.distinctBy { it.id }.size == chapters.size) { "Duplicate chapter ids for ${detailed.publicUrl}" } + check(chapters.all { it.url.contains("/chapter/") }) { "Invalid chapter URLs for ${detailed.publicUrl}" } + check(chapters.zipWithNext().all { (left, right) -> left.number <= right.number }) { + "Chapter order is not ascending for ${detailed.publicUrl}" + } + + val mangaSlug = parseMangaSlug(detailed.url) + ?: error("Unable to parse manga slug from ${detailed.url}") + + val sampledChapters = buildList { + add(chapters.first()) + chapters.getOrNull(chapters.size / 2)?.let { add(it) } + add(chapters.last()) + }.distinctBy { it.id } + check(sampledChapters.isNotEmpty()) { "No chapters sampled for ${detailed.publicUrl}" } + + val sampledChapterPageCounts = sampledChapters.map { chapter -> + val (chapterSlug, chapterNumberRaw) = parseChapterRoute(chapter.url) + ?: error("Unable to parse chapter route: ${chapter.url}") + check(chapterSlug == mangaSlug) { + "Chapter slug mismatch: chapter=$chapterSlug manga=$mangaSlug for ${chapter.url}" + } + + val chapterMeta = requestJson("https://poseidon-scans.co/api/manga/$chapterSlug/$chapterNumberRaw") + val chapterData = chapterMeta.optJSONObject("data") + ?.optJSONObject("chapterData") + ?: error("Missing chapterData from API for ${chapter.url}") + val isPremium = chapterData.optBoolean("isPremium", false) + check(!isPremium) { "Premium chapter leaked into readable list: ${chapter.url}" } + val chapterId = chapterData.optString("id").takeIf { it.isNotBlank() } + ?: error("Missing chapter id for ${chapter.url}") + + val apiImagesPayload = + requestJson("https://poseidon-scans.co/api/chapters/$chapterSlug/$chapterId/images") + val apiImages = apiImagesPayload.optJSONArray("images") + ?: apiImagesPayload.optJSONObject("data")?.optJSONArray("images") + ?: error("Missing images array for ${chapter.url}") + val apiImageCount = apiImages.length() + check(apiImageCount > 1) { "API returned $apiImageCount image(s) for ${chapter.url}" } + + val parserPages = parser.getPages(chapter) + val parserPageUrls = parserPages.map { parser.getPageUrl(it) }.distinct() + check(parserPageUrls.size == apiImageCount) { + "Pages mismatch for ${chapter.url}: parser=${parserPageUrls.size}, api=$apiImageCount" + } + check(parserPageUrls.none { it.contains("/api/covers/") }) { + "Cover fallback detected in pages for ${chapter.url}: $parserPageUrls" + } + check(parserPageUrls.all { it.contains("/api/chapters/$chapterSlug/") }) { + "Unexpected page URL pattern for ${chapter.url}: $parserPageUrls" + } + + checkImageResponse(parserPageUrls.first()) + checkImageResponse(parserPageUrls.last()) + chapter.url to parserPageUrls.size + } + + val updatedTop = updated.take(10).map { it.id } + val popularityTop = popularity.take(10).map { it.id } + println( + "POSEIDON_FULL_SMOKE|updated=${updated.size}|popular=${popularity.size}|alpha=${alphabetical.size}|" + + "rating=${rating.size}|top10_updated_eq_popular=${updatedTop == popularityTop}|" + + "manga=${detailed.publicUrl}|chapters=${chapters.size}|tag=${tag.title}|samples=$sampledChapterPageCounts|" + + "description_len=${detailed.description?.length ?: 0}", + ) + } + + private fun parseMangaSlug(url: String): String? { + val match = mangaRouteRegex.find(url) ?: return null + return match.groupValues.getOrNull(1)?.trim()?.takeIf { it.isNotEmpty() } + } + + private fun parseChapterRoute(url: String): Pair? { + val match = chapterRouteRegex.find(url) ?: return null + val slug = match.groupValues.getOrNull(1)?.trim().orEmpty() + val chapterNumberRaw = match.groupValues.getOrNull(2)?.trim().orEmpty() + if (slug.isEmpty() || chapterNumberRaw.isEmpty()) return null + return slug to chapterNumberRaw + } + + private suspend fun requestJson(url: String): JSONObject { + val rawBody = MangaLoaderContextMock.doRequest(url, source).use { response -> + check(response.isSuccessful) { "Request failed ${response.code}(${response.message}) for $url" } + response.body?.string().orEmpty() + } + return JSONObject(rawBody) + } + + private suspend fun checkImageResponse(url: String?) { + check(!url.isNullOrBlank()) { "Image url is blank" } + MangaLoaderContextMock.doRequest(url, source).use { response -> + check(response.isSuccessful) { "Image request failed ${response.code}(${response.message}) for $url" } + val contentType = response.header("Content-Type").orEmpty() + check(contentType.startsWith("image/")) { + "Invalid image Content-Type '$contentType' for $url" + } + } + } +} diff --git a/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScansPremiumVerificationTest.kt b/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScansPremiumVerificationTest.kt new file mode 100644 index 0000000..bbd26be --- /dev/null +++ b/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PoseidonScansPremiumVerificationTest.kt @@ -0,0 +1,204 @@ +package org.dokiteam.doki.parsers.site.fr + +import kotlinx.coroutines.test.runTest +import org.json.JSONArray +import org.json.JSONObject +import org.junit.jupiter.api.Test +import org.dokiteam.doki.parsers.MangaLoaderContextMock +import org.dokiteam.doki.parsers.model.Manga +import org.dokiteam.doki.parsers.model.MangaListFilter +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.SortOrder +import kotlin.math.abs +import kotlin.time.Duration.Companion.minutes + +internal class PoseidonScansPremiumVerificationTest { + + private val source = MangaParserSource.POSEIDONSCANS + + private data class LatestEntry( + val slug: String, + val title: String, + val latestNumberRaw: String, + val latestIsPremium: Boolean, + val chaptersCount: Int, + ) + + private data class ChapterEntry( + val numberRaw: String, + val numberValue: Double, + val isPremium: Boolean, + ) + + @Test + fun verifyPremiumAndNonPremiumChapters() = runTest(timeout = 6.minutes) { + val parser = MangaLoaderContextMock.newParserInstance(source) + val latestEntries = fetchLatestEntries() + + val premiumCandidate = latestEntries + .filter { it.latestIsPremium && it.chaptersCount > 1 } + .sortedBy { it.chaptersCount } + .firstOrNull() + ?: error("No premium candidate found from latest endpoint") + + val nonPremiumCandidate = latestEntries + .filter { !it.latestIsPremium && it.chaptersCount > 3 } + .sortedBy { it.chaptersCount } + .firstOrNull() + ?: error("No non-premium candidate found from latest endpoint") + + val premiumManga = findMangaBySlug(parser, premiumCandidate.slug) + ?: error("Premium manga not found in parser listing: ${premiumCandidate.slug}") + val premiumDetails = parser.getDetails(premiumManga) + val premiumExpectedAll = fetchChapterEntriesWithPremiumStatus( + slug = premiumCandidate.slug, + seedChapterNumberRaw = premiumCandidate.latestNumberRaw, + ) + val premiumExpectedReadable = takeReadableBeforeFirstPremium(premiumExpectedAll) + assertParserMatchesExpectedChapters( + actualManga = premiumDetails, + expectedReadable = premiumExpectedReadable, + label = "premium_candidate", + ) + check(premiumExpectedAll.any { it.isPremium }) { + "Expected at least one premium chapter for ${premiumCandidate.slug}" + } + check(premiumExpectedReadable.size < premiumExpectedAll.size) { + "Premium boundary was not applied for ${premiumCandidate.slug}" + } + + val nonPremiumManga = findMangaBySlug(parser, nonPremiumCandidate.slug) + ?: error("Non-premium manga not found in parser listing: ${nonPremiumCandidate.slug}") + val nonPremiumDetails = parser.getDetails(nonPremiumManga) + val nonPremiumExpectedAll = fetchChapterEntriesWithPremiumStatus( + slug = nonPremiumCandidate.slug, + seedChapterNumberRaw = nonPremiumCandidate.latestNumberRaw, + ) + val nonPremiumExpectedReadable = takeReadableBeforeFirstPremium(nonPremiumExpectedAll) + assertParserMatchesExpectedChapters( + actualManga = nonPremiumDetails, + expectedReadable = nonPremiumExpectedReadable, + label = "non_premium_candidate", + ) + check(nonPremiumExpectedAll.none { it.isPremium }) { + "Unexpected premium status in non-premium candidate ${nonPremiumCandidate.slug}" + } + + println( + "POSEIDON_PREMIUM_CHECK|" + + "premium_slug=${premiumCandidate.slug}|premium_total=${premiumCandidate.chaptersCount}|" + + "premium_readable=${premiumDetails.chapters.orEmpty().size}|premium_latest=${premiumCandidate.latestNumberRaw}|" + + "non_premium_slug=${nonPremiumCandidate.slug}|non_premium_total=${nonPremiumCandidate.chaptersCount}|" + + "non_premium_readable=${nonPremiumDetails.chapters.orEmpty().size}|non_premium_latest=${nonPremiumCandidate.latestNumberRaw}", + ) + } + + private suspend fun findMangaBySlug(parser: org.dokiteam.doki.parsers.MangaParser, slug: String): Manga? { + var offset = 0 + repeat(40) { + val list = parser.getList(offset = offset, order = SortOrder.UPDATED, filter = MangaListFilter.EMPTY) + if (list.isEmpty()) return null + list.firstOrNull { it.url == "/serie/$slug" }?.let { return it } + offset += list.size + } + return null + } + + private suspend fun fetchLatestEntries(): List { + val payload = requestJson("https://poseidon-scans.co/api/manga/latest?limit=500&page=1") + val data = payload.optJSONArray("data") ?: return emptyList() + return buildList { + for (index in 0 until data.length()) { + val entry = data.optJSONObject(index) ?: continue + val slug = entry.optString("slug").trim() + val title = entry.optString("title").trim() + val chapters = entry.optJSONArray("chapters") + val latestChapter = chapters?.optJSONObject(0) + val latestNumberRaw = normalizeNumberRaw(latestChapter?.opt("number")) ?: continue + val latestIsPremium = latestChapter?.optBoolean("isPremium", false) ?: false + val chaptersCount = entry.optJSONObject("_count")?.optInt("chapters", 0) ?: 0 + if (slug.isBlank() || title.isBlank() || chaptersCount <= 0) continue + add( + LatestEntry( + slug = slug, + title = title, + latestNumberRaw = latestNumberRaw, + latestIsPremium = latestIsPremium, + chaptersCount = chaptersCount, + ), + ) + } + } + } + + private suspend fun fetchChapterEntriesWithPremiumStatus( + slug: String, + seedChapterNumberRaw: String, + ): List { + val payload = requestJson("https://poseidon-scans.co/api/manga/$slug/$seedChapterNumberRaw") + val chapterList = payload.optJSONObject("data")?.optJSONArray("chapterList") ?: JSONArray() + val chapterEntries = buildList { + for (index in 0 until chapterList.length()) { + val chapter = chapterList.optJSONObject(index) ?: continue + val numberRaw = normalizeNumberRaw(chapter.opt("number")) ?: continue + val numberValue = numberRaw.toDoubleOrNull() ?: continue + add(numberRaw to numberValue) + } + }.distinctBy { it.first }.sortedBy { it.second } + + return chapterEntries.map { (numberRaw, numberValue) -> + val statusPayload = requestJson("https://poseidon-scans.co/api/manga/$slug/$numberRaw") + val isPremium = statusPayload.optJSONObject("data") + ?.optJSONObject("chapterData") + ?.optBoolean("isPremium", false) ?: false + ChapterEntry( + numberRaw = numberRaw, + numberValue = numberValue, + isPremium = isPremium, + ) + } + } + + private fun takeReadableBeforeFirstPremium(entries: List): List { + if (entries.isEmpty()) return emptyList() + val firstPremiumIndex = entries.indexOfFirst { it.isPremium } + return if (firstPremiumIndex == -1) entries else entries.subList(0, firstPremiumIndex) + } + + private fun assertParserMatchesExpectedChapters( + actualManga: Manga, + expectedReadable: List, + label: String, + ) { + val chapters = actualManga.chapters.orEmpty() + check(chapters.isNotEmpty()) { "No parsed chapters for $label: ${actualManga.publicUrl}" } + val actualValues = chapters.map { it.number.toDouble() } + val expectedValues = expectedReadable.map { it.numberValue } + + check(actualValues.size == expectedValues.size) { + "Chapter count mismatch for $label: parser=${actualValues.size}, expected=${expectedValues.size}" + } + check(actualValues.zipWithNext().all { (left, right) -> left <= right }) { + "Parsed chapter order is not ascending for $label: ${actualManga.publicUrl}" + } + actualValues.zip(expectedValues).forEachIndexed { index, (actual, expected) -> + check(abs(actual - expected) < 0.0001) { + "Chapter number mismatch at index=$index for $label: parser=$actual expected=$expected" + } + } + } + + private fun normalizeNumberRaw(value: Any?): String? { + val raw = value?.toString()?.trim()?.takeIf { it.isNotEmpty() && it != "null" } ?: return null + val numeric = raw.toDoubleOrNull() ?: return raw + return if (numeric % 1.0 == 0.0) numeric.toInt().toString() else raw + } + + private suspend fun requestJson(url: String): JSONObject { + val rawBody = MangaLoaderContextMock.doRequest(url, source).use { response -> + check(response.isSuccessful) { "Request failed ${response.code}(${response.message}) for $url" } + response.body?.string().orEmpty() + } + return JSONObject(rawBody) + } +} diff --git a/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PunkRecordzFullVerificationTest.kt b/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PunkRecordzFullVerificationTest.kt new file mode 100644 index 0000000..fa336bd --- /dev/null +++ b/src/test/kotlin/org/dokiteam/doki/parsers/site/fr/PunkRecordzFullVerificationTest.kt @@ -0,0 +1,166 @@ +package org.dokiteam.doki.parsers.site.fr + +import kotlinx.coroutines.test.runTest +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONObject +import org.junit.jupiter.api.Test +import org.dokiteam.doki.parsers.MangaLoaderContextMock +import org.dokiteam.doki.parsers.model.MangaChapter +import org.dokiteam.doki.parsers.model.MangaListFilter +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.SortOrder +import java.util.Locale +import kotlin.time.Duration.Companion.minutes + +internal class PunkRecordzFullVerificationTest { + + private val source = MangaParserSource.PUNKRECORDZ + private val apiUrl = "https://api.punkrecordz.com/graphql" + + @Test + fun verifyPunkRecordzParserEndToEnd() = runTest(timeout = 6.minutes) { + val parser = MangaLoaderContextMock.newParserInstance(source) + + val updated = parser.getList(offset = 0, order = SortOrder.UPDATED, filter = MangaListFilter.EMPTY) + check(updated.isNotEmpty()) { "Updated list is empty" } + val alphabetical = parser.getList(offset = 0, order = SortOrder.ALPHABETICAL, filter = MangaListFilter.EMPTY) + check(alphabetical.isNotEmpty()) { "Alphabetical list is empty" } + val relevance = parser.getList( + offset = 0, + order = SortOrder.RELEVANCE, + filter = MangaListFilter(query = "bleach"), + ) + check(relevance.isNotEmpty()) { "Relevance list is empty for query 'bleach'" } + + val expectedAlpha = alphabetical.sortedBy { it.title.lowercase(Locale.ROOT) }.map { it.id } + check(alphabetical.map { it.id } == expectedAlpha) { "Alphabetical order is incorrect" } + + val filterOptions = parser.getFilterOptions() + check(filterOptions.availableTags.isNotEmpty()) { "No tags in filter options" } + val (tag, taggedList) = filterOptions.availableTags + .firstNotNullOfOrNull { tag -> + val list = parser.getList( + offset = 0, + order = SortOrder.UPDATED, + filter = MangaListFilter(tags = setOf(tag)), + ) + if (list.isEmpty()) null else tag to list + } + ?: error("No tag with matching mangas found") + check(taggedList.all { manga -> manga.tags.contains(tag) }) { + "Tag filter mismatch for '${tag.title}'" + } + + val detailsCandidates = updated + .take(20) + .mapNotNull { manga -> runCatching { parser.getDetails(manga) }.getOrNull() } + check(detailsCandidates.isNotEmpty()) { "No details candidate loaded from updated list" } + val detailed = detailsCandidates.maxByOrNull { it.chapters.orEmpty().size } + ?: error("No detailed manga available") + + check(!detailed.coverUrl.isNullOrBlank()) { "Cover URL is blank for ${detailed.publicUrl}" } + checkImageResponse(detailed.coverUrl) + check(!detailed.description.isNullOrBlank()) { "Description is blank for ${detailed.publicUrl}" } + + val chapters = detailed.chapters.orEmpty() + check(chapters.size > 2) { "Too few chapters for ${detailed.publicUrl}: ${chapters.size}" } + check(chapters.distinctBy { it.id }.size == chapters.size) { + "Duplicate chapter ids for ${detailed.publicUrl}" + } + check(chapters.all { it.url.contains("?id=") }) { + "Expected chapter URLs with id query parameter for ${detailed.publicUrl}" + } + + val sampled = buildList { + add(chapters.first()) + chapters.getOrNull(chapters.size / 2)?.let { add(it) } + add(chapters.last()) + }.distinctBy { it.id } + + val sampledSummary = sampled.map { chapter -> + val chapterId = chapter.url.substringAfter("id=", "").substringBefore("&") + check(chapterId.isNotBlank()) { "Missing chapter id in URL ${chapter.url}" } + + val expectedPageCount = fetchPageCountByChapterId(chapterId) + val pages = parser.getPages(chapter) + val pageUrls = pages.map { parser.getPageUrl(it) }.distinct() + + check(pageUrls.size == expectedPageCount) { + "Page count mismatch for ${chapter.url}: parser=${pageUrls.size}, api=$expectedPageCount" + } + check(pageUrls.isNotEmpty()) { "No page URL returned for ${chapter.url}" } + check(pageUrls.all { it.startsWith("https://api.punkrecordz.com/images/") }) { + "Unexpected page URL format for ${chapter.url}: $pageUrls" + } + checkImageResponse(pageUrls.first()) + checkImageResponse(pageUrls.last()) + chapter.url to pageUrls.size + } + + println( + "PUNK_FULL_SMOKE|updated=${updated.size}|alpha=${alphabetical.size}|relevance=${relevance.size}|" + + "manga=${detailed.publicUrl}|chapters=${chapters.size}|tag=${tag.title}|samples=$sampledSummary", + ) + } + + private suspend fun fetchPageCountByChapterId(chapterId: String): Int { + val query = """ + query ChapterById(${'$'}id: String!) { + chapter(where: { id: ${'$'}id }) { + id + deleted + published + manga { id } + pages { + original + colored + } + } + } + """.trimIndent() + val payload = graphQl( + query = query, + variables = JSONObject().put("id", chapterId), + ) + val chapter = payload.optJSONObject("chapter") + ?: error("Chapter not found in API for id=$chapterId") + check(!chapter.optBoolean("deleted", false)) { "Chapter $chapterId is deleted" } + check(chapter.optBoolean("published", true)) { "Chapter $chapterId is unpublished" } + return chapter.optJSONArray("pages")?.length() ?: 0 + } + + private suspend fun graphQl(query: String, variables: JSONObject = JSONObject()): JSONObject { + val body = JSONObject() + .put("query", query) + .put("variables", variables) + val raw = MangaLoaderContextMock.httpClient.newCall( + Request.Builder() + .url(apiUrl) + .post( + body.toString().toRequestBody("application/json; charset=utf-8".toMediaType()), + ) + .tag(org.dokiteam.doki.parsers.model.MangaSource::class.java, source) + .build(), + ).execute().use { resp -> + check(resp.isSuccessful) { "GraphQL request failed ${resp.code}(${resp.message})" } + resp.body?.string().orEmpty() + } + val json = JSONObject(raw) + val errors = json.optJSONArray("errors") + check(errors == null || errors.length() == 0) { + "GraphQL error: ${errors?.optJSONObject(0)?.optString("message")}" + } + return json.optJSONObject("data") ?: JSONObject() + } + + private suspend fun checkImageResponse(url: String?) { + check(!url.isNullOrBlank()) { "Image URL is blank" } + MangaLoaderContextMock.doRequest(url, source).use { response -> + check(response.isSuccessful) { "Image request failed ${response.code}(${response.message}) for $url" } + val contentType = response.header("Content-Type").orEmpty() + check(contentType.startsWith("image/")) { "Invalid image Content-Type '$contentType' for $url" } + } + } +} diff --git a/src/test/kotlin/org/dokiteam/doki/parsers/site/madara/fr/MangasOriginesFullVerificationTest.kt b/src/test/kotlin/org/dokiteam/doki/parsers/site/madara/fr/MangasOriginesFullVerificationTest.kt new file mode 100644 index 0000000..62c6464 --- /dev/null +++ b/src/test/kotlin/org/dokiteam/doki/parsers/site/madara/fr/MangasOriginesFullVerificationTest.kt @@ -0,0 +1,85 @@ +package org.dokiteam.doki.parsers.site.madara.fr + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.dokiteam.doki.parsers.MangaLoaderContextMock +import org.dokiteam.doki.parsers.model.MangaChapter +import org.dokiteam.doki.parsers.model.MangaListFilter +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.SortOrder +import kotlin.time.Duration.Companion.minutes + +internal class MangasOriginesFullVerificationTest { + + @Test + fun verifyMangasOriginesParserEndToEnd() = runTest(timeout = 6.minutes) { + val parser = MangaLoaderContextMock.newParserInstance(MangaParserSource.MANGASORIGINES) + + val updated = parser.getList(offset = 0, order = SortOrder.UPDATED, filter = MangaListFilter.EMPTY) + check(updated.isNotEmpty()) { "Updated list is empty" } + + val alphabetical = parser.getList(offset = 0, order = SortOrder.ALPHABETICAL, filter = MangaListFilter.EMPTY) + check(alphabetical.isNotEmpty()) { "Alphabetical list is empty" } + + val query = updated.first().title + .split(Regex("\\s+")) + .firstOrNull { it.length >= 3 } + ?: updated.first().title.take(4) + val search = parser.getList( + offset = 0, + order = SortOrder.UPDATED, + filter = MangaListFilter(query = query), + ) + check(search.isNotEmpty()) { "Search returned empty list for query '$query'" } + + val detailsCandidates = updated + .take(20) + .mapNotNull { manga -> runCatching { parser.getDetails(manga) }.getOrNull() } + .filter { !it.chapters.isNullOrEmpty() } + check(detailsCandidates.isNotEmpty()) { "Unable to load detailed manga with chapters" } + val detailed = detailsCandidates.maxByOrNull { it.chapters.orEmpty().size } + ?: error("No detailed manga selected") + + check(!detailed.coverUrl.isNullOrBlank()) { "Cover URL is blank for ${detailed.publicUrl}" } + checkImageResponse(detailed.coverUrl) + check(!detailed.description.isNullOrBlank()) { "Description is blank for ${detailed.publicUrl}" } + + val chapters = detailed.chapters.orEmpty() + check(chapters.size > 1) { "Not enough chapters for ${detailed.publicUrl}" } + check(chapters.distinctBy { it.id }.size == chapters.size) { "Duplicate chapter IDs for ${detailed.publicUrl}" } + + val sampled = buildList { + add(chapters.first()) + chapters.getOrNull(chapters.size / 2)?.let { add(it) } + add(chapters.last()) + }.distinctBy { it.id } + + val sampleSummary = sampled.map { chapter -> + val pages = parser.getPages(chapter) + val pageUrls = pages.map { parser.getPageUrl(it) }.distinct() + check(pageUrls.isNotEmpty()) { "No pages for ${chapter.url}" } + check(pageUrls.all { it.startsWith("http://") || it.startsWith("https://") }) { + "Non-absolute page URLs for ${chapter.url}: $pageUrls" + } + checkImageResponse(pageUrls.first()) + checkImageResponse(pageUrls.last()) + chapter.url to pageUrls.size + } + + println( + "MANGASORIGINES_FULL_SMOKE|updated=${updated.size}|alpha=${alphabetical.size}|search=${search.size}|" + + "manga=${detailed.publicUrl}|chapters=${chapters.size}|samples=$sampleSummary", + ) + } + + private suspend fun checkImageResponse(url: String?) { + check(!url.isNullOrBlank()) { "Image URL is blank" } + MangaLoaderContextMock.doRequest(url, MangaParserSource.MANGASORIGINES).use { response -> + check(response.isSuccessful) { "Image request failed ${response.code}(${response.message}) for $url" } + val contentType = response.header("Content-Type").orEmpty() + check(contentType.startsWith("image/")) { + "Invalid image Content-Type '$contentType' for $url" + } + } + } +} diff --git a/src/test/kotlin/org/dokiteam/doki/parsers/site/mangareader/fr/SushiScanFRFullVerificationTest.kt b/src/test/kotlin/org/dokiteam/doki/parsers/site/mangareader/fr/SushiScanFRFullVerificationTest.kt new file mode 100644 index 0000000..77af14b --- /dev/null +++ b/src/test/kotlin/org/dokiteam/doki/parsers/site/mangareader/fr/SushiScanFRFullVerificationTest.kt @@ -0,0 +1,85 @@ +package org.dokiteam.doki.parsers.site.mangareader.fr + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.dokiteam.doki.parsers.MangaLoaderContextMock +import org.dokiteam.doki.parsers.model.MangaChapter +import org.dokiteam.doki.parsers.model.MangaListFilter +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.SortOrder +import kotlin.time.Duration.Companion.minutes + +internal class SushiScanFRFullVerificationTest { + + @Test + fun verifySushiScanFRParserEndToEnd() = runTest(timeout = 6.minutes) { + val parser = MangaLoaderContextMock.newParserInstance(MangaParserSource.SUSHISCANFR) + + val updated = parser.getList(offset = 0, order = SortOrder.UPDATED, filter = MangaListFilter.EMPTY) + check(updated.isNotEmpty()) { "Updated list is empty" } + + val alphabetical = parser.getList(offset = 0, order = SortOrder.ALPHABETICAL, filter = MangaListFilter.EMPTY) + check(alphabetical.isNotEmpty()) { "Alphabetical list is empty" } + + val sampleQuery = updated.first().title + .split(Regex("\\s+")) + .firstOrNull { it.length >= 3 } + ?: updated.first().title.take(4) + val search = parser.getList( + offset = 0, + order = SortOrder.UPDATED, + filter = MangaListFilter(query = sampleQuery), + ) + check(search.isNotEmpty()) { "Search returned empty list for query '$sampleQuery'" } + + val detailsCandidates = updated + .take(20) + .mapNotNull { manga -> runCatching { parser.getDetails(manga) }.getOrNull() } + .filter { !it.chapters.isNullOrEmpty() } + check(detailsCandidates.isNotEmpty()) { "Unable to load detailed manga with chapters" } + val detailed = detailsCandidates.maxByOrNull { it.chapters.orEmpty().size } + ?: error("No detailed manga selected") + + check(!detailed.coverUrl.isNullOrBlank()) { "Cover URL is blank for ${detailed.publicUrl}" } + checkImageResponse(detailed.coverUrl) + check(!detailed.description.isNullOrBlank()) { "Description is blank for ${detailed.publicUrl}" } + + val chapters = detailed.chapters.orEmpty() + check(chapters.size > 1) { "Not enough chapters for ${detailed.publicUrl}" } + check(chapters.distinctBy { it.id }.size == chapters.size) { "Duplicate chapter IDs for ${detailed.publicUrl}" } + + val sampled = buildList { + add(chapters.first()) + chapters.getOrNull(chapters.size / 2)?.let { add(it) } + add(chapters.last()) + }.distinctBy { it.id } + + val sampledSummary = sampled.map { chapter -> + val pages = parser.getPages(chapter) + val pageUrls = pages.map { parser.getPageUrl(it) }.distinct() + check(pageUrls.isNotEmpty()) { "No pages for ${chapter.url}" } + check(pageUrls.all { it.startsWith("http://") || it.startsWith("https://") }) { + "Non-absolute page URLs for ${chapter.url}: $pageUrls" + } + checkImageResponse(pageUrls.first()) + checkImageResponse(pageUrls.last()) + chapter.url to pageUrls.size + } + + println( + "SUSHISCANFR_FULL_SMOKE|updated=${updated.size}|alpha=${alphabetical.size}|search=${search.size}|" + + "manga=${detailed.publicUrl}|chapters=${chapters.size}|samples=$sampledSummary", + ) + } + + private suspend fun checkImageResponse(url: String?) { + check(!url.isNullOrBlank()) { "Image URL is blank" } + MangaLoaderContextMock.doRequest(url, MangaParserSource.SUSHISCANFR).use { response -> + check(response.isSuccessful) { "Image request failed ${response.code}(${response.message}) for $url" } + val contentType = response.header("Content-Type").orEmpty() + check(contentType.startsWith("image/")) { + "Invalid image Content-Type '$contentType' for $url" + } + } + } +} diff --git a/src/test/kotlin/org/dokiteam/doki/parsers/site/pizzareader/fr/PizzaReaderFrFullVerificationTest.kt b/src/test/kotlin/org/dokiteam/doki/parsers/site/pizzareader/fr/PizzaReaderFrFullVerificationTest.kt new file mode 100644 index 0000000..35dd1ea --- /dev/null +++ b/src/test/kotlin/org/dokiteam/doki/parsers/site/pizzareader/fr/PizzaReaderFrFullVerificationTest.kt @@ -0,0 +1,107 @@ +package org.dokiteam.doki.parsers.site.pizzareader.fr + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.dokiteam.doki.parsers.MangaLoaderContextMock +import org.dokiteam.doki.parsers.model.MangaChapter +import org.dokiteam.doki.parsers.model.MangaListFilter +import org.dokiteam.doki.parsers.model.MangaParserSource +import org.dokiteam.doki.parsers.model.SortOrder +import java.util.Locale +import kotlin.time.Duration.Companion.minutes + +internal class PizzaReaderFrFullVerificationTest { + + @Test + fun verifyBlueSoloParserEndToEnd() = runTest(timeout = 6.minutes) { + verifySource(MangaParserSource.BLUESOLO, "BLUESOLO_FULL_SMOKE") + } + + @Test + fun verifyFmTeamParserEndToEnd() = runTest(timeout = 6.minutes) { + verifySource(MangaParserSource.FMTEAM, "FMTEAM_FULL_SMOKE") + } + + private suspend fun verifySource(source: MangaParserSource, smokeLabel: String) { + val parser = MangaLoaderContextMock.newParserInstance(source) + + val updated = parser.getList(offset = 0, order = SortOrder.UPDATED, filter = MangaListFilter.EMPTY) + check(updated.isNotEmpty()) { "Updated list is empty for $source" } + + val alphabetical = parser.getList(offset = 0, order = SortOrder.ALPHABETICAL, filter = MangaListFilter.EMPTY) + check(alphabetical.isNotEmpty()) { "Alphabetical list is empty for $source" } + val expectedAlpha = alphabetical.sortedBy { it.title.lowercase(Locale.ROOT) }.map { it.id } + check(alphabetical.map { it.id } == expectedAlpha) { "Alphabetical order is incorrect for $source" } + + val query = updated.first().title + .split(Regex("\\s+")) + .firstOrNull { it.length >= 3 } + ?: updated.first().title.take(4) + val search = parser.getList( + offset = 0, + order = SortOrder.UPDATED, + filter = MangaListFilter(query = query), + ) + check(search.isNotEmpty()) { "Search returned empty list for $source query '$query'" } + + val detailsCandidates = (updated + alphabetical) + .distinctBy { it.id } + .take(30) + .mapNotNull { manga -> runCatching { parser.getDetails(manga) }.getOrNull() } + .filter { !it.chapters.isNullOrEmpty() } + check(detailsCandidates.isNotEmpty()) { "Unable to load detailed manga with chapters for $source" } + val detailed = detailsCandidates.maxByOrNull { it.chapters.orEmpty().size } + ?: error("No detailed manga selected for $source") + + check(!detailed.coverUrl.isNullOrBlank()) { "Cover URL is blank for ${detailed.publicUrl}" } + checkImageResponse(detailed.coverUrl, source) + check(!detailed.description.isNullOrBlank()) { "Description is blank for ${detailed.publicUrl}" } + + val chapters = detailed.chapters.orEmpty() + check(chapters.size > 1) { "Not enough chapters for ${detailed.publicUrl}" } + check(chapters.distinctBy { it.id }.size == chapters.size) { "Duplicate chapter IDs for ${detailed.publicUrl}" } + check(chapters.zipWithNext().all { (left, right) -> left.number <= right.number }) { + "Chapter order is not ascending for ${detailed.publicUrl}" + } + + val readableSamples = mutableListOf>>() + for (chapter in chapters) { + if (readableSamples.size >= 3) break + val pageUrls = runCatching { + parser.getPages(chapter).map { parser.getPageUrl(it) }.distinct() + }.getOrDefault(emptyList()) + if (pageUrls.isEmpty()) continue + readableSamples += chapter to pageUrls + } + check(readableSamples.isNotEmpty()) { "No readable chapter found for ${detailed.publicUrl}" } + + val sampleSummary = readableSamples.map { (chapter, pageUrls) -> + check(pageUrls.all { it.startsWith("http://") || it.startsWith("https://") }) { + "Non-absolute page URL for ${chapter.url}: $pageUrls" + } + checkImageResponse(pageUrls.first(), source) + checkImageResponse(pageUrls.last(), source) + val cachedPagesCount = parser.getPages(chapter).size + check(cachedPagesCount == pageUrls.size) { + "Page count mismatch between repeated calls for ${chapter.url}: ${pageUrls.size} != $cachedPagesCount" + } + chapter.url to pageUrls.size + } + + println( + "$smokeLabel|updated=${updated.size}|alpha=${alphabetical.size}|search=${search.size}|" + + "manga=${detailed.publicUrl}|chapters=${chapters.size}|samples=$sampleSummary", + ) + } + + private suspend fun checkImageResponse(url: String?, source: MangaParserSource) { + check(!url.isNullOrBlank()) { "Image URL is blank" } + MangaLoaderContextMock.doRequest(url, source).use { response -> + check(response.isSuccessful) { "Image request failed ${response.code}(${response.message}) for $url" } + val contentType = response.header("Content-Type").orEmpty() + check(contentType.startsWith("image/")) { + "Invalid image Content-Type '$contentType' for $url" + } + } + } +}