thumbnail caching

server/src/main/kotlin/ir/armor/tachidesk/util/File.kt

@@ -0,0 +1,30 @@
+package ir.armor.tachidesk.util
+
+import java.io.BufferedInputStream
+import java.io.File
+import java.io.FileInputStream
+import java.io.InputStream
+import java.nio.file.Files
+import java.nio.file.Paths
+
+fun writeStream(fileStream: InputStream, path: String) {
+    Files.newOutputStream(Paths.get(path)).use { os ->
+        val buffer = ByteArray(1024)
+        var len: Int
+        while (fileStream.read(buffer).also { len = it } > 0) {
+            os.write(buffer, 0, len)
+        }
+    }
+}
+
+fun pathToInputStream(path: String): InputStream {
+    return BufferedInputStream(FileInputStream(path))
+}
+
+fun findFileNameStartingWith(directoryPath: String, fileName: String): String? {
+    File(directoryPath).listFiles().forEach { file ->
+        if (file.name.startsWith(fileName))
+            return "$directoryPath/${file.name}"
+    }
+    return null
+}

server/src/main/kotlin/ir/armor/tachidesk/util/Manga.kt

@@ -4,77 +4,168 @@ package ir.armor.tachidesk.util
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.source.model.SManga
+import ir.armor.tachidesk.Config
 import ir.armor.tachidesk.database.dataclass.MangaDataClass
 import ir.armor.tachidesk.database.table.MangaStatus
 import ir.armor.tachidesk.database.table.MangaTable
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
+import java.io.InputStream
+import java.util.concurrent.ArrayBlockingQueue
 
-fun getManga(mangaId: Int): MangaDataClass {
-    return transaction {
-        var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
+val getMangaUpdateQueue = ArrayBlockingQueue<Pair<Int, SManga?>>(1000)
+@Volatile
+var getMangaCount = 0
 
-        return@transaction if (mangaEntry[MangaTable.initialized]) {
-            MangaDataClass(
-                mangaId,
-                mangaEntry[MangaTable.sourceReference].value,
+val getMangaUpdateQueueThread = Runnable {
+    while (true) {
+        val p = getMangaUpdateQueue.take()
+        println("took ${p.first}")
+        while (getMangaCount > 0) {
+            println("count is $getMangaCount")
+            Thread.sleep(1000)
+        }
+        val mangaId = p.first
+        println("working on $mangaId")
+        val fetchedManga = p.second!!
+        try {
+            transaction {
+                println("transaction start $mangaId")
+                MangaTable.update({ MangaTable.id eq mangaId }) {
 
-                mangaEntry[MangaTable.url],
-                mangaEntry[MangaTable.title],
-                mangaEntry[MangaTable.thumbnail_url],
+                    it[MangaTable.initialized] = true
 
-                true,
-
-                mangaEntry[MangaTable.artist],
-                mangaEntry[MangaTable.author],
-                mangaEntry[MangaTable.description],
-                mangaEntry[MangaTable.genre],
-                MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
-            )
-        } else { // initialize manga
-            val source = getHttpSource(mangaEntry[MangaTable.sourceReference].value)
-            val fetchedManga = source.fetchMangaDetails(
-                SManga.create().apply {
-                    url = mangaEntry[MangaTable.url]
-                    title = mangaEntry[MangaTable.title]
+                    it[MangaTable.artist] = fetchedManga.artist
+                    it[MangaTable.author] = fetchedManga.author
+                    it[MangaTable.description] = fetchedManga.description
+                    it[MangaTable.genre] = fetchedManga.genre
+                    it[MangaTable.status] = fetchedManga.status
+                    if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty())
+                        it[MangaTable.thumbnail_url] = fetchedManga.thumbnail_url
                 }
-            ).toBlocking().first()
-
-            // update database
-            MangaTable.update({ MangaTable.id eq mangaId }) {
-//            it[url] = fetchedManga.url
-//            it[title] = fetchedManga.title
-                it[initialized] = true
-
-                it[artist] = fetchedManga.artist
-                it[author] = fetchedManga.author
-                it[description] = fetchedManga.description
-                it[genre] = fetchedManga.genre
-                it[status] = fetchedManga.status
-                if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty())
-                    it[thumbnail_url] = fetchedManga.thumbnail_url
+                println("transaction end $mangaId")
             }
-
-            mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
-
-            MangaDataClass(
-                mangaId,
-                mangaEntry[MangaTable.sourceReference].value,
-
-                mangaEntry[MangaTable.url],
-                mangaEntry[MangaTable.title],
-                mangaEntry[MangaTable.thumbnail_url],
-
-                true,
-
-                mangaEntry[MangaTable.artist],
-                mangaEntry[MangaTable.author],
-                mangaEntry[MangaTable.description],
-                mangaEntry[MangaTable.genre],
-                MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
-            )
+        } catch (e: Exception) {
+            println(e)
+        }
+    }
+}
+
+fun getManga(mangaId: Int, proxyThumbnail: Boolean = true): MangaDataClass {
+    synchronized(getMangaCount) {
+        getMangaCount++
+    }
+    return try {
+        transaction {
+            var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
+
+            return@transaction if (mangaEntry[MangaTable.initialized]) {
+                println("${mangaEntry[MangaTable.title]} is initialized")
+                println("${mangaEntry[MangaTable.thumbnail_url]}")
+                MangaDataClass(
+                        mangaId,
+                        mangaEntry[MangaTable.sourceReference].value,
+
+                        mangaEntry[MangaTable.url],
+                        mangaEntry[MangaTable.title],
+                        if (proxyThumbnail) proxyThumbnailUrl(mangaId) else mangaEntry[MangaTable.thumbnail_url],
+
+                        true,
+
+                        mangaEntry[MangaTable.artist],
+                        mangaEntry[MangaTable.author],
+                        mangaEntry[MangaTable.description],
+                        mangaEntry[MangaTable.genre],
+                        MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
+                )
+            } else { // initialize manga
+                val source = getHttpSource(mangaEntry[MangaTable.sourceReference].value)
+                val fetchedManga = source.fetchMangaDetails(
+                        SManga.create().apply {
+                            url = mangaEntry[MangaTable.url]
+                            title = mangaEntry[MangaTable.title]
+                        }
+                ).toBlocking().first()
+
+                // update database
+                // TODO: sqlite gets fucked here
+                println("putting $mangaId")
+                getMangaUpdateQueue.put(Pair(mangaId, fetchedManga))
+
+//            mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
+                val newThumbnail =
+                        if (fetchedManga.thumbnail_url != null && fetchedManga.thumbnail_url!!.isNotEmpty()) {
+                            fetchedManga.thumbnail_url
+                        } else mangaEntry[MangaTable.thumbnail_url]
+
+                MangaDataClass(
+                        mangaId,
+                        mangaEntry[MangaTable.sourceReference].value,
+
+                        mangaEntry[MangaTable.url],
+                        mangaEntry[MangaTable.title],
+                        if (proxyThumbnail) proxyThumbnailUrl(mangaId) else newThumbnail,
+
+                        true,
+
+                        fetchedManga.artist,
+                        fetchedManga.author,
+                        fetchedManga.description,
+                        fetchedManga.genre,
+                        MangaStatus.valueOf(fetchedManga.status).name,
+                )
+            }
+        }
+    } finally {
+        synchronized(getMangaCount) {
+            getMangaCount--
+        }
+    }
+}
+
+fun getThumbnail(mangaId: Int): Pair<InputStream, String> {
+    return transaction {
+        var filePath = Config.thumbnailsRoot + "/$mangaId"
+        var mangaEntry = MangaTable.select { MangaTable.id eq mangaId }.firstOrNull()!!
+
+        val potentialCache = findFileNameStartingWith(Config.thumbnailsRoot, mangaId.toString())
+        if (potentialCache != null) {
+            println("using cached thumbnail file")
+            return@transaction Pair(
+                    pathToInputStream(potentialCache),
+                    "image/${potentialCache.substringAfter("$mangaId.")}"
+            )
+        }
+
+        val sourceId = mangaEntry[MangaTable.sourceReference].value
+        println("getting source for $mangaId")
+        val source = getHttpSource(sourceId)
+        var thumbnailUrl = mangaEntry[MangaTable.thumbnail_url]
+        if (thumbnailUrl == null || thumbnailUrl.isEmpty()) {
+            thumbnailUrl = getManga(mangaId, proxyThumbnail = false).thumbnailUrl!!
+        }
+        println(thumbnailUrl)
+        val response = source.client.newCall(
+                GET(thumbnailUrl, source.headers)
+        ).execute()
+
+        println(response.code)
+
+        if (response.code == 200) {
+            val contentType = response.headers["content-type"]!!
+            filePath += "." + contentType.substringAfter("image/")
+
+            writeStream(response.body!!.byteStream(), filePath)
+
+            return@transaction Pair(
+                    pathToInputStream(filePath),
+                    contentType
+            )
+        } else {
+            throw Exception("request error! ${response.code}")
         }
     }
 }

server/src/main/kotlin/ir/armor/tachidesk/util/MangaList.kt

@@ -13,6 +13,10 @@ import org.jetbrains.exposed.sql.insertAndGetId
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 
+fun proxyThumbnailUrl(mangaId: Int): String {
+    return "http://127.0.0.1:4567/api/v1/manga/$mangaId/thumbnail"
+}
+
 fun getMangaList(sourceId: Long, pageNum: Int = 1, popular: Boolean): PagedMangaListDataClass {
     val source = getHttpSource(sourceId.toLong())
     val mangasPage = if (popular) {
@@ -31,8 +35,8 @@ fun MangasPage.processEntries(sourceId: Long): PagedMangaListDataClass {
     val mangaList = transaction {
         return@transaction mangasPage.mangas.map { manga ->
             var mangaEntry = MangaTable.select { MangaTable.url eq manga.url }.firstOrNull()
-            var mangaEntityId = if (mangaEntry == null) { // create manga entry
-                MangaTable.insertAndGetId {
+            if (mangaEntry == null) { // create manga entry
+                val mangaId = MangaTable.insertAndGetId {
                     it[url] = manga.url
                     it[title] = manga.title
 
@@ -41,30 +45,46 @@ fun MangasPage.processEntries(sourceId: Long): PagedMangaListDataClass {
                     it[description] = manga.description
                     it[genre] = manga.genre
                     it[status] = manga.status
-                    it[thumbnail_url] = manga.genre
+                    it[thumbnail_url] = manga.thumbnail_url
 
                     it[sourceReference] = sourceId
                 }.value
+
+                MangaDataClass(
+                    mangaId,
+                    sourceId,
+
+                    manga.url,
+                    manga.title,
+                    proxyThumbnailUrl(mangaId),
+
+                    manga.initialized,
+
+                    manga.artist,
+                    manga.author,
+                    manga.description,
+                    manga.genre,
+                    MangaStatus.valueOf(manga.status).name,
+                )
             } else {
-                mangaEntry[MangaTable.id].value
+                val mangaId = mangaEntry[MangaTable.id].value
+                MangaDataClass(
+                    mangaId,
+                    sourceId,
+
+                    manga.url,
+                    manga.title,
+                    proxyThumbnailUrl(mangaId),
+
+                    true,
+
+                    mangaEntry[MangaTable.artist],
+                    mangaEntry[MangaTable.author],
+                    mangaEntry[MangaTable.description],
+                    mangaEntry[MangaTable.genre],
+                    MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
+                )
             }
-
-            MangaDataClass(
-                mangaEntityId,
-                sourceId,
-
-                manga.url,
-                manga.title,
-                manga.thumbnail_url,
-
-                manga.initialized,
-
-                manga.artist,
-                manga.author,
-                manga.description,
-                manga.genre,
-                MangaStatus.valueOf(manga.status).name,
-            )
         }
     }
     return PagedMangaListDataClass(

server/src/main/kotlin/ir/armor/tachidesk/util/Util.kt

@@ -12,6 +12,7 @@ fun applicationSetup() {
     // make dirs we need
     File(Config.dataRoot).mkdirs()
     File(Config.extensionsRoot).mkdirs()
+    File(Config.thumbnailsRoot).mkdirs()
 
     makeDataBaseTables()
 }