From 6c5fca38bdb92372d4369fb4cb51a12aea0da735 Mon Sep 17 00:00:00 2001 From: usbharu <64310155+usbharu@users.noreply.github.com> Date: Thu, 30 Nov 2023 01:19:22 +0900 Subject: [PATCH 1/5] =?UTF-8?q?fix:=20=E3=83=AA=E3=83=A2=E3=83=BC=E3=83=88?= =?UTF-8?q?=E3=81=AE=E3=83=A1=E3=83=87=E3=82=A3=E3=82=A2=E3=82=92=E5=8F=96?= =?UTF-8?q?=E5=BE=97=E3=81=99=E3=82=8B=E3=81=A8=E3=81=8D=E3=81=ABResourceR?= =?UTF-8?q?esolver=E3=82=92=E4=BD=BF=E7=94=A8=E3=81=99=E3=82=8B=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../media/RemoteMediaDownloadServiceImpl.kt | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/main/kotlin/dev/usbharu/hideout/core/service/media/RemoteMediaDownloadServiceImpl.kt b/src/main/kotlin/dev/usbharu/hideout/core/service/media/RemoteMediaDownloadServiceImpl.kt index 6bc9040c..bd1014f8 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/service/media/RemoteMediaDownloadServiceImpl.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/service/media/RemoteMediaDownloadServiceImpl.kt @@ -1,10 +1,6 @@ package dev.usbharu.hideout.core.service.media -import io.ktor.client.* -import io.ktor.client.request.* -import io.ktor.client.statement.* -import io.ktor.http.* -import io.ktor.utils.io.jvm.javaio.* +import dev.usbharu.hideout.core.service.resource.KtorResourceResolveService import org.slf4j.LoggerFactory import org.springframework.stereotype.Service import java.nio.file.Files @@ -12,16 +8,16 @@ import java.nio.file.Path import kotlin.io.path.outputStream @Service -class RemoteMediaDownloadServiceImpl(private val httpClient: HttpClient) : RemoteMediaDownloadService { +class RemoteMediaDownloadServiceImpl(private val resourceResolveService: KtorResourceResolveService) : + RemoteMediaDownloadService { override suspend fun download(url: String): Path { logger.info("START Download remote file. url: {}", url) - val httpResponse = httpClient.get(url) - httpResponse.contentLength() + val httpResponse = resourceResolveService.resolve(url).body() val createTempFile = Files.createTempFile("hideout-remote-download", ".tmp") logger.debug("Save to {} url: {} ", createTempFile, url) - httpResponse.bodyAsChannel().toInputStream().use { inputStream -> + httpResponse.use { inputStream -> createTempFile.outputStream().use { inputStream.transferTo(it) } From 0cfd078d4be22fe78094e94b7276edde8236a504 Mon Sep 17 00:00:00 2001 From: usbharu <64310155+usbharu@users.noreply.github.com> Date: Thu, 30 Nov 2023 01:28:44 +0900 Subject: [PATCH 2/5] =?UTF-8?q?feat:=20=E3=83=AA=E3=83=A2=E3=83=BC?= =?UTF-8?q?=E3=83=88=E3=81=AE=E3=83=A1=E3=83=87=E3=82=A3=E3=82=A2=E4=BF=9D?= =?UTF-8?q?=E5=AD=98=E5=89=8D=E3=81=AB=E9=87=8D=E8=A4=87=E3=83=81=E3=82=A7?= =?UTF-8?q?=E3=83=83=E3=82=AF=E3=82=92=E5=AE=9F=E6=96=BD=E3=81=99=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../exposedquery/MediaQueryServiceImpl.kt | 15 ++++++++++++--- .../exposedrepository/MediaRepositoryImpl.kt | 6 +++--- .../hideout/core/query/MediaQueryService.kt | 1 + .../core/service/media/MediaServiceImpl.kt | 12 +++++++++++- src/main/resources/db/migration/V1__Init_DB.sql | 6 +++--- 5 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedquery/MediaQueryServiceImpl.kt b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedquery/MediaQueryServiceImpl.kt index 473c5f66..7b5a5d8e 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedquery/MediaQueryServiceImpl.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedquery/MediaQueryServiceImpl.kt @@ -1,17 +1,20 @@ package dev.usbharu.hideout.core.infrastructure.exposedquery -import dev.usbharu.hideout.core.domain.model.media.Media +import dev.usbharu.hideout.core.domain.exception.FailedToGetResourcesException +import dev.usbharu.hideout.core.infrastructure.exposedrepository.Media import dev.usbharu.hideout.core.infrastructure.exposedrepository.PostsMedia import dev.usbharu.hideout.core.infrastructure.exposedrepository.toMedia import dev.usbharu.hideout.core.query.MediaQueryService +import dev.usbharu.hideout.util.singleOr import org.jetbrains.exposed.sql.innerJoin import org.jetbrains.exposed.sql.select import org.springframework.stereotype.Repository +import dev.usbharu.hideout.core.domain.model.media.Media as MediaEntity @Repository class MediaQueryServiceImpl : MediaQueryService { - override suspend fun findByPostId(postId: Long): List { - return dev.usbharu.hideout.core.infrastructure.exposedrepository.Media.innerJoin( + override suspend fun findByPostId(postId: Long): List { + return Media.innerJoin( PostsMedia, onColumn = { id }, otherColumn = { mediaId } @@ -19,4 +22,10 @@ class MediaQueryServiceImpl : MediaQueryService { .select { PostsMedia.postId eq postId } .map { it.toMedia() } } + + override suspend fun findByRemoteUrl(remoteUrl: String): MediaEntity { + return Media.select { Media.remoteUrl eq remoteUrl } + .singleOr { FailedToGetResourcesException("remoteUrl: $remoteUrl is duplicate or not exist.", it) } + .toMedia() + } } diff --git a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/MediaRepositoryImpl.kt b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/MediaRepositoryImpl.kt index 0206feb9..97b7c527 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/MediaRepositoryImpl.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/MediaRepositoryImpl.kt @@ -99,9 +99,9 @@ fun ResultRow.toMediaOrNull(): EntityMedia? { object Media : Table("media") { val id = long("id") val name = varchar("name", 255) - val url = varchar("url", 255) - val remoteUrl = varchar("remote_url", 255).nullable() - val thumbnailUrl = varchar("thumbnail_url", 255).nullable() + val url = varchar("url", 255).uniqueIndex() + val remoteUrl = varchar("remote_url", 255).uniqueIndex().nullable() + val thumbnailUrl = varchar("thumbnail_url", 255).uniqueIndex().nullable() val type = integer("type") val blurhash = varchar("blurhash", 255).nullable() val mimeType = varchar("mime_type", 255) diff --git a/src/main/kotlin/dev/usbharu/hideout/core/query/MediaQueryService.kt b/src/main/kotlin/dev/usbharu/hideout/core/query/MediaQueryService.kt index fc7fb675..876c2f1e 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/query/MediaQueryService.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/query/MediaQueryService.kt @@ -4,4 +4,5 @@ import dev.usbharu.hideout.core.domain.model.media.Media interface MediaQueryService { suspend fun findByPostId(postId: Long): List + suspend fun findByRemoteUrl(remoteUrl: String): Media } diff --git a/src/main/kotlin/dev/usbharu/hideout/core/service/media/MediaServiceImpl.kt b/src/main/kotlin/dev/usbharu/hideout/core/service/media/MediaServiceImpl.kt index 5337e53a..80d26318 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/service/media/MediaServiceImpl.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/service/media/MediaServiceImpl.kt @@ -1,9 +1,11 @@ package dev.usbharu.hideout.core.service.media +import dev.usbharu.hideout.core.domain.exception.FailedToGetResourcesException import dev.usbharu.hideout.core.domain.exception.media.MediaSaveException import dev.usbharu.hideout.core.domain.exception.media.UnsupportedMediaException import dev.usbharu.hideout.core.domain.model.media.Media import dev.usbharu.hideout.core.domain.model.media.MediaRepository +import dev.usbharu.hideout.core.query.MediaQueryService import dev.usbharu.hideout.core.service.media.converter.MediaProcessService import dev.usbharu.hideout.mastodon.interfaces.api.media.MediaRequest import dev.usbharu.hideout.util.withDelete @@ -22,7 +24,8 @@ class MediaServiceImpl( private val mediaRepository: MediaRepository, private val mediaProcessServices: List, private val remoteMediaDownloadService: RemoteMediaDownloadService, - private val renameService: MediaFileRenameService + private val renameService: MediaFileRenameService, + private val mediaQueryService: MediaQueryService ) : MediaService { @Suppress("LongMethod", "NestedBlockDepth") override suspend fun uploadLocalMedia(mediaRequest: MediaRequest): EntityMedia { @@ -99,6 +102,13 @@ class MediaServiceImpl( override suspend fun uploadRemoteMedia(remoteMedia: RemoteMedia): Media { logger.info("MEDIA Remote media. filename:${remoteMedia.name} url:${remoteMedia.url}") + try { + val findByRemoteUrl = mediaQueryService.findByRemoteUrl(remoteMedia.url) + logger.warn("DUPLICATED Remote media is duplicated. url: {}", remoteMedia.url) + return findByRemoteUrl + } catch (_: FailedToGetResourcesException) { + } + remoteMediaDownloadService.download(remoteMedia.url).withDelete().use { val mimeType = fileTypeDeterminationService.fileType(it.path, remoteMedia.name) diff --git a/src/main/resources/db/migration/V1__Init_DB.sql b/src/main/resources/db/migration/V1__Init_DB.sql index 4ea80255..e0188588 100644 --- a/src/main/resources/db/migration/V1__Init_DB.sql +++ b/src/main/resources/db/migration/V1__Init_DB.sql @@ -46,9 +46,9 @@ create table if not exists media ( id bigint primary key, "name" varchar(255) not null, - url varchar(255) not null, - remote_url varchar(255) null, - thumbnail_url varchar(255) null, + url varchar(255) not null unique, + remote_url varchar(255) null unique, + thumbnail_url varchar(255) null unique, "type" int not null, blurhash varchar(255) null, mime_type varchar(255) not null, From 4b936004ad6905bc053b1862a5e19b6bed093f0e Mon Sep 17 00:00:00 2001 From: usbharu <64310155+usbharu@users.noreply.github.com> Date: Thu, 30 Nov 2023 01:50:44 +0900 Subject: [PATCH 3/5] =?UTF-8?q?test:=20=E3=83=A6=E3=83=8B=E3=83=BC?= =?UTF-8?q?=E3=82=AF=E3=82=A4=E3=83=B3=E3=83=87=E3=83=83=E3=82=AF=E3=82=B9?= =?UTF-8?q?=E3=81=AE=E5=88=B6=E7=B4=84=E3=81=AB=E5=AF=BE=E5=BF=9C=E3=81=99?= =?UTF-8?q?=E3=82=8B=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/intTest/kotlin/mastodon/media/MediaTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/intTest/kotlin/mastodon/media/MediaTest.kt b/src/intTest/kotlin/mastodon/media/MediaTest.kt index 4c9ee710..43a881cf 100644 --- a/src/intTest/kotlin/mastodon/media/MediaTest.kt +++ b/src/intTest/kotlin/mastodon/media/MediaTest.kt @@ -52,7 +52,7 @@ class MediaTest { @Test fun メディアをアップロードできる() = runTest { - whenever(mediaDataStore.save(any())).doReturn(SuccessSavedMedia("", "", "")) + whenever(mediaDataStore.save(any())).doReturn(SuccessSavedMedia("", "a", "a")) mockMvc .multipart("/api/v1/media") { @@ -73,7 +73,7 @@ class MediaTest { @Test fun write_mediaスコープでメディアをアップロードできる() = runTest { - whenever(mediaDataStore.save(any())).doReturn(SuccessSavedMedia("", "", "")) + whenever(mediaDataStore.save(any())).doReturn(SuccessSavedMedia("", "b", "b")) mockMvc .multipart("/api/v1/media") { From 7cbdfd45567305d8d8109fc04c9573cad8b8bf5e Mon Sep 17 00:00:00 2001 From: usbharu <64310155+usbharu@users.noreply.github.com> Date: Thu, 30 Nov 2023 01:51:06 +0900 Subject: [PATCH 4/5] =?UTF-8?q?test:=20=E7=B5=90=E5=90=88=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=82=E3=83=A9=E3=83=B3=E3=83=80=E3=83=A0?= =?UTF-8?q?=E9=A0=86=E3=81=A7=E5=AE=9F=E8=A1=8C=E3=81=99=E3=82=8B=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/intTest/resources/junit-platform.properties | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src/intTest/resources/junit-platform.properties diff --git a/src/intTest/resources/junit-platform.properties b/src/intTest/resources/junit-platform.properties new file mode 100644 index 00000000..acfa9e5a --- /dev/null +++ b/src/intTest/resources/junit-platform.properties @@ -0,0 +1,2 @@ +junit.jupiter.testclass.order.default=org.junit.jupiter.api.ClassOrderer$Random +junit.jupiter.testmethod.order.default=org.junit.jupiter.api.MethodOrderer$Random From 33ea67fd7526e12493b882ac90073b2627b9ea81 Mon Sep 17 00:00:00 2001 From: usbharu <64310155+usbharu@users.noreply.github.com> Date: Thu, 30 Nov 2023 01:56:29 +0900 Subject: [PATCH 5/5] =?UTF-8?q?test:=20=E5=AD=98=E5=9C=A8=E3=81=97?= =?UTF-8?q?=E3=81=AA=E3=81=84=E3=83=A6=E3=83=BC=E3=82=B6=E3=83=BC=E3=81=AE?= =?UTF-8?q?=E5=90=8D=E5=89=8D=E3=82=92=E8=A4=87=E9=9B=91=E3=81=AB=E3=81=97?= =?UTF-8?q?=E3=81=A6=E3=81=86=E3=81=A3=E3=81=8B=E3=82=8A=E8=A2=AB=E3=81=A3?= =?UTF-8?q?=E3=81=9F=E3=82=8A=E3=81=97=E3=81=AA=E3=81=84=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/intTest/kotlin/activitypub/webfinger/WebFingerTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/intTest/kotlin/activitypub/webfinger/WebFingerTest.kt b/src/intTest/kotlin/activitypub/webfinger/WebFingerTest.kt index abee25d6..8e0b0294 100644 --- a/src/intTest/kotlin/activitypub/webfinger/WebFingerTest.kt +++ b/src/intTest/kotlin/activitypub/webfinger/WebFingerTest.kt @@ -55,7 +55,7 @@ class WebFingerTest { @Test fun `webfinger 存在しないユーザーに404`() { mockMvc - .get("/.well-known/webfinger?resource=acct:test-user@example.com") + .get("/.well-known/webfinger?resource=acct:invalid-user-notfound-afdjashfal@example.com") .andExpect { status { isNotFound() } } }