usbharu
/
Hideout
mirror of https://github.com/usbharu/Hideout.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #145 from usbharu/feature/refactor-media 

メディア関係の改善
This commit is contained in:
usbharu 2023-11-16 21:04:07 +09:00 committed by GitHub
parent 19de2d544d ac0e05ade8
commit 7c105e946c
37 changed files with 756 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build.gradle.kts
View File

@ -179,7 +179,9 @@ dependencies {
implementation("org.postgresql:postgresql:42.6.0")
implementation("com.twelvemonkeys.imageio:imageio-webp:3.10.0")
implementation("org.apache.tika:tika-core:2.9.1")
implementation("net.coobird:thumbnailator:0.4.20")
implementation("org.bytedeco:javacv-platform:1.5.9")
implementation("io.ktor:ktor-client-logging-jvm:$ktor_version")

6
src/intTest/resources/sql/note/メディア付き投稿はattachmentにDocumentとして画像が存在する.sql
View File

@ -13,9 +13,9 @@ insert into POSTS (ID, "userId", OVERVIEW, TEXT, "createdAt", VISIBILITY, URL, "
VALUES (1242, 11, null, 'test post', 12345680, 0, 'https://example.com/users/test-user11/posts/1242', null, null, false,
'https://example.com/users/test-user11/posts/1242');
insert into MEDIA (ID, NAME, URL, REMOTE_URL, THUMBNAIL_URL, TYPE, BLURHASH)
VALUES (1, 'test-media', 'https://example.com/media/test-media.png', null, null, 0, null),
(2, 'test-media2', 'https://example.com/media/test-media2.png', null, null, 0, null);
insert into MEDIA (ID, NAME, URL, REMOTE_URL, THUMBNAIL_URL, TYPE, BLURHASH, MIME_TYPE, DESCRIPTION)
VALUES (1, 'test-media', 'https://example.com/media/test-media.png', null, null, 0, null, 'image/png', null),
(2, 'test-media2', 'https://example.com/media/test-media2.png', null, null, 0, null, 'image/png', null);
insert into POSTSMEDIA(POST_ID, MEDIA_ID)
VALUES (1242, 1),

4
src/main/kotlin/dev/usbharu/hideout/activitypub/domain/model/Delete.kt
View File

@ -41,7 +41,5 @@ open class Delete : Object {
return result
}
override fun toString(): String {
return "Delete(`object`=$`object`, published=$published) ${super.toString()}"
}
override fun toString(): String = "Delete(`object`=$`object`, published=$published) ${super.toString()}"
}

8
src/main/kotlin/dev/usbharu/hideout/activitypub/domain/model/JsonLd.kt
View File

@ -62,13 +62,9 @@ class ContextDeserializer : JsonDeserializer<String>() {
class ContextSerializer : JsonSerializer<List<String>>() {
override fun isEmpty(value: List<String>?): Boolean {
return value.isNullOrEmpty()
}
override fun isEmpty(value: List<String>?): Boolean = value.isNullOrEmpty()
override fun isEmpty(provider: SerializerProvider?, value: List<String>?): Boolean {
return value.isNullOrEmpty()
}
override fun isEmpty(provider: SerializerProvider?, value: List<String>?): Boolean = value.isNullOrEmpty()
override fun serialize(value: List<String>?, gen: JsonGenerator?, serializers: SerializerProvider) {
if (value.isNullOrEmpty()) {

4
src/main/kotlin/dev/usbharu/hideout/activitypub/domain/model/objects/Object.kt
View File

@ -46,9 +46,7 @@ open class Object : JsonLd {
return result
}
override fun toString(): String {
return "Object(type=$type, name=$name, actor=$actor, id=$id) ${super.toString()}"
}
override fun toString(): String = "Object(type=$type, name=$name, actor=$actor, id=$id) ${super.toString()}"
companion object {
@JvmStatic

2
src/main/kotlin/dev/usbharu/hideout/activitypub/interfaces/api/actor/UserAPControllerImpl.kt
View File

@ -12,7 +12,7 @@ class UserAPControllerImpl(private val apUserService: APUserService) : UserAPCon
override suspend fun userAp(username: String): ResponseEntity<Person> {
val person = try {
apUserService.getPersonByName(username)
} catch (e: FailedToGetResourcesException) {
} catch (_: FailedToGetResourcesException) {
return ResponseEntity.notFound().build()
}
person.context += listOf("https://www.w3.org/ns/activitystreams")

2
src/main/kotlin/dev/usbharu/hideout/activitypub/service/activity/delete/APReceiveDeleteServiceImpl.kt
View File

@ -22,7 +22,7 @@ class APReceiveDeleteServiceImpl(
val post = try {
postQueryService.findByApId(deleteId)
} catch (e: FailedToGetResourcesException) {
} catch (_: FailedToGetResourcesException) {
return@transaction ActivityPubStringResponse(HttpStatusCode.OK, "Resource not found or already deleted")
}
postRepository.delete(post.id)

6
src/main/kotlin/dev/usbharu/hideout/activitypub/service/objects/note/APNoteService.kt
View File

@ -1,6 +1,5 @@
package dev.usbharu.hideout.activitypub.service.objects.note
import com.fasterxml.jackson.databind.ObjectMapper
import dev.usbharu.hideout.activitypub.domain.exception.FailedToGetActivityPubResourceException
import dev.usbharu.hideout.activitypub.domain.exception.IllegalActivityPubObjectException
import dev.usbharu.hideout.activitypub.domain.model.Note
@ -24,7 +23,6 @@ import kotlinx.coroutines.async
import kotlinx.coroutines.slf4j.MDCContext
import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.cache.annotation.Cacheable
import org.springframework.stereotype.Service
import java.time.Instant
@ -50,7 +48,6 @@ class APNoteServiceImpl(
private val postRepository: PostRepository,
private val apUserService: APUserService,
private val postQueryService: PostQueryService,
@Qualifier("activitypub") private val objectMapper: ObjectMapper,
private val postService: PostService,
private val apResourceResolveService: APResourceResolveService,
private val postBuilder: Post.PostBuilder,
@ -133,7 +130,8 @@ class APNoteServiceImpl(
RemoteMedia(
(it.name ?: it.url)!!,
it.url!!,
it.mediaType ?: "application/octet-stream"
it.mediaType ?: "application/octet-stream",
description = it.name
)
)
}

2
src/main/kotlin/dev/usbharu/hideout/application/config/HttpClientConfig.kt
View File

@ -13,7 +13,7 @@ class HttpClientConfig {
fun httpClient(): HttpClient = HttpClient(CIO).config {
install(Logging) {
logger = Logger.DEFAULT
level = LogLevel.INFO
level = LogLevel.ALL
}
install(HttpCache) {
}

16
src/main/kotlin/dev/usbharu/hideout/application/config/SecurityConfig.kt
View File

@ -59,7 +59,7 @@ import java.security.interfaces.RSAPrivateKey
import java.security.interfaces.RSAPublicKey
import java.util.*
@EnableWebSecurity(debug = true)
@EnableWebSecurity(debug = false)
@Configuration
@Suppress("FunctionMaxLength", "TooManyFunctions")
class SecurityConfig {
@ -73,7 +73,12 @@ class SecurityConfig {
@Bean
@Order(1)
fun httpSignatureFilterChain(http: HttpSecurity, httpSignatureFilter: HttpSignatureFilter): SecurityFilterChain {
fun httpSignatureFilterChain(
http: HttpSecurity,
httpSignatureFilter: HttpSignatureFilter,
introspector: HandlerMappingIntrospector
): SecurityFilterChain {
val builder = MvcRequestMatcher.Builder(introspector)
http
.securityMatcher("/inbox", "/outbox", "/users/*/inbox", "/users/*/outbox", "/users/*/posts/*")
.addFilter(httpSignatureFilter)
@ -82,7 +87,12 @@ class SecurityConfig {
HttpSignatureFilter::class.java
)
.authorizeHttpRequests {
it.requestMatchers("/inbox", "/outbox", "/users/*/inbox", "/users/*/outbox").authenticated()
it.requestMatchers(
builder.pattern("/inbox"),
builder.pattern("/outbox"),
builder.pattern("/users/*/inbox"),
builder.pattern("/users/*/outbox")
).authenticated()
it.anyRequest().permitAll()
}
.csrf {

14
src/main/kotlin/dev/usbharu/hideout/core/domain/exception/media/MediaProcessException.kt Normal file
View File

@ -0,0 +1,14 @@
package dev.usbharu.hideout.core.domain.exception.media
class MediaProcessException : MediaException {
constructor() : super()
constructor(message: String?) : super(message)
constructor(message: String?, cause: Throwable?) : super(message, cause)
constructor(cause: Throwable?) : super(cause)
constructor(message: String?, cause: Throwable?, enableSuppression: Boolean, writableStackTrace: Boolean) : super(
message,
cause,
enableSuppression,
writableStackTrace
)
}

5
src/main/kotlin/dev/usbharu/hideout/core/domain/model/media/Media.kt
View File

@ -1,6 +1,7 @@
package dev.usbharu.hideout.core.domain.model.media
import dev.usbharu.hideout.core.service.media.FileType
import dev.usbharu.hideout.core.service.media.MimeType
data class Media(
val id: Long,
@ -9,5 +10,7 @@ data class Media(
val remoteUrl: String?,
val thumbnailUrl: String?,
val type: FileType,
val blurHash: String?
val mimeType: MimeType,
val blurHash: String?,
val description: String? = null
)

18
src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/MediaRepositoryImpl.kt
View File

@ -3,7 +3,9 @@ package dev.usbharu.hideout.core.infrastructure.exposedrepository
import dev.usbharu.hideout.application.service.id.IdGenerateService
import dev.usbharu.hideout.core.domain.exception.FailedToGetResourcesException
import dev.usbharu.hideout.core.domain.model.media.MediaRepository
import dev.usbharu.hideout.core.infrastructure.exposedrepository.Media.mimeType
import dev.usbharu.hideout.core.service.media.FileType
import dev.usbharu.hideout.core.service.media.MimeType
import dev.usbharu.hideout.util.singleOr
import org.jetbrains.exposed.sql.*
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
@ -26,6 +28,8 @@ class MediaRepositoryImpl(private val idGenerateService: IdGenerateService) : Me
it[thumbnailUrl] = media.thumbnailUrl
it[type] = media.type.ordinal
it[blurhash] = media.blurHash
it[mimeType] = media.mimeType.type + "/" + media.mimeType.subtype
it[description] = media.description
}
} else {
Media.insert {
@ -36,6 +40,8 @@ class MediaRepositoryImpl(private val idGenerateService: IdGenerateService) : Me
it[thumbnailUrl] = media.thumbnailUrl
it[type] = media.type.ordinal
it[blurhash] = media.blurHash
it[mimeType] = media.mimeType.type + "/" + media.mimeType.subtype
it[description] = media.description
}
}
return media
@ -59,18 +65,24 @@ class MediaRepositoryImpl(private val idGenerateService: IdGenerateService) : Me
}
fun ResultRow.toMedia(): EntityMedia {
val fileType = FileType.values().first { it.ordinal == this[Media.type] }
val mimeType = this[Media.mimeType]
return EntityMedia(
id = this[Media.id],
name = this[Media.name],
url = this[Media.url],
remoteUrl = this[Media.remoteUrl],
thumbnailUrl = this[Media.thumbnailUrl],
type = FileType.values().first { it.ordinal == this[Media.type] },
type = fileType,
blurHash = this[Media.blurhash],
mimeType = MimeType(mimeType.substringBefore("/"), mimeType.substringAfter("/"), fileType),
description = this[Media.description]
)
}
fun ResultRow.toMediaOrNull(): EntityMedia? {
val fileType = FileType.values().first { it.ordinal == (this.getOrNull(Media.type) ?: return null) }
val mimeType = this.getOrNull(Media.mimeType) ?: return null
return EntityMedia(
id = this.getOrNull(Media.id) ?: return null,
name = this.getOrNull(Media.name) ?: return null,
@ -79,6 +91,8 @@ fun ResultRow.toMediaOrNull(): EntityMedia? {
thumbnailUrl = this[Media.thumbnailUrl],
type = FileType.values().first { it.ordinal == this.getOrNull(Media.type) },
blurHash = this[Media.blurhash],
mimeType = MimeType(mimeType.substringBefore("/"), mimeType.substringAfter("/"), fileType),
description = this[Media.description]
)
}
@ -90,5 +104,7 @@ object Media : Table("media") {
val thumbnailUrl = varchar("thumbnail_url", 255).nullable()
val type = integer("type")
val blurhash = varchar("blurhash", 255).nullable()
val mimeType = varchar("mime_type", 255)
val description = varchar("description", 4000).nullable()
override val primaryKey = PrimaryKey(id)
}

1
src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/PostRepositoryImpl.kt
View File

@ -73,7 +73,6 @@ class PostRepositoryImpl(
.let(postQueryMapper::map)
.singleOr { FailedToGetResourcesException("id: $id was not found.", it) }
override suspend fun delete(id: Long) {
Posts.deleteWhere { Posts.id eq id }
}

89
src/main/kotlin/dev/usbharu/hideout/core/service/media/ApatcheTikaFileTypeDeterminationService.kt Normal file
View File

@ -0,0 +1,89 @@
package dev.usbharu.hideout.core.service.media
import org.apache.tika.Tika
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Component
import java.nio.file.Path
@Component
class ApatcheTikaFileTypeDeterminationService : FileTypeDeterminationService {
override fun fileType(
byteArray: ByteArray,
filename: String,
contentType: String?
): MimeType {
logger.info("START Detect file type name: {}", filename)
val tika = Tika()
val detect = try {
tika.detect(byteArray, filename)
} catch (e: IllegalStateException) {
logger.warn("FAILED Detect file type", e)
"application/octet-stream"
}
val type = detect.substringBefore("/")
val fileType = when (type) {
"image" -> {
FileType.Image
}
"video" -> {
FileType.Video
}
"audio" -> {
FileType.Audio
}
else -> {
FileType.Unknown
}
}
val mimeType = MimeType(type, detect.substringAfter("/"), fileType)
logger.info("SUCCESS Detect file type name: {},MimeType: {}", filename, mimeType)
return mimeType
}
override fun fileType(path: Path, filename: String): MimeType {
logger.info("START Detect file type name: {}", filename)
val tika = Tika()
val detect = try {
tika.detect(path)
} catch (e: IllegalStateException) {
logger.warn("FAILED Detect file type", e)
"application/octet-stream"
}
val type = detect.substringBefore("/")
val fileType = when (type) {
"image" -> {
FileType.Image
}
"video" -> {
FileType.Video
}
"audio" -> {
FileType.Audio
}
else -> {
FileType.Unknown
}
}
val mimeType = MimeType(type, detect.substringAfter("/"), fileType)
logger.info("SUCCESS Detect file type name: {},MimeType: {}", filename, mimeType)
return mimeType
}
companion object {
private val logger = LoggerFactory.getLogger(ApatcheTikaFileTypeDeterminationService::class.java)
}
}

5
src/main/kotlin/dev/usbharu/hideout/core/service/media/FileTypeDeterminationService.kt
View File

@ -1,5 +1,8 @@
package dev.usbharu.hideout.core.service.media
import java.nio.file.Path
interface FileTypeDeterminationService {
fun fileType(byteArray: ByteArray, filename: String, contentType: String?): FileType
fun fileType(byteArray: ByteArray, filename: String, contentType: String?): MimeType
fun fileType(path: Path, filename: String): MimeType
}

26
src/main/kotlin/dev/usbharu/hideout/core/service/media/FileTypeDeterminationServiceImpl.kt
View File

@ -1,26 +0,0 @@
package dev.usbharu.hideout.core.service.media
import org.springframework.stereotype.Component
@Component
class FileTypeDeterminationServiceImpl : FileTypeDeterminationService {
override fun fileType(
byteArray: ByteArray,
filename: String,
contentType: String?
): FileType {
if (contentType == null) {
return FileType.Unknown
}
if (contentType.startsWith("image")) {
return FileType.Image
}
if (contentType.startsWith("video")) {
return FileType.Video
}
if (contentType.startsWith("audio")) {
return FileType.Audio
}
return FileType.Unknown
}
}

1
src/main/kotlin/dev/usbharu/hideout/core/service/media/MediaDataStore.kt
View File

@ -2,5 +2,6 @@ package dev.usbharu.hideout.core.service.media
interface MediaDataStore {
suspend fun save(dataMediaSave: MediaSave): SavedMedia
suspend fun save(dataSaveRequest: MediaSaveRequest): SavedMedia
suspend fun delete(id: String)
}

14
src/main/kotlin/dev/usbharu/hideout/core/service/media/MediaFileRenameService.kt Normal file
View File

@ -0,0 +1,14 @@
package dev.usbharu.hideout.core.service.media
interface MediaFileRenameService {
/**
* メディアをリネームします
*
* @param uploadName アップロードされた時点でのファイル名
* @param uploadMimeType アップロードされた時点でのMimeType
* @param processedName 処理後のファイル名
* @param processedMimeType 処理後のMimeType
* @return リネーム後のファイル名
*/
fun rename(uploadName: String, uploadMimeType: MimeType, processedName: String, processedMimeType: MimeType): String
}

10
src/main/kotlin/dev/usbharu/hideout/core/service/media/MediaSaveRequest.kt Normal file
View File

@ -0,0 +1,10 @@
package dev.usbharu.hideout.core.service.media
import java.nio.file.Path
data class MediaSaveRequest(
val name: String,
val preffix: String,
val filePath: Path,
val thumbnailPath: Path?
)

192
src/main/kotlin/dev/usbharu/hideout/core/service/media/MediaServiceImpl.kt
View File

@ -1,149 +1,136 @@
package dev.usbharu.hideout.core.service.media
import dev.usbharu.hideout.core.domain.exception.media.MediaFileSizeIsZeroException
import dev.usbharu.hideout.core.domain.exception.media.MediaSaveException
import dev.usbharu.hideout.core.domain.exception.media.UnsupportedMediaException
import dev.usbharu.hideout.core.domain.model.media.Media
import dev.usbharu.hideout.core.domain.model.media.MediaRepository
import dev.usbharu.hideout.core.service.media.converter.MediaProcessService
import dev.usbharu.hideout.mastodon.interfaces.api.media.MediaRequest
import io.ktor.client.*
import io.ktor.client.request.*
import io.ktor.client.statement.*
import io.ktor.http.*
import io.ktor.util.*
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import dev.usbharu.hideout.util.withDelete
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
import java.util.*
import java.nio.file.Files
import javax.imageio.ImageIO
import dev.usbharu.hideout.core.domain.model.media.Media as EntityMedia
@Service
@Suppress("TooGenericExceptionCaught")
class MediaServiceImpl(
open class MediaServiceImpl(
private val mediaDataStore: MediaDataStore,
private val fileTypeDeterminationService: FileTypeDeterminationService,
private val mediaBlurhashService: MediaBlurhashService,
private val mediaRepository: MediaRepository,
private val mediaProcessService: MediaProcessService,
private val httpClient: HttpClient
private val mediaProcessServices: List<MediaProcessService>,
private val remoteMediaDownloadService: RemoteMediaDownloadService,
private val renameService: MediaFileRenameService
) : MediaService {
@Suppress("LongMethod")
override suspend fun uploadLocalMedia(mediaRequest: MediaRequest): EntityMedia {
val fileName = mediaRequest.file.name
logger.info(
"Media upload. filename:${mediaRequest.file.name} size:${mediaRequest.file.size} " +
"Media upload. filename:$fileName " +
"contentType:${mediaRequest.file.contentType}"
)
if (mediaRequest.file.size == 0L) {
throw MediaFileSizeIsZeroException("Media file size is zero.")
}
val tempFile = Files.createTempFile("hideout-tmp-file", ".tmp")
val fileType = fileTypeDeterminationService.fileType(
mediaRequest.file.bytes,
mediaRequest.file.name,
mediaRequest.file.contentType
)
if (fileType != FileType.Image) {
throw UnsupportedMediaException("FileType: $fileType is not supported.")
tempFile.withDelete().use {
Files.newOutputStream(tempFile).use { outputStream ->
mediaRequest.file.inputStream.use {
it.transferTo(outputStream)
}
}
val mimeType = fileTypeDeterminationService.fileType(tempFile, fileName)
val process = mediaProcessService.process(
fileType,
mediaRequest.file.contentType.orEmpty(),
mediaRequest.file.name,
mediaRequest.file.bytes,
mediaRequest.thumbnail?.bytes
val process = findMediaProcessor(mimeType).process(
mimeType,
fileName,
tempFile,
null
)
val dataMediaSave = MediaSave(
"${UUID.randomUUID()}.${process.file.extension}",
val dataMediaSave = MediaSaveRequest(
renameService.rename(
mediaRequest.file.name,
mimeType,
process.filePath.fileName.toString(),
process.fileMimeType
),
"",
process.file.byteArray,
process.thumbnail?.byteArray
process.filePath,
process.thumbnailPath
)
dataMediaSave.filePath.withDelete().use {
dataMediaSave.thumbnailPath.withDelete().use {
val save = try {
mediaDataStore.save(dataMediaSave)
} catch (e: Exception) {
logger.warn("Failed save media", e)
throw MediaSaveException("Failed save media.", e)
logger.warn("Failed to save the media", e)
throw MediaSaveException("Failed to save the media.", e)
}
if (save.success.not()) {
save as FaildSavedMedia
logger.warn("Failed save media. reason: ${save.reason}")
logger.warn("Failed to save the media. reason: ${save.reason}")
logger.warn(save.description, save.trace)
throw MediaSaveException("Failed save media.")
throw MediaSaveException("Failed to save the media.")
}
save as SuccessSavedMedia
val blurHash = withContext(Dispatchers.IO) {
mediaBlurhashService.generateBlurhash(ImageIO.read(mediaRequest.file.bytes.inputStream()))
}
val blurHash = generateBlurhash(process)
return mediaRepository.save(
EntityMedia(
id = mediaRepository.generateId(),
name = mediaRequest.file.name,
name = fileName,
url = save.url,
remoteUrl = null,
thumbnailUrl = save.thumbnailUrl,
type = fileType,
blurHash = blurHash
type = process.fileMimeType.fileType,
mimeType = process.fileMimeType,
blurHash = blurHash,
description = mediaRequest.description
)
)
}
}
}
}
// TODO: 仮の処理として保存したように動かす
override suspend fun uploadRemoteMedia(remoteMedia: RemoteMedia): Media {
logger.info("MEDIA Remote media. filename:${remoteMedia.name} url:${remoteMedia.url}")
val httpResponse = httpClient.get(remoteMedia.url)
val bytes = httpResponse.bodyAsChannel().toByteArray()
remoteMediaDownloadService.download(remoteMedia.url).withDelete().use {
val mimeType = fileTypeDeterminationService.fileType(it.path, remoteMedia.name)
val contentType = httpResponse.contentType()?.toString()
val fileType =
fileTypeDeterminationService.fileType(bytes, remoteMedia.name, contentType)
val process = findMediaProcessor(mimeType).process(mimeType, remoteMedia.name, it.path, null)
if (fileType != FileType.Image) {
throw UnsupportedMediaException("FileType: $fileType is not supported.")
}
val processedMedia = mediaProcessService.process(
fileType = fileType,
contentType = contentType.orEmpty(),
fileName = remoteMedia.name,
file = bytes,
thumbnail = null
)
val mediaSave = MediaSave(
"${UUID.randomUUID()}.${processedMedia.file.extension}",
val mediaSaveRequest = MediaSaveRequest(
renameService.rename(
remoteMedia.name,
mimeType,
process.filePath.fileName.toString(),
process.fileMimeType
),
"",
processedMedia.file.byteArray,
processedMedia.thumbnail?.byteArray
process.filePath,
process.thumbnailPath
)
mediaSaveRequest.filePath.withDelete().use {
mediaSaveRequest.filePath.withDelete().use {
val save = try {
mediaDataStore.save(mediaSave)
mediaDataStore.save(mediaSaveRequest)
} catch (e: Exception) {
logger.warn("Failed save media", e)
throw MediaSaveException("Failed save media.", e)
logger.warn("Failed to save the media", e)
throw MediaSaveException("Failed to save the media.", e)
}
if (save.success.not()) {
save as FaildSavedMedia
logger.warn("Failed save media. reason: ${save.reason}")
if (save is FaildSavedMedia) {
logger.warn("Failed to save the media. reason: ${save.reason}")
logger.warn(save.description, save.trace)
throw MediaSaveException("Failed save media.")
throw MediaSaveException("Failed to save the media.")
}
save as SuccessSavedMedia
val blurhash = withContext(Dispatchers.IO) {
mediaBlurhashService.generateBlurhash(ImageIO.read(bytes.inputStream()))
}
val blurhash = generateBlurhash(process)
return mediaRepository.save(
EntityMedia(
id = mediaRepository.generateId(),
@ -151,11 +138,56 @@ class MediaServiceImpl(
url = save.url,
remoteUrl = remoteMedia.url,
thumbnailUrl = save.thumbnailUrl,
type = fileType,
type = process.fileMimeType.fileType,
mimeType = process.fileMimeType,
blurHash = blurhash
)
)
}
}
}
}
protected fun findMediaProcessor(mimeType: MimeType): MediaProcessService {
try {
return mediaProcessServices.first {
try {
it.isSupport(mimeType)
} catch (_: Exception) {
false
}
}
} catch (_: NoSuchElementException) {
throw UnsupportedMediaException("MediaType: $mimeType isn't supported.")
}
}
protected fun generateBlurhash(process: ProcessedMediaPath): String {
val path = if (process.thumbnailPath != null && process.thumbnailMimeType != null) {
process.thumbnailPath
} else {
process.filePath
}
val mimeType = if (process.thumbnailPath != null && process.thumbnailMimeType != null) {
process.thumbnailMimeType
} else {
process.fileMimeType
}
val imageReadersByMIMEType = ImageIO.getImageReadersByMIMEType(mimeType.type + "/" + mimeType.subtype)
for (imageReader in imageReadersByMIMEType) {
try {
val bufferedImage = ImageIO.createImageInputStream(path.toFile()).use {
imageReader.input = it
imageReader.read(0)
}
return mediaBlurhashService.generateBlurhash(bufferedImage)
} catch (e: Exception) {
logger.warn("Failed to read thumbnail", e)
}
}
return ""
}
companion object {
private val logger = LoggerFactory.getLogger(MediaServiceImpl::class.java)

3
src/main/kotlin/dev/usbharu/hideout/core/service/media/MimeType.kt Normal file
View File

@ -0,0 +1,3 @@
package dev.usbharu.hideout.core.service.media
data class MimeType(val type: String, val subtype: String, val fileType: FileType)

10
src/main/kotlin/dev/usbharu/hideout/core/service/media/ProcessedMediaPath.kt Normal file
View File

@ -0,0 +1,10 @@
package dev.usbharu.hideout.core.service.media
import java.nio.file.Path
data class ProcessedMediaPath(
val filePath: Path,
val thumbnailPath: Path?,
val fileMimeType: MimeType,
val thumbnailMimeType: MimeType?
)

3
src/main/kotlin/dev/usbharu/hideout/core/service/media/RemoteMedia.kt
View File

@ -3,5 +3,6 @@ package dev.usbharu.hideout.core.service.media
data class RemoteMedia(
val name: String,
val url: String,
val mediaType: String
val mediaType: String,
val description: String? = null
)

7
src/main/kotlin/dev/usbharu/hideout/core/service/media/RemoteMediaDownloadService.kt Normal file
View File

@ -0,0 +1,7 @@
package dev.usbharu.hideout.core.service.media
import java.nio.file.Path
interface RemoteMediaDownloadService {
suspend fun download(url: String): Path
}

37
src/main/kotlin/dev/usbharu/hideout/core/service/media/RemoteMediaDownloadServiceImpl.kt Normal file
View File

@ -0,0 +1,37 @@
package dev.usbharu.hideout.core.service.media
import io.ktor.client.*
import io.ktor.client.request.*
import io.ktor.client.statement.*
import io.ktor.http.*
import io.ktor.utils.io.jvm.javaio.*
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
import java.nio.file.Files
import java.nio.file.Path
import kotlin.io.path.outputStream
@Service
class RemoteMediaDownloadServiceImpl(private val httpClient: HttpClient) : RemoteMediaDownloadService {
override suspend fun download(url: String): Path {
logger.info("START Download remote file. url: {}", url)
val httpResponse = httpClient.get(url)
httpResponse.contentLength()
val createTempFile = Files.createTempFile("hideout-remote-download", ".tmp")
logger.debug("Save to {} url: {} ", createTempFile, url)
httpResponse.bodyAsChannel().toInputStream().use { inputStream ->
createTempFile.outputStream().use {
inputStream.transferTo(it)
}
}
logger.info("SUCCESS Download remote file. url: {}", url)
return createTempFile
}
companion object {
private val logger = LoggerFactory.getLogger(RemoteMediaDownloadServiceImpl::class.java)
}
}

57
src/main/kotlin/dev/usbharu/hideout/core/service/media/S3MediaDataStore.kt
View File

@ -5,6 +5,7 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.withContext
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
import software.amazon.awssdk.core.sync.RequestBody
import software.amazon.awssdk.services.s3.S3Client
@ -54,6 +55,58 @@ class S3MediaDataStore(private val s3Client: S3Client, private val storageConfig
)
}
override suspend fun save(dataSaveRequest: MediaSaveRequest): SavedMedia {
logger.info("MEDIA upload. {}", dataSaveRequest.name)
val fileUploadRequest = PutObjectRequest.builder()
.bucket(storageConfig.bucket)
.key(dataSaveRequest.name)
.build()
logger.info("MEDIA upload. bucket: {} key: {}", storageConfig.bucket, dataSaveRequest.name)
val thumbnailKey = "thumbnail-${dataSaveRequest.name}"
val thumbnailUploadRequest = PutObjectRequest.builder()
.bucket(storageConfig.bucket)
.key(thumbnailKey)
.build()
logger.info("MEDIA upload. bucket: {} key: {}", storageConfig.bucket, thumbnailKey)
withContext(Dispatchers.IO) {
awaitAll(
async {
if (dataSaveRequest.thumbnailPath != null) {
s3Client.putObject(
thumbnailUploadRequest,
RequestBody.fromFile(dataSaveRequest.thumbnailPath)
)
} else {
null
}
},
async {
s3Client.putObject(fileUploadRequest, RequestBody.fromFile(dataSaveRequest.filePath))
}
)
}
val successSavedMedia = SuccessSavedMedia(
name = dataSaveRequest.name,
url = "${storageConfig.publicUrl}/${storageConfig.bucket}/${dataSaveRequest.name}",
thumbnailUrl = "${storageConfig.publicUrl}/${storageConfig.bucket}/$thumbnailKey"
)
logger.info("SUCCESS Media upload. {}", dataSaveRequest.name)
logger.debug(
"name: {} url: {} thumbnail url: {}",
successSavedMedia.name,
successSavedMedia.url,
successSavedMedia.thumbnailUrl
)
return successSavedMedia
}
override suspend fun delete(id: String) {
val fileDeleteRequest = DeleteObjectRequest.builder().bucket(storageConfig.bucket).key(id).build()
val thumbnailDeleteRequest =
@ -61,4 +114,8 @@ class S3MediaDataStore(private val s3Client: S3Client, private val storageConfig
s3Client.deleteObject(fileDeleteRequest)
s3Client.deleteObject(thumbnailDeleteRequest)
}
companion object {
private val logger = LoggerFactory.getLogger(S3MediaDataStore::class.java)
}
}

16
src/main/kotlin/dev/usbharu/hideout/core/service/media/UUIDMediaFileRenameService.kt Normal file
View File

@ -0,0 +1,16 @@
package dev.usbharu.hideout.core.service.media
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.stereotype.Service
import java.util.*
@Qualifier("uuid")
@Service
class UUIDMediaFileRenameService : MediaFileRenameService {
override fun rename(
uploadName: String,
uploadMimeType: MimeType,
processedName: String,
processedMimeType: MimeType
): String = "${UUID.randomUUID()}.${uploadMimeType.subtype}.${processedMimeType.subtype}"
}

12
src/main/kotlin/dev/usbharu/hideout/core/service/media/converter/MediaProcessService.kt
View File

@ -1,9 +1,14 @@
package dev.usbharu.hideout.core.service.media.converter
import dev.usbharu.hideout.core.service.media.FileType
import dev.usbharu.hideout.core.service.media.MimeType
import dev.usbharu.hideout.core.service.media.ProcessedMedia
import dev.usbharu.hideout.core.service.media.ProcessedMediaPath
import java.nio.file.Path
interface MediaProcessService {
fun isSupport(mimeType: MimeType): Boolean
suspend fun process(
fileType: FileType,
contentType: String,
@ -11,4 +16,11 @@ interface MediaProcessService {
file: ByteArray,
thumbnail: ByteArray?
): ProcessedMedia
suspend fun process(
mimeType: MimeType,
fileName: String,
filePath: Path,
thumbnails: Path?
): ProcessedMediaPath
}

16
src/main/kotlin/dev/usbharu/hideout/core/service/media/converter/MediaProcessServiceImpl.kt
View File

@ -1,11 +1,10 @@
package dev.usbharu.hideout.core.service.media.converter
import dev.usbharu.hideout.core.domain.exception.media.MediaConvertException
import dev.usbharu.hideout.core.service.media.FileType
import dev.usbharu.hideout.core.service.media.ProcessedMedia
import dev.usbharu.hideout.core.service.media.ThumbnailGenerateService
import dev.usbharu.hideout.core.service.media.*
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
import java.nio.file.Path
@Service
@Suppress("TooGenericExceptionCaught")
@ -13,6 +12,8 @@ class MediaProcessServiceImpl(
private val mediaConverterRoot: MediaConverterRoot,
private val thumbnailGenerateService: ThumbnailGenerateService
) : MediaProcessService {
override fun isSupport(mimeType: MimeType): Boolean = false
override suspend fun process(
fileType: FileType,
contentType: String,
@ -42,6 +43,15 @@ class MediaProcessServiceImpl(
)
}
override suspend fun process(
mimeType: MimeType,
fileName: String,
filePath: Path,
thumbnails: Path?
): ProcessedMediaPath {
TODO("Not yet implemented")
}
companion object {
private val logger = LoggerFactory.getLogger(MediaProcessServiceImpl::class.java)
}

104
src/main/kotlin/dev/usbharu/hideout/core/service/media/converter/image/ImageMediaProcessService.kt Normal file
View File

@ -0,0 +1,104 @@
package dev.usbharu.hideout.core.service.media.converter.image
import dev.usbharu.hideout.core.domain.exception.media.MediaProcessException
import dev.usbharu.hideout.core.service.media.FileType
import dev.usbharu.hideout.core.service.media.MimeType
import dev.usbharu.hideout.core.service.media.ProcessedMedia
import dev.usbharu.hideout.core.service.media.ProcessedMediaPath
import dev.usbharu.hideout.core.service.media.converter.MediaProcessService
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.slf4j.MDCContext
import kotlinx.coroutines.withContext
import net.coobird.thumbnailator.Thumbnails
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.stereotype.Service
import java.nio.file.Files
import java.nio.file.Path
import java.util.*
import javax.imageio.ImageIO
import kotlin.io.path.inputStream
import kotlin.io.path.outputStream
@Service
@Qualifier("image")
class ImageMediaProcessService(private val imageMediaProcessorConfiguration: ImageMediaProcessorConfiguration?) :
MediaProcessService {
private val convertType = imageMediaProcessorConfiguration?.convert ?: "jpeg"
private val supportedTypes = imageMediaProcessorConfiguration?.supportedType ?: listOf("webp", "jpeg", "png")
private val genThumbnail = imageMediaProcessorConfiguration?.thubnail?.generate ?: true
private val width = imageMediaProcessorConfiguration?.thubnail?.width ?: 1000
private val height = imageMediaProcessorConfiguration?.thubnail?.height ?: 1000
override fun isSupport(mimeType: MimeType): Boolean {
if (mimeType.type != "image") {
return false
}
return supportedTypes.contains(mimeType.subtype)
}
override suspend fun process(
fileType: FileType,
contentType: String,
fileName: String,
file: ByteArray,
thumbnail: ByteArray?
): ProcessedMedia {
TODO("Not yet implemented")
}
override suspend fun process(
mimeType: MimeType,
fileName: String,
filePath: Path,
thumbnails: Path?
): ProcessedMediaPath = withContext(Dispatchers.IO + MDCContext()) {
val bufferedImage = ImageIO.read(filePath.inputStream())
val tempFileName = UUID.randomUUID().toString()
val tempFile = Files.createTempFile(tempFileName, "tmp")
val thumbnailPath = if (genThumbnail) {
val tempThumbnailFile = Files.createTempFile("thumbnail-$tempFileName", ".tmp")
tempThumbnailFile.outputStream().use {
val write = ImageIO.write(
if (thumbnails != null) {
Thumbnails.of(thumbnails.toFile()).size(width, height).asBufferedImage()
} else {
Thumbnails.of(bufferedImage).size(width, height).asBufferedImage()
},
convertType,
it
)
if (write) {
tempThumbnailFile
} else {
null
}
}
} else {
null
}
tempFile.outputStream().use {
if (ImageIO.write(bufferedImage, convertType, it).not()) {
logger.warn("Failed to save a temporary file. type: {} ,path: {}", convertType, tempFile)
throw MediaProcessException("Failed to save a temporary file.")
}
}
ProcessedMediaPath(
tempFile,
thumbnailPath,
MimeType("image", convertType, FileType.Image),
MimeType("image", convertType, FileType.Image).takeIf { genThumbnail }
)
}
companion object {
private val logger = LoggerFactory.getLogger(ImageMediaProcessService::class.java)
}
}

17
src/main/kotlin/dev/usbharu/hideout/core/service/media/converter/image/ImageMediaProcessorConfiguration.kt Normal file
View File

@ -0,0 +1,17 @@
package dev.usbharu.hideout.core.service.media.converter.image
import org.springframework.boot.context.properties.ConfigurationProperties
@ConfigurationProperties("hideout.media.image")
data class ImageMediaProcessorConfiguration(
val convert: String?,
val thubnail: ImageMediaProcessorThumbnailConfiguration?,
val supportedType: List<String>?,
)
data class ImageMediaProcessorThumbnailConfiguration(
val generate: Boolean,
val width: Int,
val height: Int
)

126
src/main/kotlin/dev/usbharu/hideout/core/service/media/converter/movie/MovieMediaProcessService.kt Normal file
View File

@ -0,0 +1,126 @@
package dev.usbharu.hideout.core.service.media.converter.movie
import dev.usbharu.hideout.core.service.media.FileType
import dev.usbharu.hideout.core.service.media.MimeType
import dev.usbharu.hideout.core.service.media.ProcessedMedia
import dev.usbharu.hideout.core.service.media.ProcessedMediaPath
import dev.usbharu.hideout.core.service.media.converter.MediaProcessService
import org.bytedeco.ffmpeg.global.avcodec
import org.bytedeco.javacv.FFmpegFrameFilter
import org.bytedeco.javacv.FFmpegFrameGrabber
import org.bytedeco.javacv.FFmpegFrameRecorder
import org.bytedeco.javacv.Java2DFrameConverter
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.stereotype.Service
import java.awt.image.BufferedImage
import java.nio.file.Files
import java.nio.file.Path
import javax.imageio.ImageIO
import kotlin.math.min
@Service
@Qualifier("video")
class MovieMediaProcessService : MediaProcessService {
override fun isSupport(mimeType: MimeType): Boolean = mimeType.type == "video"
override suspend fun process(
fileType: FileType,
contentType: String,
fileName: String,
file: ByteArray,
thumbnail: ByteArray?
): ProcessedMedia {
TODO("Not yet implemented")
}
override suspend fun process(
mimeType: MimeType,
fileName: String,
filePath: Path,
thumbnails: Path?
): ProcessedMediaPath {
val tempFile = Files.createTempFile("hideout-movie-processor-", ".tmp")
val thumbnailFile = Files.createTempFile("hideout-movie-thumbnail-generate-", ".tmp")
logger.info("START Convert Movie Media {}", fileName)
FFmpegFrameGrabber(filePath.toFile()).use { grabber ->
grabber.start()
val width = grabber.imageWidth
val height = grabber.imageHeight
val frameRate = 60.0
logger.debug("Movie Media Width {}, Height {}", width, height)
FFmpegFrameFilter(
"fps=fps=${frameRate.toInt()}",
"anull",
width,
height,
grabber.audioChannels
).use { filter ->
filter.sampleFormat = grabber.sampleFormat
filter.sampleRate = grabber.sampleRate
filter.pixelFormat = grabber.pixelFormat
filter.frameRate = grabber.frameRate
filter.start()
val videoBitRate = min(1300000, (width * height * frameRate * 1 * 0.07).toInt())
logger.debug("Movie Media BitRate {}", videoBitRate)
FFmpegFrameRecorder(tempFile.toFile(), width, height, grabber.audioChannels).use {
it.sampleRate = grabber.sampleRate
it.format = "mp4"
it.videoCodec = avcodec.AV_CODEC_ID_H264
it.audioCodec = avcodec.AV_CODEC_ID_AAC
it.audioChannels = grabber.audioChannels
it.videoQuality = 1.0
it.frameRate = frameRate
it.setVideoOption("preset", "ultrafast")
it.timestamp = 0
it.gopSize = frameRate.toInt()
it.videoBitrate = videoBitRate
it.start()
var bufferedImage: BufferedImage? = null
val frameConverter = Java2DFrameConverter()
while (true) {
val grab = grabber.grab() ?: break
if (bufferedImage == null) {
bufferedImage = frameConverter.convert(grab)
}
if (grab.image != null || grab.samples != null) {
filter.push(grab)
}
while (true) {
val frame = filter.pull() ?: break
it.record(frame)
}
}
if (bufferedImage != null) {
ImageIO.write(bufferedImage, "jpeg", thumbnailFile.toFile())
}
}
}
}
logger.info("SUCCESS Convert Movie Media {}", fileName)
return ProcessedMediaPath(
tempFile,
thumbnailFile,
MimeType("video", "mp4", FileType.Video),
MimeType("image", "jpeg", FileType.Image)
)
}
companion object {
private val logger = LoggerFactory.getLogger(MovieMediaProcessService::class.java)
}
}

2
src/main/kotlin/dev/usbharu/hideout/core/service/reaction/ReactionServiceImpl.kt
View File

@ -45,7 +45,7 @@ class ReactionServiceImpl(
reactionQueryService.findByPostIdAndUserIdAndEmojiId(postId, userId, 0)
reactionRepository.delete(findByPostIdAndUserIdAndEmojiId)
apReactionService.removeReaction(findByPostIdAndUserIdAndEmojiId)
} catch (e: FailedToGetResourcesException) {
} catch (_: FailedToGetResourcesException) {
}
}

12
src/main/kotlin/dev/usbharu/hideout/util/TempFileUtil.kt Normal file
View File

@ -0,0 +1,12 @@
package dev.usbharu.hideout.util
import java.nio.file.Files
import java.nio.file.Path
fun <T : Path?> T.withDelete(): TempFile<T> = TempFile(this)
class TempFile<T : Path?>(val path: T) : AutoCloseable {
override fun close() {
path?.let { Files.deleteIfExists(it) }
}
}

8
src/main/resources/application.yml
View File

@ -30,7 +30,10 @@ spring:
database: hideout
# username: hideoutuser
# password: hideoutpass
servlet:
multipart:
max-file-size: 40MB
max-request-size: 40MB
h2:
console:
enabled: true
@ -42,5 +45,6 @@ server:
basedir: tomcat
accesslog:
enabled: true
max-http-form-post-size: 40MB
max-swallow-size: 40MB
port: 8081

6
src/test/kotlin/dev/usbharu/hideout/activitypub/service/objects/note/APNoteServiceImplTest.kt
View File

@ -37,7 +37,6 @@ import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.mockito.kotlin.*
import utils.JsonObjectMapper.objectMapper
import utils.PostBuilder
import utils.UserBuilder
import java.time.Instant
@ -74,7 +73,6 @@ class APNoteServiceImplTest {
postRepository = mock(),
apUserService = mock(),
postQueryService = mock(),
objectMapper = objectMapper,
postService = mock(),
apResourceResolveService = mock(),
postBuilder = Post.PostBuilder(CharacterLimit()),
@ -152,7 +150,6 @@ class APNoteServiceImplTest {
postRepository = postRepository,
apUserService = apUserService,
postQueryService = postQueryService,
objectMapper = objectMapper,
postService = mock(),
apResourceResolveService = apResourceResolveService,
postBuilder = Post.PostBuilder(CharacterLimit()),
@ -221,7 +218,6 @@ class APNoteServiceImplTest {
postRepository = mock(),
apUserService = mock(),
postQueryService = postQueryService,
objectMapper = objectMapper,
postService = mock(),
apResourceResolveService = apResourceResolveService,
postBuilder = Post.PostBuilder(CharacterLimit()),
@ -274,7 +270,6 @@ class APNoteServiceImplTest {
postRepository = postRepository,
apUserService = apUserService,
postQueryService = mock(),
objectMapper = objectMapper,
postService = postService,
apResourceResolveService = mock(),
postBuilder = postBuilder,
@ -333,7 +328,6 @@ class APNoteServiceImplTest {
postRepository = mock(),
apUserService = mock(),
postQueryService = mock(),
objectMapper = objectMapper,
postService = mock(),
apResourceResolveService = mock(),
postBuilder = postBuilder,