Merge pull request #145 from usbharu/feature/refactor-media

メディア関係の改善
This commit is contained in:
usbharu 2023-11-16 21:04:07 +09:00 committed by GitHub
commit 7c105e946c
37 changed files with 756 additions and 189 deletions

View File

@ -179,7 +179,9 @@ dependencies {
implementation("org.postgresql:postgresql:42.6.0")
implementation("com.twelvemonkeys.imageio:imageio-webp:3.10.0")
implementation("org.apache.tika:tika-core:2.9.1")
implementation("net.coobird:thumbnailator:0.4.20")
implementation("org.bytedeco:javacv-platform:1.5.9")
implementation("io.ktor:ktor-client-logging-jvm:$ktor_version")

View File

@ -13,9 +13,9 @@ insert into POSTS (ID, "userId", OVERVIEW, TEXT, "createdAt", VISIBILITY, URL, "
VALUES (1242, 11, null, 'test post', 12345680, 0, 'https://example.com/users/test-user11/posts/1242', null, null, false,
'https://example.com/users/test-user11/posts/1242');
insert into MEDIA (ID, NAME, URL, REMOTE_URL, THUMBNAIL_URL, TYPE, BLURHASH)
VALUES (1, 'test-media', 'https://example.com/media/test-media.png', null, null, 0, null),
(2, 'test-media2', 'https://example.com/media/test-media2.png', null, null, 0, null);
insert into MEDIA (ID, NAME, URL, REMOTE_URL, THUMBNAIL_URL, TYPE, BLURHASH, MIME_TYPE, DESCRIPTION)
VALUES (1, 'test-media', 'https://example.com/media/test-media.png', null, null, 0, null, 'image/png', null),
(2, 'test-media2', 'https://example.com/media/test-media2.png', null, null, 0, null, 'image/png', null);
insert into POSTSMEDIA(POST_ID, MEDIA_ID)
VALUES (1242, 1),

View File

@ -41,7 +41,5 @@ open class Delete : Object {
return result
}
override fun toString(): String {
return "Delete(`object`=$`object`, published=$published) ${super.toString()}"
}
override fun toString(): String = "Delete(`object`=$`object`, published=$published) ${super.toString()}"
}

View File

@ -62,13 +62,9 @@ class ContextDeserializer : JsonDeserializer<String>() {
class ContextSerializer : JsonSerializer<List<String>>() {
override fun isEmpty(value: List<String>?): Boolean {
return value.isNullOrEmpty()
}
override fun isEmpty(value: List<String>?): Boolean = value.isNullOrEmpty()
override fun isEmpty(provider: SerializerProvider?, value: List<String>?): Boolean {
return value.isNullOrEmpty()
}
override fun isEmpty(provider: SerializerProvider?, value: List<String>?): Boolean = value.isNullOrEmpty()
override fun serialize(value: List<String>?, gen: JsonGenerator?, serializers: SerializerProvider) {
if (value.isNullOrEmpty()) {

View File

@ -46,9 +46,7 @@ open class Object : JsonLd {
return result
}
override fun toString(): String {
return "Object(type=$type, name=$name, actor=$actor, id=$id) ${super.toString()}"
}
override fun toString(): String = "Object(type=$type, name=$name, actor=$actor, id=$id) ${super.toString()}"
companion object {
@JvmStatic

View File

@ -12,7 +12,7 @@ class UserAPControllerImpl(private val apUserService: APUserService) : UserAPCon
override suspend fun userAp(username: String): ResponseEntity<Person> {
val person = try {
apUserService.getPersonByName(username)
} catch (e: FailedToGetResourcesException) {
} catch (_: FailedToGetResourcesException) {
return ResponseEntity.notFound().build()
}
person.context += listOf("https://www.w3.org/ns/activitystreams")

View File

@ -22,7 +22,7 @@ class APReceiveDeleteServiceImpl(
val post = try {
postQueryService.findByApId(deleteId)
} catch (e: FailedToGetResourcesException) {
} catch (_: FailedToGetResourcesException) {
return@transaction ActivityPubStringResponse(HttpStatusCode.OK, "Resource not found or already deleted")
}
postRepository.delete(post.id)

View File

@ -1,6 +1,5 @@
package dev.usbharu.hideout.activitypub.service.objects.note
import com.fasterxml.jackson.databind.ObjectMapper
import dev.usbharu.hideout.activitypub.domain.exception.FailedToGetActivityPubResourceException
import dev.usbharu.hideout.activitypub.domain.exception.IllegalActivityPubObjectException
import dev.usbharu.hideout.activitypub.domain.model.Note
@ -24,7 +23,6 @@ import kotlinx.coroutines.async
import kotlinx.coroutines.slf4j.MDCContext
import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.cache.annotation.Cacheable
import org.springframework.stereotype.Service
import java.time.Instant
@ -50,7 +48,6 @@ class APNoteServiceImpl(
private val postRepository: PostRepository,
private val apUserService: APUserService,
private val postQueryService: PostQueryService,
@Qualifier("activitypub") private val objectMapper: ObjectMapper,
private val postService: PostService,
private val apResourceResolveService: APResourceResolveService,
private val postBuilder: Post.PostBuilder,
@ -133,7 +130,8 @@ class APNoteServiceImpl(
RemoteMedia(
(it.name ?: it.url)!!,
it.url!!,
it.mediaType ?: "application/octet-stream"
it.mediaType ?: "application/octet-stream",
description = it.name
)
)
}

View File

@ -13,7 +13,7 @@ class HttpClientConfig {
fun httpClient(): HttpClient = HttpClient(CIO).config {
install(Logging) {
logger = Logger.DEFAULT
level = LogLevel.INFO
level = LogLevel.ALL
}
install(HttpCache) {
}

View File

@ -59,7 +59,7 @@ import java.security.interfaces.RSAPrivateKey
import java.security.interfaces.RSAPublicKey
import java.util.*
@EnableWebSecurity(debug = true)
@EnableWebSecurity(debug = false)
@Configuration
@Suppress("FunctionMaxLength", "TooManyFunctions")
class SecurityConfig {
@ -73,7 +73,12 @@ class SecurityConfig {
@Bean
@Order(1)
fun httpSignatureFilterChain(http: HttpSecurity, httpSignatureFilter: HttpSignatureFilter): SecurityFilterChain {
fun httpSignatureFilterChain(
http: HttpSecurity,
httpSignatureFilter: HttpSignatureFilter,
introspector: HandlerMappingIntrospector
): SecurityFilterChain {
val builder = MvcRequestMatcher.Builder(introspector)
http
.securityMatcher("/inbox", "/outbox", "/users/*/inbox", "/users/*/outbox", "/users/*/posts/*")
.addFilter(httpSignatureFilter)
@ -82,7 +87,12 @@ class SecurityConfig {
HttpSignatureFilter::class.java
)
.authorizeHttpRequests {
it.requestMatchers("/inbox", "/outbox", "/users/*/inbox", "/users/*/outbox").authenticated()
it.requestMatchers(
builder.pattern("/inbox"),
builder.pattern("/outbox"),
builder.pattern("/users/*/inbox"),
builder.pattern("/users/*/outbox")
).authenticated()
it.anyRequest().permitAll()
}
.csrf {

View File

@ -0,0 +1,14 @@
package dev.usbharu.hideout.core.domain.exception.media
class MediaProcessException : MediaException {
constructor() : super()
constructor(message: String?) : super(message)
constructor(message: String?, cause: Throwable?) : super(message, cause)
constructor(cause: Throwable?) : super(cause)
constructor(message: String?, cause: Throwable?, enableSuppression: Boolean, writableStackTrace: Boolean) : super(
message,
cause,
enableSuppression,
writableStackTrace
)
}

View File

@ -1,6 +1,7 @@
package dev.usbharu.hideout.core.domain.model.media
import dev.usbharu.hideout.core.service.media.FileType
import dev.usbharu.hideout.core.service.media.MimeType
data class Media(
val id: Long,
@ -9,5 +10,7 @@ data class Media(
val remoteUrl: String?,
val thumbnailUrl: String?,
val type: FileType,
val blurHash: String?
val mimeType: MimeType,
val blurHash: String?,
val description: String? = null
)

View File

@ -3,7 +3,9 @@ package dev.usbharu.hideout.core.infrastructure.exposedrepository
import dev.usbharu.hideout.application.service.id.IdGenerateService
import dev.usbharu.hideout.core.domain.exception.FailedToGetResourcesException
import dev.usbharu.hideout.core.domain.model.media.MediaRepository
import dev.usbharu.hideout.core.infrastructure.exposedrepository.Media.mimeType
import dev.usbharu.hideout.core.service.media.FileType
import dev.usbharu.hideout.core.service.media.MimeType
import dev.usbharu.hideout.util.singleOr
import org.jetbrains.exposed.sql.*
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
@ -26,6 +28,8 @@ class MediaRepositoryImpl(private val idGenerateService: IdGenerateService) : Me
it[thumbnailUrl] = media.thumbnailUrl
it[type] = media.type.ordinal
it[blurhash] = media.blurHash
it[mimeType] = media.mimeType.type + "/" + media.mimeType.subtype
it[description] = media.description
}
} else {
Media.insert {
@ -36,6 +40,8 @@ class MediaRepositoryImpl(private val idGenerateService: IdGenerateService) : Me
it[thumbnailUrl] = media.thumbnailUrl
it[type] = media.type.ordinal
it[blurhash] = media.blurHash
it[mimeType] = media.mimeType.type + "/" + media.mimeType.subtype
it[description] = media.description
}
}
return media
@ -59,18 +65,24 @@ class MediaRepositoryImpl(private val idGenerateService: IdGenerateService) : Me
}
fun ResultRow.toMedia(): EntityMedia {
val fileType = FileType.values().first { it.ordinal == this[Media.type] }
val mimeType = this[Media.mimeType]
return EntityMedia(
id = this[Media.id],
name = this[Media.name],
url = this[Media.url],
remoteUrl = this[Media.remoteUrl],
thumbnailUrl = this[Media.thumbnailUrl],
type = FileType.values().first { it.ordinal == this[Media.type] },
type = fileType,
blurHash = this[Media.blurhash],
mimeType = MimeType(mimeType.substringBefore("/"), mimeType.substringAfter("/"), fileType),
description = this[Media.description]
)
}
fun ResultRow.toMediaOrNull(): EntityMedia? {
val fileType = FileType.values().first { it.ordinal == (this.getOrNull(Media.type) ?: return null) }
val mimeType = this.getOrNull(Media.mimeType) ?: return null
return EntityMedia(
id = this.getOrNull(Media.id) ?: return null,
name = this.getOrNull(Media.name) ?: return null,
@ -79,6 +91,8 @@ fun ResultRow.toMediaOrNull(): EntityMedia? {
thumbnailUrl = this[Media.thumbnailUrl],
type = FileType.values().first { it.ordinal == this.getOrNull(Media.type) },
blurHash = this[Media.blurhash],
mimeType = MimeType(mimeType.substringBefore("/"), mimeType.substringAfter("/"), fileType),
description = this[Media.description]
)
}
@ -90,5 +104,7 @@ object Media : Table("media") {
val thumbnailUrl = varchar("thumbnail_url", 255).nullable()
val type = integer("type")
val blurhash = varchar("blurhash", 255).nullable()
val mimeType = varchar("mime_type", 255)
val description = varchar("description", 4000).nullable()
override val primaryKey = PrimaryKey(id)
}

View File

@ -73,7 +73,6 @@ class PostRepositoryImpl(
.let(postQueryMapper::map)
.singleOr { FailedToGetResourcesException("id: $id was not found.", it) }
override suspend fun delete(id: Long) {
Posts.deleteWhere { Posts.id eq id }
}

View File

@ -0,0 +1,89 @@
package dev.usbharu.hideout.core.service.media
import org.apache.tika.Tika
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Component
import java.nio.file.Path
@Component
class ApatcheTikaFileTypeDeterminationService : FileTypeDeterminationService {
override fun fileType(
byteArray: ByteArray,
filename: String,
contentType: String?
): MimeType {
logger.info("START Detect file type name: {}", filename)
val tika = Tika()
val detect = try {
tika.detect(byteArray, filename)
} catch (e: IllegalStateException) {
logger.warn("FAILED Detect file type", e)
"application/octet-stream"
}
val type = detect.substringBefore("/")
val fileType = when (type) {
"image" -> {
FileType.Image
}
"video" -> {
FileType.Video
}
"audio" -> {
FileType.Audio
}
else -> {
FileType.Unknown
}
}
val mimeType = MimeType(type, detect.substringAfter("/"), fileType)
logger.info("SUCCESS Detect file type name: {},MimeType: {}", filename, mimeType)
return mimeType
}
override fun fileType(path: Path, filename: String): MimeType {
logger.info("START Detect file type name: {}", filename)
val tika = Tika()
val detect = try {
tika.detect(path)
} catch (e: IllegalStateException) {
logger.warn("FAILED Detect file type", e)
"application/octet-stream"
}
val type = detect.substringBefore("/")
val fileType = when (type) {
"image" -> {
FileType.Image
}
"video" -> {
FileType.Video
}
"audio" -> {
FileType.Audio
}
else -> {
FileType.Unknown
}
}
val mimeType = MimeType(type, detect.substringAfter("/"), fileType)
logger.info("SUCCESS Detect file type name: {},MimeType: {}", filename, mimeType)
return mimeType
}
companion object {
private val logger = LoggerFactory.getLogger(ApatcheTikaFileTypeDeterminationService::class.java)
}
}

View File

@ -1,5 +1,8 @@
package dev.usbharu.hideout.core.service.media
import java.nio.file.Path
interface FileTypeDeterminationService {
fun fileType(byteArray: ByteArray, filename: String, contentType: String?): FileType
fun fileType(byteArray: ByteArray, filename: String, contentType: String?): MimeType
fun fileType(path: Path, filename: String): MimeType
}

View File

@ -1,26 +0,0 @@
package dev.usbharu.hideout.core.service.media
import org.springframework.stereotype.Component
@Component
class FileTypeDeterminationServiceImpl : FileTypeDeterminationService {
override fun fileType(
byteArray: ByteArray,
filename: String,
contentType: String?
): FileType {
if (contentType == null) {
return FileType.Unknown
}
if (contentType.startsWith("image")) {
return FileType.Image
}
if (contentType.startsWith("video")) {
return FileType.Video
}
if (contentType.startsWith("audio")) {
return FileType.Audio
}
return FileType.Unknown
}
}

View File

@ -2,5 +2,6 @@ package dev.usbharu.hideout.core.service.media
interface MediaDataStore {
suspend fun save(dataMediaSave: MediaSave): SavedMedia
suspend fun save(dataSaveRequest: MediaSaveRequest): SavedMedia
suspend fun delete(id: String)
}

View File

@ -0,0 +1,14 @@
package dev.usbharu.hideout.core.service.media
interface MediaFileRenameService {
/**
* メディアをリネームします
*
* @param uploadName アップロードされた時点でのファイル名
* @param uploadMimeType アップロードされた時点でのMimeType
* @param processedName 処理後のファイル名
* @param processedMimeType 処理後のMimeType
* @return リネーム後のファイル名
*/
fun rename(uploadName: String, uploadMimeType: MimeType, processedName: String, processedMimeType: MimeType): String
}

View File

@ -0,0 +1,10 @@
package dev.usbharu.hideout.core.service.media
import java.nio.file.Path
data class MediaSaveRequest(
val name: String,
val preffix: String,
val filePath: Path,
val thumbnailPath: Path?
)

View File

@ -1,160 +1,192 @@
package dev.usbharu.hideout.core.service.media
import dev.usbharu.hideout.core.domain.exception.media.MediaFileSizeIsZeroException
import dev.usbharu.hideout.core.domain.exception.media.MediaSaveException
import dev.usbharu.hideout.core.domain.exception.media.UnsupportedMediaException
import dev.usbharu.hideout.core.domain.model.media.Media
import dev.usbharu.hideout.core.domain.model.media.MediaRepository
import dev.usbharu.hideout.core.service.media.converter.MediaProcessService
import dev.usbharu.hideout.mastodon.interfaces.api.media.MediaRequest
import io.ktor.client.*
import io.ktor.client.request.*
import io.ktor.client.statement.*
import io.ktor.http.*
import io.ktor.util.*
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import dev.usbharu.hideout.util.withDelete
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
import java.util.*
import java.nio.file.Files
import javax.imageio.ImageIO
import dev.usbharu.hideout.core.domain.model.media.Media as EntityMedia
@Service
@Suppress("TooGenericExceptionCaught")
class MediaServiceImpl(
open class MediaServiceImpl(
private val mediaDataStore: MediaDataStore,
private val fileTypeDeterminationService: FileTypeDeterminationService,
private val mediaBlurhashService: MediaBlurhashService,
private val mediaRepository: MediaRepository,
private val mediaProcessService: MediaProcessService,
private val httpClient: HttpClient
private val mediaProcessServices: List<MediaProcessService>,
private val remoteMediaDownloadService: RemoteMediaDownloadService,
private val renameService: MediaFileRenameService
) : MediaService {
@Suppress("LongMethod")
override suspend fun uploadLocalMedia(mediaRequest: MediaRequest): EntityMedia {
val fileName = mediaRequest.file.name
logger.info(
"Media upload. filename:${mediaRequest.file.name} size:${mediaRequest.file.size} " +
"Media upload. filename:$fileName " +
"contentType:${mediaRequest.file.contentType}"
)
if (mediaRequest.file.size == 0L) {
throw MediaFileSizeIsZeroException("Media file size is zero.")
}
val tempFile = Files.createTempFile("hideout-tmp-file", ".tmp")
val fileType = fileTypeDeterminationService.fileType(
mediaRequest.file.bytes,
mediaRequest.file.name,
mediaRequest.file.contentType
)
if (fileType != FileType.Image) {
throw UnsupportedMediaException("FileType: $fileType is not supported.")
}
tempFile.withDelete().use {
Files.newOutputStream(tempFile).use { outputStream ->
mediaRequest.file.inputStream.use {
it.transferTo(outputStream)
}
}
val mimeType = fileTypeDeterminationService.fileType(tempFile, fileName)
val process = mediaProcessService.process(
fileType,
mediaRequest.file.contentType.orEmpty(),
mediaRequest.file.name,
mediaRequest.file.bytes,
mediaRequest.thumbnail?.bytes
)
val dataMediaSave = MediaSave(
"${UUID.randomUUID()}.${process.file.extension}",
"",
process.file.byteArray,
process.thumbnail?.byteArray
)
val save = try {
mediaDataStore.save(dataMediaSave)
} catch (e: Exception) {
logger.warn("Failed save media", e)
throw MediaSaveException("Failed save media.", e)
}
if (save.success.not()) {
save as FaildSavedMedia
logger.warn("Failed save media. reason: ${save.reason}")
logger.warn(save.description, save.trace)
throw MediaSaveException("Failed save media.")
}
save as SuccessSavedMedia
val blurHash = withContext(Dispatchers.IO) {
mediaBlurhashService.generateBlurhash(ImageIO.read(mediaRequest.file.bytes.inputStream()))
}
return mediaRepository.save(
EntityMedia(
id = mediaRepository.generateId(),
name = mediaRequest.file.name,
url = save.url,
remoteUrl = null,
thumbnailUrl = save.thumbnailUrl,
type = fileType,
blurHash = blurHash
val process = findMediaProcessor(mimeType).process(
mimeType,
fileName,
tempFile,
null
)
)
val dataMediaSave = MediaSaveRequest(
renameService.rename(
mediaRequest.file.name,
mimeType,
process.filePath.fileName.toString(),
process.fileMimeType
),
"",
process.filePath,
process.thumbnailPath
)
dataMediaSave.filePath.withDelete().use {
dataMediaSave.thumbnailPath.withDelete().use {
val save = try {
mediaDataStore.save(dataMediaSave)
} catch (e: Exception) {
logger.warn("Failed to save the media", e)
throw MediaSaveException("Failed to save the media.", e)
}
if (save.success.not()) {
save as FaildSavedMedia
logger.warn("Failed to save the media. reason: ${save.reason}")
logger.warn(save.description, save.trace)
throw MediaSaveException("Failed to save the media.")
}
save as SuccessSavedMedia
val blurHash = generateBlurhash(process)
return mediaRepository.save(
EntityMedia(
id = mediaRepository.generateId(),
name = fileName,
url = save.url,
remoteUrl = null,
thumbnailUrl = save.thumbnailUrl,
type = process.fileMimeType.fileType,
mimeType = process.fileMimeType,
blurHash = blurHash,
description = mediaRequest.description
)
)
}
}
}
}
// TODO: 仮の処理として保存したように動かす
override suspend fun uploadRemoteMedia(remoteMedia: RemoteMedia): Media {
logger.info("MEDIA Remote media. filename:${remoteMedia.name} url:${remoteMedia.url}")
val httpResponse = httpClient.get(remoteMedia.url)
val bytes = httpResponse.bodyAsChannel().toByteArray()
remoteMediaDownloadService.download(remoteMedia.url).withDelete().use {
val mimeType = fileTypeDeterminationService.fileType(it.path, remoteMedia.name)
val contentType = httpResponse.contentType()?.toString()
val fileType =
fileTypeDeterminationService.fileType(bytes, remoteMedia.name, contentType)
val process = findMediaProcessor(mimeType).process(mimeType, remoteMedia.name, it.path, null)
if (fileType != FileType.Image) {
throw UnsupportedMediaException("FileType: $fileType is not supported.")
}
val processedMedia = mediaProcessService.process(
fileType = fileType,
contentType = contentType.orEmpty(),
fileName = remoteMedia.name,
file = bytes,
thumbnail = null
)
val mediaSave = MediaSave(
"${UUID.randomUUID()}.${processedMedia.file.extension}",
"",
processedMedia.file.byteArray,
processedMedia.thumbnail?.byteArray
)
val save = try {
mediaDataStore.save(mediaSave)
} catch (e: Exception) {
logger.warn("Failed save media", e)
throw MediaSaveException("Failed save media.", e)
}
if (save.success.not()) {
save as FaildSavedMedia
logger.warn("Failed save media. reason: ${save.reason}")
logger.warn(save.description, save.trace)
throw MediaSaveException("Failed save media.")
}
save as SuccessSavedMedia
val blurhash = withContext(Dispatchers.IO) {
mediaBlurhashService.generateBlurhash(ImageIO.read(bytes.inputStream()))
}
return mediaRepository.save(
EntityMedia(
id = mediaRepository.generateId(),
name = remoteMedia.name,
url = save.url,
remoteUrl = remoteMedia.url,
thumbnailUrl = save.thumbnailUrl,
type = fileType,
blurHash = blurhash
val mediaSaveRequest = MediaSaveRequest(
renameService.rename(
remoteMedia.name,
mimeType,
process.filePath.fileName.toString(),
process.fileMimeType
),
"",
process.filePath,
process.thumbnailPath
)
)
mediaSaveRequest.filePath.withDelete().use {
mediaSaveRequest.filePath.withDelete().use {
val save = try {
mediaDataStore.save(mediaSaveRequest)
} catch (e: Exception) {
logger.warn("Failed to save the media", e)
throw MediaSaveException("Failed to save the media.", e)
}
if (save is FaildSavedMedia) {
logger.warn("Failed to save the media. reason: ${save.reason}")
logger.warn(save.description, save.trace)
throw MediaSaveException("Failed to save the media.")
}
save as SuccessSavedMedia
val blurhash = generateBlurhash(process)
return mediaRepository.save(
EntityMedia(
id = mediaRepository.generateId(),
name = remoteMedia.name,
url = save.url,
remoteUrl = remoteMedia.url,
thumbnailUrl = save.thumbnailUrl,
type = process.fileMimeType.fileType,
mimeType = process.fileMimeType,
blurHash = blurhash
)
)
}
}
}
}
protected fun findMediaProcessor(mimeType: MimeType): MediaProcessService {
try {
return mediaProcessServices.first {
try {
it.isSupport(mimeType)
} catch (_: Exception) {
false
}
}
} catch (_: NoSuchElementException) {
throw UnsupportedMediaException("MediaType: $mimeType isn't supported.")
}
}
protected fun generateBlurhash(process: ProcessedMediaPath): String {
val path = if (process.thumbnailPath != null && process.thumbnailMimeType != null) {
process.thumbnailPath
} else {
process.filePath
}
val mimeType = if (process.thumbnailPath != null && process.thumbnailMimeType != null) {
process.thumbnailMimeType
} else {
process.fileMimeType
}
val imageReadersByMIMEType = ImageIO.getImageReadersByMIMEType(mimeType.type + "/" + mimeType.subtype)
for (imageReader in imageReadersByMIMEType) {
try {
val bufferedImage = ImageIO.createImageInputStream(path.toFile()).use {
imageReader.input = it
imageReader.read(0)
}
return mediaBlurhashService.generateBlurhash(bufferedImage)
} catch (e: Exception) {
logger.warn("Failed to read thumbnail", e)
}
}
return ""
}
companion object {

View File

@ -0,0 +1,3 @@
package dev.usbharu.hideout.core.service.media
data class MimeType(val type: String, val subtype: String, val fileType: FileType)

View File

@ -0,0 +1,10 @@
package dev.usbharu.hideout.core.service.media
import java.nio.file.Path
data class ProcessedMediaPath(
val filePath: Path,
val thumbnailPath: Path?,
val fileMimeType: MimeType,
val thumbnailMimeType: MimeType?
)

View File

@ -3,5 +3,6 @@ package dev.usbharu.hideout.core.service.media
data class RemoteMedia(
val name: String,
val url: String,
val mediaType: String
val mediaType: String,
val description: String? = null
)

View File

@ -0,0 +1,7 @@
package dev.usbharu.hideout.core.service.media
import java.nio.file.Path
interface RemoteMediaDownloadService {
suspend fun download(url: String): Path
}

View File

@ -0,0 +1,37 @@
package dev.usbharu.hideout.core.service.media
import io.ktor.client.*
import io.ktor.client.request.*
import io.ktor.client.statement.*
import io.ktor.http.*
import io.ktor.utils.io.jvm.javaio.*
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
import java.nio.file.Files
import java.nio.file.Path
import kotlin.io.path.outputStream
@Service
class RemoteMediaDownloadServiceImpl(private val httpClient: HttpClient) : RemoteMediaDownloadService {
override suspend fun download(url: String): Path {
logger.info("START Download remote file. url: {}", url)
val httpResponse = httpClient.get(url)
httpResponse.contentLength()
val createTempFile = Files.createTempFile("hideout-remote-download", ".tmp")
logger.debug("Save to {} url: {} ", createTempFile, url)
httpResponse.bodyAsChannel().toInputStream().use { inputStream ->
createTempFile.outputStream().use {
inputStream.transferTo(it)
}
}
logger.info("SUCCESS Download remote file. url: {}", url)
return createTempFile
}
companion object {
private val logger = LoggerFactory.getLogger(RemoteMediaDownloadServiceImpl::class.java)
}
}

View File

@ -5,6 +5,7 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.withContext
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
import software.amazon.awssdk.core.sync.RequestBody
import software.amazon.awssdk.services.s3.S3Client
@ -54,6 +55,58 @@ class S3MediaDataStore(private val s3Client: S3Client, private val storageConfig
)
}
override suspend fun save(dataSaveRequest: MediaSaveRequest): SavedMedia {
logger.info("MEDIA upload. {}", dataSaveRequest.name)
val fileUploadRequest = PutObjectRequest.builder()
.bucket(storageConfig.bucket)
.key(dataSaveRequest.name)
.build()
logger.info("MEDIA upload. bucket: {} key: {}", storageConfig.bucket, dataSaveRequest.name)
val thumbnailKey = "thumbnail-${dataSaveRequest.name}"
val thumbnailUploadRequest = PutObjectRequest.builder()
.bucket(storageConfig.bucket)
.key(thumbnailKey)
.build()
logger.info("MEDIA upload. bucket: {} key: {}", storageConfig.bucket, thumbnailKey)
withContext(Dispatchers.IO) {
awaitAll(
async {
if (dataSaveRequest.thumbnailPath != null) {
s3Client.putObject(
thumbnailUploadRequest,
RequestBody.fromFile(dataSaveRequest.thumbnailPath)
)
} else {
null
}
},
async {
s3Client.putObject(fileUploadRequest, RequestBody.fromFile(dataSaveRequest.filePath))
}
)
}
val successSavedMedia = SuccessSavedMedia(
name = dataSaveRequest.name,
url = "${storageConfig.publicUrl}/${storageConfig.bucket}/${dataSaveRequest.name}",
thumbnailUrl = "${storageConfig.publicUrl}/${storageConfig.bucket}/$thumbnailKey"
)
logger.info("SUCCESS Media upload. {}", dataSaveRequest.name)
logger.debug(
"name: {} url: {} thumbnail url: {}",
successSavedMedia.name,
successSavedMedia.url,
successSavedMedia.thumbnailUrl
)
return successSavedMedia
}
override suspend fun delete(id: String) {
val fileDeleteRequest = DeleteObjectRequest.builder().bucket(storageConfig.bucket).key(id).build()
val thumbnailDeleteRequest =
@ -61,4 +114,8 @@ class S3MediaDataStore(private val s3Client: S3Client, private val storageConfig
s3Client.deleteObject(fileDeleteRequest)
s3Client.deleteObject(thumbnailDeleteRequest)
}
companion object {
private val logger = LoggerFactory.getLogger(S3MediaDataStore::class.java)
}
}

View File

@ -0,0 +1,16 @@
package dev.usbharu.hideout.core.service.media
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.stereotype.Service
import java.util.*
@Qualifier("uuid")
@Service
class UUIDMediaFileRenameService : MediaFileRenameService {
override fun rename(
uploadName: String,
uploadMimeType: MimeType,
processedName: String,
processedMimeType: MimeType
): String = "${UUID.randomUUID()}.${uploadMimeType.subtype}.${processedMimeType.subtype}"
}

View File

@ -1,9 +1,14 @@
package dev.usbharu.hideout.core.service.media.converter
import dev.usbharu.hideout.core.service.media.FileType
import dev.usbharu.hideout.core.service.media.MimeType
import dev.usbharu.hideout.core.service.media.ProcessedMedia
import dev.usbharu.hideout.core.service.media.ProcessedMediaPath
import java.nio.file.Path
interface MediaProcessService {
fun isSupport(mimeType: MimeType): Boolean
suspend fun process(
fileType: FileType,
contentType: String,
@ -11,4 +16,11 @@ interface MediaProcessService {
file: ByteArray,
thumbnail: ByteArray?
): ProcessedMedia
suspend fun process(
mimeType: MimeType,
fileName: String,
filePath: Path,
thumbnails: Path?
): ProcessedMediaPath
}

View File

@ -1,11 +1,10 @@
package dev.usbharu.hideout.core.service.media.converter
import dev.usbharu.hideout.core.domain.exception.media.MediaConvertException
import dev.usbharu.hideout.core.service.media.FileType
import dev.usbharu.hideout.core.service.media.ProcessedMedia
import dev.usbharu.hideout.core.service.media.ThumbnailGenerateService
import dev.usbharu.hideout.core.service.media.*
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
import java.nio.file.Path
@Service
@Suppress("TooGenericExceptionCaught")
@ -13,6 +12,8 @@ class MediaProcessServiceImpl(
private val mediaConverterRoot: MediaConverterRoot,
private val thumbnailGenerateService: ThumbnailGenerateService
) : MediaProcessService {
override fun isSupport(mimeType: MimeType): Boolean = false
override suspend fun process(
fileType: FileType,
contentType: String,
@ -42,6 +43,15 @@ class MediaProcessServiceImpl(
)
}
override suspend fun process(
mimeType: MimeType,
fileName: String,
filePath: Path,
thumbnails: Path?
): ProcessedMediaPath {
TODO("Not yet implemented")
}
companion object {
private val logger = LoggerFactory.getLogger(MediaProcessServiceImpl::class.java)
}

View File

@ -0,0 +1,104 @@
package dev.usbharu.hideout.core.service.media.converter.image
import dev.usbharu.hideout.core.domain.exception.media.MediaProcessException
import dev.usbharu.hideout.core.service.media.FileType
import dev.usbharu.hideout.core.service.media.MimeType
import dev.usbharu.hideout.core.service.media.ProcessedMedia
import dev.usbharu.hideout.core.service.media.ProcessedMediaPath
import dev.usbharu.hideout.core.service.media.converter.MediaProcessService
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.slf4j.MDCContext
import kotlinx.coroutines.withContext
import net.coobird.thumbnailator.Thumbnails
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.stereotype.Service
import java.nio.file.Files
import java.nio.file.Path
import java.util.*
import javax.imageio.ImageIO
import kotlin.io.path.inputStream
import kotlin.io.path.outputStream
@Service
@Qualifier("image")
class ImageMediaProcessService(private val imageMediaProcessorConfiguration: ImageMediaProcessorConfiguration?) :
MediaProcessService {
private val convertType = imageMediaProcessorConfiguration?.convert ?: "jpeg"
private val supportedTypes = imageMediaProcessorConfiguration?.supportedType ?: listOf("webp", "jpeg", "png")
private val genThumbnail = imageMediaProcessorConfiguration?.thubnail?.generate ?: true
private val width = imageMediaProcessorConfiguration?.thubnail?.width ?: 1000
private val height = imageMediaProcessorConfiguration?.thubnail?.height ?: 1000
override fun isSupport(mimeType: MimeType): Boolean {
if (mimeType.type != "image") {
return false
}
return supportedTypes.contains(mimeType.subtype)
}
override suspend fun process(
fileType: FileType,
contentType: String,
fileName: String,
file: ByteArray,
thumbnail: ByteArray?
): ProcessedMedia {
TODO("Not yet implemented")
}
override suspend fun process(
mimeType: MimeType,
fileName: String,
filePath: Path,
thumbnails: Path?
): ProcessedMediaPath = withContext(Dispatchers.IO + MDCContext()) {
val bufferedImage = ImageIO.read(filePath.inputStream())
val tempFileName = UUID.randomUUID().toString()
val tempFile = Files.createTempFile(tempFileName, "tmp")
val thumbnailPath = if (genThumbnail) {
val tempThumbnailFile = Files.createTempFile("thumbnail-$tempFileName", ".tmp")
tempThumbnailFile.outputStream().use {
val write = ImageIO.write(
if (thumbnails != null) {
Thumbnails.of(thumbnails.toFile()).size(width, height).asBufferedImage()
} else {
Thumbnails.of(bufferedImage).size(width, height).asBufferedImage()
},
convertType,
it
)
if (write) {
tempThumbnailFile
} else {
null
}
}
} else {
null
}
tempFile.outputStream().use {
if (ImageIO.write(bufferedImage, convertType, it).not()) {
logger.warn("Failed to save a temporary file. type: {} ,path: {}", convertType, tempFile)
throw MediaProcessException("Failed to save a temporary file.")
}
}
ProcessedMediaPath(
tempFile,
thumbnailPath,
MimeType("image", convertType, FileType.Image),
MimeType("image", convertType, FileType.Image).takeIf { genThumbnail }
)
}
companion object {
private val logger = LoggerFactory.getLogger(ImageMediaProcessService::class.java)
}
}

View File

@ -0,0 +1,17 @@
package dev.usbharu.hideout.core.service.media.converter.image
import org.springframework.boot.context.properties.ConfigurationProperties
@ConfigurationProperties("hideout.media.image")
data class ImageMediaProcessorConfiguration(
val convert: String?,
val thubnail: ImageMediaProcessorThumbnailConfiguration?,
val supportedType: List<String>?,
)
data class ImageMediaProcessorThumbnailConfiguration(
val generate: Boolean,
val width: Int,
val height: Int
)

View File

@ -0,0 +1,126 @@
package dev.usbharu.hideout.core.service.media.converter.movie
import dev.usbharu.hideout.core.service.media.FileType
import dev.usbharu.hideout.core.service.media.MimeType
import dev.usbharu.hideout.core.service.media.ProcessedMedia
import dev.usbharu.hideout.core.service.media.ProcessedMediaPath
import dev.usbharu.hideout.core.service.media.converter.MediaProcessService
import org.bytedeco.ffmpeg.global.avcodec
import org.bytedeco.javacv.FFmpegFrameFilter
import org.bytedeco.javacv.FFmpegFrameGrabber
import org.bytedeco.javacv.FFmpegFrameRecorder
import org.bytedeco.javacv.Java2DFrameConverter
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.stereotype.Service
import java.awt.image.BufferedImage
import java.nio.file.Files
import java.nio.file.Path
import javax.imageio.ImageIO
import kotlin.math.min
@Service
@Qualifier("video")
class MovieMediaProcessService : MediaProcessService {
override fun isSupport(mimeType: MimeType): Boolean = mimeType.type == "video"
override suspend fun process(
fileType: FileType,
contentType: String,
fileName: String,
file: ByteArray,
thumbnail: ByteArray?
): ProcessedMedia {
TODO("Not yet implemented")
}
override suspend fun process(
mimeType: MimeType,
fileName: String,
filePath: Path,
thumbnails: Path?
): ProcessedMediaPath {
val tempFile = Files.createTempFile("hideout-movie-processor-", ".tmp")
val thumbnailFile = Files.createTempFile("hideout-movie-thumbnail-generate-", ".tmp")
logger.info("START Convert Movie Media {}", fileName)
FFmpegFrameGrabber(filePath.toFile()).use { grabber ->
grabber.start()
val width = grabber.imageWidth
val height = grabber.imageHeight
val frameRate = 60.0
logger.debug("Movie Media Width {}, Height {}", width, height)
FFmpegFrameFilter(
"fps=fps=${frameRate.toInt()}",
"anull",
width,
height,
grabber.audioChannels
).use { filter ->
filter.sampleFormat = grabber.sampleFormat
filter.sampleRate = grabber.sampleRate
filter.pixelFormat = grabber.pixelFormat
filter.frameRate = grabber.frameRate
filter.start()
val videoBitRate = min(1300000, (width * height * frameRate * 1 * 0.07).toInt())
logger.debug("Movie Media BitRate {}", videoBitRate)
FFmpegFrameRecorder(tempFile.toFile(), width, height, grabber.audioChannels).use {
it.sampleRate = grabber.sampleRate
it.format = "mp4"
it.videoCodec = avcodec.AV_CODEC_ID_H264
it.audioCodec = avcodec.AV_CODEC_ID_AAC
it.audioChannels = grabber.audioChannels
it.videoQuality = 1.0
it.frameRate = frameRate
it.setVideoOption("preset", "ultrafast")
it.timestamp = 0
it.gopSize = frameRate.toInt()
it.videoBitrate = videoBitRate
it.start()
var bufferedImage: BufferedImage? = null
val frameConverter = Java2DFrameConverter()
while (true) {
val grab = grabber.grab() ?: break
if (bufferedImage == null) {
bufferedImage = frameConverter.convert(grab)
}
if (grab.image != null || grab.samples != null) {
filter.push(grab)
}
while (true) {
val frame = filter.pull() ?: break
it.record(frame)
}
}
if (bufferedImage != null) {
ImageIO.write(bufferedImage, "jpeg", thumbnailFile.toFile())
}
}
}
}
logger.info("SUCCESS Convert Movie Media {}", fileName)
return ProcessedMediaPath(
tempFile,
thumbnailFile,
MimeType("video", "mp4", FileType.Video),
MimeType("image", "jpeg", FileType.Image)
)
}
companion object {
private val logger = LoggerFactory.getLogger(MovieMediaProcessService::class.java)
}
}

View File

@ -45,7 +45,7 @@ class ReactionServiceImpl(
reactionQueryService.findByPostIdAndUserIdAndEmojiId(postId, userId, 0)
reactionRepository.delete(findByPostIdAndUserIdAndEmojiId)
apReactionService.removeReaction(findByPostIdAndUserIdAndEmojiId)
} catch (e: FailedToGetResourcesException) {
} catch (_: FailedToGetResourcesException) {
}
}

View File

@ -0,0 +1,12 @@
package dev.usbharu.hideout.util
import java.nio.file.Files
import java.nio.file.Path
fun <T : Path?> T.withDelete(): TempFile<T> = TempFile(this)
class TempFile<T : Path?>(val path: T) : AutoCloseable {
override fun close() {
path?.let { Files.deleteIfExists(it) }
}
}

View File

@ -30,7 +30,10 @@ spring:
database: hideout
# username: hideoutuser
# password: hideoutpass
servlet:
multipart:
max-file-size: 40MB
max-request-size: 40MB
h2:
console:
enabled: true
@ -42,5 +45,6 @@ server:
basedir: tomcat
accesslog:
enabled: true
max-http-form-post-size: 40MB
max-swallow-size: 40MB
port: 8081

View File

@ -37,7 +37,6 @@ import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.mockito.kotlin.*
import utils.JsonObjectMapper.objectMapper
import utils.PostBuilder
import utils.UserBuilder
import java.time.Instant
@ -74,7 +73,6 @@ class APNoteServiceImplTest {
postRepository = mock(),
apUserService = mock(),
postQueryService = mock(),
objectMapper = objectMapper,
postService = mock(),
apResourceResolveService = mock(),
postBuilder = Post.PostBuilder(CharacterLimit()),
@ -152,7 +150,6 @@ class APNoteServiceImplTest {
postRepository = postRepository,
apUserService = apUserService,
postQueryService = postQueryService,
objectMapper = objectMapper,
postService = mock(),
apResourceResolveService = apResourceResolveService,
postBuilder = Post.PostBuilder(CharacterLimit()),
@ -221,7 +218,6 @@ class APNoteServiceImplTest {
postRepository = mock(),
apUserService = mock(),
postQueryService = postQueryService,
objectMapper = objectMapper,
postService = mock(),
apResourceResolveService = apResourceResolveService,
postBuilder = Post.PostBuilder(CharacterLimit()),
@ -274,7 +270,6 @@ class APNoteServiceImplTest {
postRepository = postRepository,
apUserService = apUserService,
postQueryService = mock(),
objectMapper = objectMapper,
postService = postService,
apResourceResolveService = mock(),
postBuilder = postBuilder,
@ -333,7 +328,6 @@ class APNoteServiceImplTest {
postRepository = mock(),
apUserService = mock(),
postQueryService = mock(),
objectMapper = objectMapper,
postService = mock(),
apResourceResolveService = mock(),
postBuilder = postBuilder,