From 6268850a5b64359d6d92f0014fbfe3277c887500 Mon Sep 17 00:00:00 2001 From: usbharu <64310155+usbharu@users.noreply.github.com> Date: Thu, 16 Nov 2023 23:33:10 +0900 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20=E3=82=BF=E3=82=A4=E3=83=A0?= =?UTF-8?q?=E3=83=A9=E3=82=A4=E3=83=B3=E3=81=AEJDBC=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../exposed/ExposedTransaction.kt | 3 +- .../ExposedTimelineRepository.kt | 120 ++++++++++++++++++ .../exposedrepository/PostRepositoryImpl.kt | 5 - .../kjobexposed/KJobJobQueueParentService.kt | 10 +- .../kjobexposed/KJobJobQueueWorkerService.kt | 6 +- .../MongoTimelineRepositoryWrapper.kt | 2 +- .../core/service/post/PostServiceImpl.kt | 6 +- .../ExposedGenerateTimelineService.kt | 53 ++++++++ .../core/service/timeline/TimelineService.kt | 6 + src/main/resources/application.yml | 16 +-- 10 files changed, 203 insertions(+), 24 deletions(-) create mode 100644 src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/ExposedTimelineRepository.kt create mode 100644 src/main/kotlin/dev/usbharu/hideout/core/service/timeline/ExposedGenerateTimelineService.kt diff --git a/src/main/kotlin/dev/usbharu/hideout/application/infrastructure/exposed/ExposedTransaction.kt b/src/main/kotlin/dev/usbharu/hideout/application/infrastructure/exposed/ExposedTransaction.kt index 3438d428..4dc8316b 100644 --- a/src/main/kotlin/dev/usbharu/hideout/application/infrastructure/exposed/ExposedTransaction.kt +++ b/src/main/kotlin/dev/usbharu/hideout/application/infrastructure/exposed/ExposedTransaction.kt @@ -4,11 +4,12 @@ import dev.usbharu.hideout.application.external.Transaction import kotlinx.coroutines.slf4j.MDCContext import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction import org.springframework.stereotype.Service +import java.sql.Connection @Service class ExposedTransaction : Transaction { override suspend fun transaction(block: suspend () -> T): T { - return newSuspendedTransaction(MDCContext()) { + return newSuspendedTransaction(MDCContext(), transactionIsolation = Connection.TRANSACTION_SERIALIZABLE) { block() } } diff --git a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/ExposedTimelineRepository.kt b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/ExposedTimelineRepository.kt new file mode 100644 index 00000000..21865a35 --- /dev/null +++ b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/ExposedTimelineRepository.kt @@ -0,0 +1,120 @@ +package dev.usbharu.hideout.core.infrastructure.exposedrepository + +import dev.usbharu.hideout.application.service.id.IdGenerateService +import dev.usbharu.hideout.core.domain.model.post.Visibility +import dev.usbharu.hideout.core.domain.model.timeline.Timeline +import dev.usbharu.hideout.core.domain.model.timeline.TimelineRepository +import org.jetbrains.exposed.sql.* +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.stereotype.Repository + +@Repository +@Qualifier("jdbc") +@ConditionalOnProperty("hideout.use-mongodb", havingValue = "false", matchIfMissing = true) +class ExposedTimelineRepository(private val idGenerateService: IdGenerateService) : TimelineRepository { + override suspend fun generateId(): Long = idGenerateService.generateId() + + override suspend fun save(timeline: Timeline): Timeline { + if (Timelines.select { Timelines.id eq timeline.id }.singleOrNull() == null) { + Timelines.insert { + it[id] = timeline.id + it[userId] = timeline.userId + it[timelineId] = timeline.timelineId + it[postId] = timeline.postId + it[postUserId] = timeline.postUserId + it[createdAt] = timeline.createdAt + it[replyId] = timeline.replyId + it[repostId] = timeline.repostId + it[visibility] = timeline.visibility.ordinal + it[sensitive] = timeline.sensitive + it[isLocal] = timeline.isLocal + it[isPureRepost] = timeline.isPureRepost + it[mediaIds] = timeline.mediaIds.joinToString(",") + } + } else { + Timelines.update({ Timelines.id eq timeline.id }) { + it[userId] = timeline.userId + it[timelineId] = timeline.timelineId + it[postId] = timeline.postId + it[postUserId] = timeline.postUserId + it[createdAt] = timeline.createdAt + it[replyId] = timeline.replyId + it[repostId] = timeline.repostId + it[visibility] = timeline.visibility.ordinal + it[sensitive] = timeline.sensitive + it[isLocal] = timeline.isLocal + it[isPureRepost] = timeline.isPureRepost + it[mediaIds] = timeline.mediaIds.joinToString(",") + + } + } + return timeline + } + + override suspend fun saveAll(timelines: List): List { + Timelines.batchInsert(timelines, true, false) { + this[Timelines.id] = it.id + this[Timelines.userId] = it.userId + this[Timelines.timelineId] = it.timelineId + this[Timelines.postId] = it.postId + this[Timelines.postUserId] = it.postUserId + this[Timelines.createdAt] = it.createdAt + this[Timelines.replyId] = it.replyId + this[Timelines.repostId] = it.repostId + this[Timelines.visibility] = it.visibility.ordinal + this[Timelines.sensitive] = it.sensitive + this[Timelines.isLocal] = it.isLocal + this[Timelines.isPureRepost] = it.isPureRepost + this[Timelines.mediaIds] = it.mediaIds.joinToString(",") + } + return timelines + } + + override suspend fun findByUserId(id: Long): List = + Timelines.select { Timelines.userId eq id }.map { it.toTimeline() } + + override suspend fun findByUserIdAndTimelineId(userId: Long, timelineId: Long): List = + Timelines.select { Timelines.userId eq userId and (Timelines.timelineId eq timelineId) } + .map { it.toTimeline() } +} + +fun ResultRow.toTimeline(): Timeline { + return Timeline( + id = this[Timelines.id], + userId = this[Timelines.userId], + timelineId = this[Timelines.timelineId], + postId = this[Timelines.postId], + postUserId = this[Timelines.postUserId], + createdAt = this[Timelines.createdAt], + replyId = this[Timelines.replyId], + repostId = this[Timelines.repostId], + visibility = Visibility.values().first { it.ordinal == this[Timelines.visibility] }, + sensitive = this[Timelines.sensitive], + isLocal = this[Timelines.isLocal], + isPureRepost = this[Timelines.isPureRepost], + mediaIds = this[Timelines.mediaIds].split(",").mapNotNull { it.toLongOrNull() } + ) +} + +object Timelines : Table("timelines") { + val id = long("id") + val userId = long("user_id") + val timelineId = long("timeline_id") + val postId = long("post_id") + val postUserId = long("post_user_id") + val createdAt = long("created_at") + val replyId = long("reply_id").nullable() + val repostId = long("repost_id").nullable() + val visibility = integer("visibility") + val sensitive = bool("sensitive") + val isLocal = bool("is_local") + val isPureRepost = bool("is_pure_repost") + val mediaIds = varchar("media_ids", 255) + + override val primaryKey: PrimaryKey = PrimaryKey(id) + + init { + uniqueIndex(userId, timelineId, postId) + } +} diff --git a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/PostRepositoryImpl.kt b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/PostRepositoryImpl.kt index 16322ce0..a7f8c065 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/PostRepositoryImpl.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/PostRepositoryImpl.kt @@ -59,11 +59,6 @@ class PostRepositoryImpl( it[apId] = post.apId } } - - assert(Posts.select { Posts.id eq post.id }.singleOrNull() != null) { - "Faild to insert" - } - return singleOrNull == null } diff --git a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/kjobexposed/KJobJobQueueParentService.kt b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/kjobexposed/KJobJobQueueParentService.kt index c8f711ab..cc68d15c 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/kjobexposed/KJobJobQueueParentService.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/kjobexposed/KJobJobQueueParentService.kt @@ -5,21 +5,23 @@ import kjob.core.Job import kjob.core.KJob import kjob.core.dsl.ScheduleContext import kjob.core.kjob -import org.jetbrains.exposed.sql.Database +import org.jetbrains.exposed.sql.transactions.TransactionManager import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.stereotype.Service @Service @ConditionalOnProperty(name = ["hideout.use-mongodb"], havingValue = "false", matchIfMissing = true) -class KJobJobQueueParentService(private val database: Database) : JobQueueParentService { +class KJobJobQueueParentService() : JobQueueParentService { private val logger = LoggerFactory.getLogger(this::class.java) - val kjob: KJob = kjob(ExposedKJob) { - connectionDatabase = database + val kjob: KJob by lazy { + kjob(ExposedKJob) { + connectionDatabase = TransactionManager.defaultDatabase isWorker = false }.start() + } override fun init(jobDefines: List) = Unit diff --git a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/kjobexposed/KJobJobQueueWorkerService.kt b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/kjobexposed/KJobJobQueueWorkerService.kt index d7c4fca2..98c3a488 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/kjobexposed/KJobJobQueueWorkerService.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/kjobexposed/KJobJobQueueWorkerService.kt @@ -4,7 +4,7 @@ import dev.usbharu.hideout.core.service.job.JobQueueWorkerService import kjob.core.dsl.JobRegisterContext import kjob.core.dsl.KJobFunctions import kjob.core.kjob -import org.jetbrains.exposed.sql.Database +import org.jetbrains.exposed.sql.transactions.TransactionManager import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.stereotype.Service import dev.usbharu.hideout.core.external.job.HideoutJob as HJ @@ -12,11 +12,11 @@ import kjob.core.dsl.JobContextWithProps as JCWP @Service @ConditionalOnProperty(name = ["hideout.use-mongodb"], havingValue = "false", matchIfMissing = true) -class KJobJobQueueWorkerService(private val database: Database) : JobQueueWorkerService { +class KJobJobQueueWorkerService() : JobQueueWorkerService { val kjob by lazy { kjob(ExposedKJob) { - connectionDatabase = database + connectionDatabase = TransactionManager.defaultDatabase nonBlockingMaxJobs = 10 blockingMaxJobs = 10 jobExecutionPeriodInSeconds = 1 diff --git a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/mongorepository/MongoTimelineRepositoryWrapper.kt b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/mongorepository/MongoTimelineRepositoryWrapper.kt index edf01212..1fdbea8e 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/mongorepository/MongoTimelineRepositoryWrapper.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/mongorepository/MongoTimelineRepositoryWrapper.kt @@ -10,7 +10,7 @@ import org.springframework.stereotype.Repository @Repository @Suppress("InjectDispatcher") -@ConditionalOnProperty("hideout.use-mongodb", havingValue = "", matchIfMissing = false) +@ConditionalOnProperty("hideout.use-mongodb", havingValue = "true", matchIfMissing = false) class MongoTimelineRepositoryWrapper( private val mongoTimelineRepository: MongoTimelineRepository, private val idGenerateService: IdGenerateService diff --git a/src/main/kotlin/dev/usbharu/hideout/core/service/post/PostServiceImpl.kt b/src/main/kotlin/dev/usbharu/hideout/core/service/post/PostServiceImpl.kt index d6bb13e0..027b3274 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/service/post/PostServiceImpl.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/service/post/PostServiceImpl.kt @@ -43,11 +43,13 @@ class PostServiceImpl( if (postRepository.save(post)) { try { timelineService.publishTimeline(post, isLocal) - } catch (_: DuplicateKeyException) { + } catch (e: DuplicateKeyException) { + logger.trace("Timeline already exists.", e) } } post - } catch (_: ExposedSQLException) { + } catch (e: ExposedSQLException) { + logger.warn("FAILED Save to post. url: ${post.apId}", e) postQueryService.findByApId(post.apId) } } diff --git a/src/main/kotlin/dev/usbharu/hideout/core/service/timeline/ExposedGenerateTimelineService.kt b/src/main/kotlin/dev/usbharu/hideout/core/service/timeline/ExposedGenerateTimelineService.kt new file mode 100644 index 00000000..5989c3e4 --- /dev/null +++ b/src/main/kotlin/dev/usbharu/hideout/core/service/timeline/ExposedGenerateTimelineService.kt @@ -0,0 +1,53 @@ +package dev.usbharu.hideout.core.service.timeline + +import dev.usbharu.hideout.core.infrastructure.exposedrepository.Timelines +import dev.usbharu.hideout.domain.mastodon.model.generated.Status +import dev.usbharu.hideout.mastodon.interfaces.api.status.StatusQuery +import dev.usbharu.hideout.mastodon.query.StatusQueryService +import org.jetbrains.exposed.sql.SortOrder +import org.jetbrains.exposed.sql.andWhere +import org.jetbrains.exposed.sql.selectAll +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.stereotype.Service + +@Service +@ConditionalOnProperty("hideout.use-mongodb", havingValue = "false", matchIfMissing = true) +class ExposedGenerateTimelineService(private val statusQueryService: StatusQueryService) : GenerateTimelineService { + override suspend fun getTimeline( + forUserId: Long?, + localOnly: Boolean, + mediaOnly: Boolean, + maxId: Long?, + minId: Long?, + sinceId: Long?, + limit: Int + ): List { + val query = Timelines.selectAll() + + if (forUserId != null) { + query.andWhere { Timelines.userId eq forUserId } + } + if (localOnly) { + query.andWhere { Timelines.isLocal eq true } + } + if (maxId != null) { + query.andWhere { Timelines.id lessEq maxId } + } + if (minId != null) { + query.andWhere { Timelines.id greaterEq minId } + } + val result = query + .limit(limit) + .orderBy(Timelines.createdAt, SortOrder.DESC) + + val statusQueries = result.map { + StatusQuery( + it[Timelines.postId], + it[Timelines.replyId], + it[Timelines.repostId], + it[Timelines.mediaIds].split(",").mapNotNull { it.toLongOrNull() }) + } + + return statusQueryService.findByPostIdsWithMediaIds(statusQueries) + } +} diff --git a/src/main/kotlin/dev/usbharu/hideout/core/service/timeline/TimelineService.kt b/src/main/kotlin/dev/usbharu/hideout/core/service/timeline/TimelineService.kt index 2b514508..73958fb4 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/service/timeline/TimelineService.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/service/timeline/TimelineService.kt @@ -6,6 +6,7 @@ import dev.usbharu.hideout.core.domain.model.timeline.Timeline import dev.usbharu.hideout.core.domain.model.timeline.TimelineRepository import dev.usbharu.hideout.core.query.FollowerQueryService import dev.usbharu.hideout.core.query.UserQueryService +import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @Service @@ -58,5 +59,10 @@ class TimelineService( ) } timelineRepository.saveAll(timelines) + logger.debug("SUCCESS Timeline published. {}", timelines.size) + } + + companion object { + private val logger = LoggerFactory.getLogger(TimelineService::class.java) } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 1aef91a7..b805eb58 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,6 +1,6 @@ hideout: url: "https://test-hideout.usbharu.dev" - use-mongodb: true + use-mongodb: false security: jwt: generate: true @@ -19,15 +19,15 @@ spring: default-property-inclusion: always datasource: driver-class-name: org.h2.Driver - url: "jdbc:h2:./test-dev3;MODE=POSTGRESQL" + url: "jdbc:h2:./test-dev3;MODE=POSTGRESQL;TRACE_LEVEL_FILE=4" username: "" password: "" - data: - mongodb: - auto-index-creation: true - host: localhost - port: 27017 - database: hideout + # data: + # mongodb: + # auto-index-creation: true + # host: localhost + # port: 27017 + # database: hideout # username: hideoutuser # password: hideoutpass servlet: From e11872aac53a1fdd4fbe3c27174c7652e42a0d15 Mon Sep 17 00:00:00 2001 From: usbharu Date: Thu, 16 Nov 2023 23:38:24 +0900 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../exposedrepository/ExposedTimelineRepository.kt | 1 - .../infrastructure/kjobexposed/KJobJobQueueParentService.kt | 4 ++-- .../core/service/timeline/ExposedGenerateTimelineService.kt | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/ExposedTimelineRepository.kt b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/ExposedTimelineRepository.kt index 21865a35..ee372e1c 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/ExposedTimelineRepository.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/exposedrepository/ExposedTimelineRepository.kt @@ -46,7 +46,6 @@ class ExposedTimelineRepository(private val idGenerateService: IdGenerateService it[isLocal] = timeline.isLocal it[isPureRepost] = timeline.isPureRepost it[mediaIds] = timeline.mediaIds.joinToString(",") - } } return timeline diff --git a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/kjobexposed/KJobJobQueueParentService.kt b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/kjobexposed/KJobJobQueueParentService.kt index cc68d15c..0e02b011 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/kjobexposed/KJobJobQueueParentService.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/infrastructure/kjobexposed/KJobJobQueueParentService.kt @@ -19,8 +19,8 @@ class KJobJobQueueParentService() : JobQueueParentService { val kjob: KJob by lazy { kjob(ExposedKJob) { connectionDatabase = TransactionManager.defaultDatabase - isWorker = false - }.start() + isWorker = false + }.start() } override fun init(jobDefines: List) = Unit diff --git a/src/main/kotlin/dev/usbharu/hideout/core/service/timeline/ExposedGenerateTimelineService.kt b/src/main/kotlin/dev/usbharu/hideout/core/service/timeline/ExposedGenerateTimelineService.kt index 5989c3e4..5fc098b2 100644 --- a/src/main/kotlin/dev/usbharu/hideout/core/service/timeline/ExposedGenerateTimelineService.kt +++ b/src/main/kotlin/dev/usbharu/hideout/core/service/timeline/ExposedGenerateTimelineService.kt @@ -45,7 +45,8 @@ class ExposedGenerateTimelineService(private val statusQueryService: StatusQuery it[Timelines.postId], it[Timelines.replyId], it[Timelines.repostId], - it[Timelines.mediaIds].split(",").mapNotNull { it.toLongOrNull() }) + it[Timelines.mediaIds].split(",").mapNotNull { it.toLongOrNull() } + ) } return statusQueryService.findByPostIdsWithMediaIds(statusQueries)