usbharu
/
Hideout
mirror of https://github.com/usbharu/Hideout.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

style: fix lint

This commit is contained in:
usbharu 2024-09-17 22:02:29 +09:00
parent 1a2142e83b
commit 4bc9e3abd2
Signed by: usbharu
GPG Key ID: 95CBCF7046307B77
92 changed files with 413 additions and 461 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
owl/build.gradle.kts
View File

@ -53,7 +53,6 @@ subprojects {
autoCorrect = true
}
project.gradle.taskGraph.whenReady {
if (this.hasTask(":koverGenerateArtifact")) {
val task = this.allTasks.find { println(it.name);it.name == "test" }
@ -61,9 +60,24 @@ subprojects {
verificationTask.ignoreFailures = true
}
}
tasks.test {
tasks {
withType<io.gitlab.arturbosch.detekt.Detekt> {
exclude("**/generated/**")
setSource("src/main/kotlin")
exclude("build/")
configureEach {
exclude("**/org/koin/ksp/generated/**", "**/generated/**")
}
}
withType<io.gitlab.arturbosch.detekt.DetektCreateBaselineTask>() {
configureEach {
exclude("**/org/koin/ksp/generated/**", "**/generated/**")
}
}
withType<Test> {
useJUnitPlatform()
}
}
publishing {
repositories {

2
owl/owl-broker/owl-broker-mongodb/src/main/kotlin/dev/usbharu/owl/broker/mongodb/MongoModuleContext.kt
View File

@ -32,7 +32,6 @@ import org.koin.dsl.module
class MongoModuleContext : ModuleContext {
override fun module(): Module {
return module {
single {
val clientSettings =
@ -47,7 +46,6 @@ class MongoModuleContext : ModuleContext {
)
.uuidRepresentation(UuidRepresentation.STANDARD).build()
MongoClient.create(clientSettings)
.getDatabase(System.getProperty("owl.broker.mongo.database", "mongo-test"))
}

12
owl/owl-broker/owl-broker-mongodb/src/main/kotlin/dev/usbharu/owl/broker/mongodb/MongodbConsumerRepository.kt
View File

@ -33,7 +33,11 @@ class MongodbConsumerRepository(database: MongoDatabase) : ConsumerRepository {
private val collection = database.getCollection<ConsumerMongodb>("consumers")
override suspend fun save(consumer: Consumer): Consumer = withContext(Dispatchers.IO) {
collection.replaceOne(Filters.eq("_id", consumer.id.toString()), ConsumerMongodb.of(consumer), ReplaceOptions().upsert(true))
collection.replaceOne(
Filters.eq("_id", consumer.id.toString()),
ConsumerMongodb.of(consumer),
ReplaceOptions().upsert(true)
)
return@withContext consumer
}
@ -53,9 +57,13 @@ data class ConsumerMongodb(
fun toConsumer(): Consumer {
return Consumer(
UUID.fromString(id), name, hostname, tasks
UUID.fromString(id),
name,
hostname,
tasks
)
}
companion object {
fun of(consumer: Consumer): ConsumerMongodb {
return ConsumerMongodb(

38
owl/owl-broker/owl-broker-mongodb/src/main/kotlin/dev/usbharu/owl/broker/mongodb/MongodbQueuedTaskRepository.kt
View File

@ -48,7 +48,8 @@ class MongodbQueuedTaskRepository(
override suspend fun save(queuedTask: QueuedTask): QueuedTask {
withContext(Dispatchers.IO) {
collection.replaceOne(
eq("_id", queuedTask.task.id.toString()), QueuedTaskMongodb.of(propertySerializerFactory, queuedTask),
eq("_id", queuedTask.task.id.toString()),
QueuedTaskMongodb.of(propertySerializerFactory, queuedTask),
ReplaceOptions().upsert(true)
)
}
@ -57,7 +58,6 @@ class MongodbQueuedTaskRepository(
override suspend fun findByTaskIdAndAssignedConsumerIsNullAndUpdate(id: UUID, update: QueuedTask): QueuedTask {
return withContext(Dispatchers.IO) {
val findOneAndUpdate = collection.findOneAndUpdate(
and(
eq("_id", id.toString()),
@ -155,14 +155,14 @@ data class QueuedTaskMongodb(
companion object {
fun of(propertySerializerFactory: PropertySerializerFactory, task: Task): TaskMongodb {
return TaskMongodb(
task.name,
task.id.toString(),
task.publishProducerId.toString(),
task.publishedAt,
task.nextRetry,
task.completedAt,
task.attempt,
PropertySerializeUtils.serialize(propertySerializerFactory, task.properties)
name = task.name,
id = task.id.toString(),
publishProducerId = task.publishProducerId.toString(),
publishedAt = task.publishedAt,
nextRetry = task.nextRetry,
completedAt = task.completedAt,
attempt = task.attempt,
properties = PropertySerializeUtils.serialize(propertySerializerFactory, task.properties)
)
}
}
@ -171,15 +171,15 @@ data class QueuedTaskMongodb(
companion object {
fun of(propertySerializerFactory: PropertySerializerFactory, queuedTask: QueuedTask): QueuedTaskMongodb {
return QueuedTaskMongodb(
queuedTask.task.id.toString(),
TaskMongodb.of(propertySerializerFactory, queuedTask.task),
queuedTask.attempt,
queuedTask.queuedAt,
queuedTask.priority,
queuedTask.isActive,
queuedTask.timeoutAt,
queuedTask.assignedConsumer?.toString(),
queuedTask.assignedAt
id = queuedTask.task.id.toString(),
task = TaskMongodb.of(propertySerializerFactory, queuedTask.task),
attempt = queuedTask.attempt,
queuedAt = queuedTask.queuedAt,
priority = queuedTask.priority,
isActive = queuedTask.isActive,
timeoutAt = queuedTask.timeoutAt,
assignedConsumer = queuedTask.assignedConsumer?.toString(),
assignedAt = queuedTask.assignedAt
)
}
}

31
owl/owl-broker/owl-broker-mongodb/src/main/kotlin/dev/usbharu/owl/broker/mongodb/MongodbTaskRepository.kt
View File

@ -36,27 +36,29 @@ import org.bson.codecs.pojo.annotations.BsonRepresentation
import java.time.Instant
import java.util.*
class MongodbTaskRepository(database: MongoDatabase, private val propertySerializerFactory: PropertySerializerFactory) :
TaskRepository {
private val collection = database.getCollection<TaskMongodb>("tasks")
override suspend fun save(task: Task): Task = withContext(Dispatchers.IO) {
collection.replaceOne(
Filters.eq("_id", task.id.toString()), TaskMongodb.of(propertySerializerFactory, task),
Filters.eq("_id", task.id.toString()),
TaskMongodb.of(propertySerializerFactory, task),
ReplaceOptions().upsert(true)
)
return@withContext task
}
override suspend fun saveAll(tasks: List<Task>): Unit = withContext(Dispatchers.IO) {
collection.bulkWrite(tasks.map {
collection.bulkWrite(
tasks.map {
ReplaceOneModel(
Filters.eq(it.id.toString()),
TaskMongodb.of(propertySerializerFactory, it),
ReplaceOptions().upsert(true)
)
})
}
)
}
override fun findByNextRetryBeforeAndCompletedAtIsNull(timestamp: Instant): Flow<Task> {
@ -75,12 +77,13 @@ class MongodbTaskRepository(database: MongoDatabase, private val propertySeriali
override suspend fun findByIdAndUpdate(id: UUID, task: Task) {
collection.replaceOne(
Filters.eq("_id", task.id.toString()), TaskMongodb.of(propertySerializerFactory, task),
Filters.eq("_id", task.id.toString()),
TaskMongodb.of(propertySerializerFactory, task),
ReplaceOptions().upsert(false)
)
}
override suspend fun findByPublishProducerIdAndCompletedAtIsNotNull(publishProducerId: UUID): Flow<Task> {
override fun findByPublishProducerIdAndCompletedAtIsNotNull(publishProducerId: UUID): Flow<Task> {
return collection
.find(Filters.eq(TaskMongodb::publishProducerId.name, publishProducerId.toString()))
.map { it.toTask(propertySerializerFactory) }
@ -116,14 +119,14 @@ data class TaskMongodb(
companion object {
fun of(propertySerializerFactory: PropertySerializerFactory, task: Task): TaskMongodb {
return TaskMongodb(
task.name,
task.id.toString(),
task.publishProducerId.toString(),
task.publishedAt,
task.nextRetry,
task.completedAt,
task.attempt,
PropertySerializeUtils.serialize(propertySerializerFactory, task.properties)
name = task.name,
id = task.id.toString(),
publishProducerId = task.publishProducerId.toString(),
publishedAt = task.publishedAt,
nextRetry = task.nextRetry,
completedAt = task.completedAt,
attempt = task.attempt,
properties = PropertySerializeUtils.serialize(propertySerializerFactory, task.properties)
)
}
}

35
owl/owl-broker/owl-broker-mongodb/src/main/kotlin/dev/usbharu/owl/broker/mongodb/MongodbTaskResultRepository.kt
View File

@ -41,14 +41,17 @@ class MongodbTaskResultRepository(
private val collection = database.getCollection<TaskResultMongodb>("task_results")
override suspend fun save(taskResult: TaskResult): TaskResult = withContext(Dispatchers.IO) {
collection.replaceOne(
Filters.eq(taskResult.id.toString()), TaskResultMongodb.of(propertySerializerFactory, taskResult),
Filters.eq(taskResult.id.toString()),
TaskResultMongodb.of(propertySerializerFactory, taskResult),
ReplaceOptions().upsert(true)
)
return@withContext taskResult
}
override fun findByTaskId(id: UUID): Flow<TaskResult> {
return collection.find(Filters.eq(id.toString())).map { it.toTaskResult(propertySerializerFactory) }.flowOn(Dispatchers.IO)
return collection.find(
Filters.eq(id.toString())
).map { it.toTaskResult(propertySerializerFactory) }.flowOn(Dispatchers.IO)
}
}
@ -65,27 +68,25 @@ data class TaskResultMongodb(
fun toTaskResult(propertySerializerFactory: PropertySerializerFactory): TaskResult {
return TaskResult(
UUID.fromString(id),
UUID.fromString(taskId),
success,
attempt,
PropertySerializeUtils.deserialize(propertySerializerFactory, result),
message
id = UUID.fromString(id),
taskId = UUID.fromString(taskId),
success = success,
attempt = attempt,
result = PropertySerializeUtils.deserialize(propertySerializerFactory, result),
message = message
)
}
companion object {
fun of(propertySerializerFactory: PropertySerializerFactory, taskResult: TaskResult): TaskResultMongodb {
return TaskResultMongodb(
taskResult.id.toString(),
taskResult.taskId.toString(),
taskResult.success,
taskResult.attempt,
PropertySerializeUtils.serialize(propertySerializerFactory, taskResult.result),
taskResult.message
id = taskResult.id.toString(),
taskId = taskResult.taskId.toString(),
success = taskResult.success,
attempt = taskResult.attempt,
result = PropertySerializeUtils.serialize(propertySerializerFactory, taskResult.result),
message = taskResult.message
)
}
}
}
}

2
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/Main.kt
View File

@ -90,7 +90,6 @@ fun main() {
logger.info("Use module name: {}", moduleContext)
val koin = startKoin {
printLogger()
@ -98,7 +97,6 @@ fun main() {
single<RetryPolicyFactory> {
DefaultRetryPolicyFactory(mapOf("" to ExponentialRetryPolicy()))
}
}
modules(mainModule, module, moduleContext.module())
}

4
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/ModuleContext.kt
View File

@ -23,7 +23,5 @@ interface ModuleContext {
}
data object EmptyModuleContext : ModuleContext {
override fun module(): Module {
return org.koin.dsl.module { }
}
override fun module(): Module = org.koin.dsl.module { }
}

1
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/OwlBrokerApplication.kt
View File

@ -64,5 +64,4 @@ class OwlBrokerApplication(
fun stop() {
server.shutdown()
}
}

2
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/domain/model/task/TaskRepository.kt
View File

@ -31,5 +31,5 @@ interface TaskRepository {
suspend fun findByIdAndUpdate(id: UUID, task: Task)
suspend fun findByPublishProducerIdAndCompletedAtIsNotNull(publishProducerId:UUID):Flow<Task>
fun findByPublishProducerIdAndCompletedAtIsNotNull(publishProducerId: UUID): Flow<Task>
}

3
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/interfaces/grpc/AssignmentTaskService.kt
View File

@ -16,7 +16,6 @@
package dev.usbharu.owl.broker.interfaces.grpc
import dev.usbharu.owl.broker.external.toTimestamp
import dev.usbharu.owl.broker.external.toUUID
import dev.usbharu.owl.broker.service.QueuedTaskAssigner
@ -33,7 +32,6 @@ import org.slf4j.LoggerFactory
import kotlin.coroutines.CoroutineContext
import kotlin.coroutines.EmptyCoroutineContext
class AssignmentTaskService(
coroutineContext: CoroutineContext = EmptyCoroutineContext,
private val queuedTaskAssigner: QueuedTaskAssigner,
@ -42,7 +40,6 @@ class AssignmentTaskService(
AssignmentTaskServiceGrpcKt.AssignmentTaskServiceCoroutineImplBase(coroutineContext) {
override fun ready(requests: Flow<Task.ReadyRequest>): Flow<Task.TaskRequest> {
return try {
requests
.flatMapMerge {

17
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/interfaces/grpc/DefinitionTaskService.kt
View File

@ -25,17 +25,20 @@ import dev.usbharu.owl.generated.DefinitionTaskServiceGrpcKt.DefinitionTaskServi
import kotlin.coroutines.CoroutineContext
import kotlin.coroutines.EmptyCoroutineContext
class DefinitionTaskService(coroutineContext: CoroutineContext = EmptyCoroutineContext,private val registerTaskService: RegisterTaskService) :
class DefinitionTaskService(
coroutineContext: CoroutineContext = EmptyCoroutineContext,
private val registerTaskService: RegisterTaskService
) :
DefinitionTaskServiceCoroutineImplBase(coroutineContext) {
override suspend fun register(request: DefinitionTask.TaskDefinition): TaskDefined {
registerTaskService.registerTask(
TaskDefinition(
request.name,
request.priority,
request.maxRetry,
request.timeoutMilli,
request.propertyDefinitionHash,
request.retryPolicy
name = request.name,
priority = request.priority,
maxRetry = request.maxRetry,
timeoutMilli = request.timeoutMilli,
propertyDefinitionHash = request.propertyDefinitionHash,
retryPolicy = request.retryPolicy
)
)
return TaskDefined

8
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/interfaces/grpc/ProducerService.kt
View File

@ -24,16 +24,18 @@ import dev.usbharu.owl.generated.ProducerServiceGrpcKt.ProducerServiceCoroutineI
import kotlin.coroutines.CoroutineContext
import kotlin.coroutines.EmptyCoroutineContext
class ProducerService(
coroutineContext: CoroutineContext = EmptyCoroutineContext,
private val producerService: ProducerService
) :
ProducerServiceCoroutineImplBase(coroutineContext) {
override suspend fun registerProducer(request: ProducerOuterClass.Producer): ProducerOuterClass.RegisterProducerResponse {
override suspend fun registerProducer(
request: ProducerOuterClass.Producer
): ProducerOuterClass.RegisterProducerResponse {
val registerProducer = producerService.registerProducer(
RegisterProducerRequest(
request.name, request.hostname
request.name,
request.hostname
)
)
return ProducerOuterClass.RegisterProducerResponse.newBuilder().setId(registerProducer.toUUID()).build()

5
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/interfaces/grpc/TaskPublishService.kt
View File

@ -39,13 +39,9 @@ class TaskPublishService(
TaskPublishServiceCoroutineImplBase(coroutineContext) {
override suspend fun publishTask(request: PublishTaskOuterClass.PublishTask): PublishedTask {
logger.warn("aaaaaaaaaaa")
return try {
val publishedTask = taskPublishService.publishTask(
PublishTask(
request.name,
@ -61,7 +57,6 @@ class TaskPublishService(
}
override suspend fun publishTasks(request: PublishTaskOuterClass.PublishTasks): PublishedTasks {
val tasks = request.propertiesArrayList.map {
PublishTask(
request.name,

6
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/interfaces/grpc/TaskResultSubscribeService.kt
View File

@ -41,7 +41,8 @@ class TaskResultSubscribeService(
name = it.name
attempt = it.attempt
success = it.success
results.addAll(it.results.map {
results.addAll(
it.results.map {
taskResult {
id = it.taskId.toUUID()
success = it.success
@ -49,7 +50,8 @@ class TaskResultSubscribeService(
result.putAll(PropertySerializeUtils.serialize(propertySerializerFactory, it.result))
message = it.message
}
})
}
)
}
}
}

2
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/service/AssignQueuedTaskDecider.kt
View File

@ -42,7 +42,5 @@ class AssignQueuedTaskDeciderImpl(
).take(numberOfConcurrent)
)
}
}
}

2
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/service/ProducerService.kt
View File

@ -26,10 +26,8 @@ interface ProducerService {
suspend fun registerProducer(producer: RegisterProducerRequest): UUID
}
class ProducerServiceImpl(private val producerRepository: ProducerRepository) : ProducerService {
override suspend fun registerProducer(producer: RegisterProducerRequest): UUID {
val id = UUID.randomUUID()
val saveProducer = Producer(

11
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/service/QueueScanner.kt
View File

@ -29,19 +29,14 @@ interface QueueScanner {
fun startScan(): Flow<QueuedTask>
}
class QueueScannerImpl(private val queueStore: QueueStore) : QueueScanner {
override fun startScan(): Flow<QueuedTask> {
return flow {
override fun startScan(): Flow<QueuedTask> = flow {
while (currentCoroutineContext().isActive) {
emitAll(scanQueue())
delay(1000)
}
}
}
private fun scanQueue(): Flow<QueuedTask> {
return queueStore.findByQueuedAtBeforeAndIsActiveIsTrue(Instant.now().minusSeconds(10))
}
private fun scanQueue(): Flow<QueuedTask> =
queueStore.findByQueuedAtBeforeAndIsActiveIsTrue(Instant.now().minusSeconds(10))
}

19
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/service/QueueStore.kt
View File

@ -32,33 +32,24 @@ interface QueueStore {
fun findByQueuedAtBeforeAndIsActiveIsTrue(instant: Instant): Flow<QueuedTask>
}
class QueueStoreImpl(private val queuedTaskRepository: QueuedTaskRepository) : QueueStore {
override suspend fun enqueue(queuedTask: QueuedTask) {
queuedTaskRepository.save(queuedTask)
}
override suspend fun enqueueAll(queuedTaskList: List<QueuedTask>) {
queuedTaskList.forEach { enqueue(it) }
}
override suspend fun enqueueAll(queuedTaskList: List<QueuedTask>) = queuedTaskList.forEach { enqueue(it) }
override suspend fun dequeue(queuedTask: QueuedTask) {
queuedTaskRepository.findByTaskIdAndAssignedConsumerIsNullAndUpdate(queuedTask.task.id, queuedTask)
}
override suspend fun dequeueAll(queuedTaskList: List<QueuedTask>) {
return queuedTaskList.forEach { dequeue(it) }
}
override suspend fun dequeueAll(queuedTaskList: List<QueuedTask>) = queuedTaskList.forEach { dequeue(it) }
override fun findByTaskNameInAndIsActiveIsTrueAndOrderByPriority(
tasks: List<String>,
limit: Int
): Flow<QueuedTask> {
return queuedTaskRepository.findByTaskNameInAndIsActiveIsTrueAndOrderByPriority(tasks, limit)
}
override fun findByQueuedAtBeforeAndIsActiveIsTrue(instant: Instant): Flow<QueuedTask> {
return queuedTaskRepository.findByQueuedAtBeforeAndIsActiveIsTrue(instant)
}
): Flow<QueuedTask> = queuedTaskRepository.findByTaskNameInAndIsActiveIsTrueAndOrderByPriority(tasks, limit)
override fun findByQueuedAtBeforeAndIsActiveIsTrue(instant: Instant): Flow<QueuedTask> =
queuedTaskRepository.findByQueuedAtBeforeAndIsActiveIsTrue(instant)
}

2
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/service/QueuedTaskAssigner.kt
View File

@ -27,7 +27,6 @@ interface QueuedTaskAssigner {
fun ready(consumerId: UUID, numberOfConcurrent: Int): Flow<QueuedTask>
}
class QueuedTaskAssignerImpl(
private val taskManagementService: TaskManagementService,
private val queueStore: QueueStore
@ -49,7 +48,6 @@ class QueuedTaskAssignerImpl(
private suspend fun assignTask(queuedTask: QueuedTask, consumerId: UUID): QueuedTask? {
return try {
val assignedTaskQueue =
queuedTask.copy(assignedConsumer = consumerId, assignedAt = Instant.now(), isActive = false)
logger.trace(

5
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/service/RegisterTaskService.kt
View File

@ -27,14 +27,15 @@ interface RegisterTaskService {
suspend fun unregisterTask(name: String)
}
class RegisterTaskServiceImpl(private val taskDefinitionRepository: TaskDefinitionRepository) : RegisterTaskService {
override suspend fun registerTask(taskDefinition: TaskDefinition) {
val definedTask = taskDefinitionRepository.findByName(taskDefinition.name)
if (definedTask != null) {
logger.debug("Task already defined. name: ${taskDefinition.name}")
if (taskDefinition.propertyDefinitionHash != definedTask.propertyDefinitionHash) {
throw IncompatibleTaskException("Task ${taskDefinition.name} has already been defined, and the parameters are incompatible.")
throw IncompatibleTaskException(
"Task ${taskDefinition.name} has already been defined, and the parameters are incompatible."
)
}
return
}

10
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/service/TaskManagementService.kt
View File

@ -31,7 +31,6 @@ import org.slf4j.LoggerFactory
import java.time.Instant
import java.util.*
interface TaskManagementService {
suspend fun startManagement(coroutineScope: CoroutineScope)
@ -75,13 +74,11 @@ class TaskManagementServiceImpl(
}
}
override fun findAssignableTask(consumerId: UUID, numberOfConcurrent: Int): Flow<QueuedTask> {
return assignQueuedTaskDecider.findAssignableQueue(consumerId, numberOfConcurrent)
}
private suspend fun enqueueTask(task: Task): QueuedTask {
val definedTask = taskDefinitionRepository.findByName(task.name)
?: throw TaskNotRegisterException("Task ${task.name} not definition.")
@ -113,7 +110,6 @@ class TaskManagementServiceImpl(
queueStore.dequeue(timeoutQueue)
val task = taskRepository.findById(timeoutQueue.task.id)
?: throw RecordNotFoundException("Task not found. id: ${timeoutQueue.task.id}")
val copy = task.copy(attempt = timeoutQueue.attempt)
@ -148,12 +144,10 @@ class TaskManagementServiceImpl(
taskResult.taskId,
task.copy(completedAt = completedAt, attempt = taskResult.attempt)
)
}
override fun subscribeResult(producerId: UUID): Flow<TaskResults> {
return flow {
while (currentCoroutineContext().isActive) {
taskRepository
.findByPublishProducerIdAndCompletedAtIsNotNull(producerId)
@ -163,7 +157,7 @@ class TaskManagementServiceImpl(
TaskResults(
it.name,
it.id,
results.any { it.success },
results.any { taskResult -> taskResult.success },
it.attempt,
results
)
@ -171,9 +165,7 @@ class TaskManagementServiceImpl(
}
delay(500)
}
}
}
companion object {

17
owl/owl-broker/src/main/kotlin/dev/usbharu/owl/broker/service/TaskPublishService.kt
View File

@ -78,7 +78,6 @@ class TaskPublishServiceImpl(
}
override suspend fun publishTasks(list: List<PublishTask>): List<PublishedTask> {
val first = list.first()
val definition = taskDefinitionRepository.findByName(first.name)
@ -90,14 +89,14 @@ class TaskPublishServiceImpl(
val tasks = list.map {
Task(
it.name,
UUID.randomUUID(),
first.producerId,
published,
nextRetry,
null,
0,
it.properties
name = it.name,
id = UUID.randomUUID(),
publishProducerId = first.producerId,
publishedAt = published,
nextRetry = nextRetry,
completedAt = null,
attempt = 0,
properties = it.properties
)
}

1
owl/owl-common/owl-common-serialize-jackson/src/main/kotlin/dev/usbharu/owl/common/property/ObjectPropertyValue.kt
View File

@ -45,6 +45,5 @@ class ObjectPropertySerializer(private val objectMapper: ObjectMapper) : Propert
Class.forName(string.substringAfter("jackson:").substringBefore(":"))
)
)
}
}

5
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/ReflectionUtils.kt
View File

@ -23,4 +23,7 @@ val Class<*>.allFields: List<Field>
superclass.allFields + declaredFields
} else {
declaredFields.toList()
}.map { it.trySetAccessible();it }
}.map {
it.trySetAccessible()
it
}

17
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/property/BooleanPropertyValue.kt
View File

@ -15,19 +15,12 @@ class BooleanPropertyValue(override val value: Boolean) : PropertyValue<Boolean>
*
*/
class BooleanPropertySerializer : PropertySerializer<Boolean> {
override fun isSupported(propertyValue: PropertyValue<*>): Boolean {
return propertyValue.value is Boolean
}
override fun isSupported(propertyValue: PropertyValue<*>): Boolean = propertyValue.value is Boolean
override fun isSupported(string: String): Boolean {
return string.startsWith("bool:")
}
override fun isSupported(string: String): Boolean = string.startsWith("bool:")
override fun serialize(propertyValue: PropertyValue<*>): String {
return "bool:" + propertyValue.value.toString()
}
override fun serialize(propertyValue: PropertyValue<*>): String = "bool:" + propertyValue.value.toString()
override fun deserialize(string: String): PropertyValue<Boolean> {
return BooleanPropertyValue(string.replace("bool:", "").toBoolean())
}
override fun deserialize(string: String): PropertyValue<Boolean> =
BooleanPropertyValue(string.replace("bool:", "").toBoolean())
}

9
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/property/CustomPropertySerializerFactory.kt
View File

@ -23,12 +23,9 @@ package dev.usbharu.owl.common.property
*/
open class CustomPropertySerializerFactory(private val propertySerializers: Set<PropertySerializer<*>>) :
PropertySerializerFactory {
override fun <T> factory(propertyValue: PropertyValue<T>): PropertySerializer<T> {
return propertySerializers.firstOrNull { it.isSupported(propertyValue) } as PropertySerializer<T>?
override fun <T> factory(propertyValue: PropertyValue<T>): PropertySerializer<T> =
propertySerializers.firstOrNull { it.isSupported(propertyValue) } as PropertySerializer<T>?
?: throw IllegalArgumentException("PropertySerializer not found: $propertyValue")
}
override fun factory(string: String): PropertySerializer<*> {
return propertySerializers.first { it.isSupported(string) }
}
override fun factory(string: String): PropertySerializer<*> = propertySerializers.first { it.isSupported(string) }
}

17
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/property/DoublePropertyValue.kt
View File

@ -15,19 +15,12 @@ class DoublePropertyValue(override val value: Double) : PropertyValue<Double>()
*
*/
class DoublePropertySerializer : PropertySerializer<Double> {
override fun isSupported(propertyValue: PropertyValue<*>): Boolean {
return propertyValue.value is Double
}
override fun isSupported(propertyValue: PropertyValue<*>): Boolean = propertyValue.value is Double
override fun isSupported(string: String): Boolean {
return string.startsWith("double:")
}
override fun isSupported(string: String): Boolean = string.startsWith("double:")
override fun serialize(propertyValue: PropertyValue<*>): String {
return "double:" + propertyValue.value.toString()
}
override fun serialize(propertyValue: PropertyValue<*>): String = "double:" + propertyValue.value.toString()
override fun deserialize(string: String): PropertyValue<Double> {
return DoublePropertyValue(string.replace("double:", "").toDouble())
}
override fun deserialize(string: String): PropertyValue<Double> =
DoublePropertyValue(string.replace("double:", "").toDouble())
}

17
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/property/FloatPropertyValue.kt
View File

@ -26,19 +26,12 @@ class FloatPropertyValue(override val value: Float) : PropertyValue<Float>() {
*
*/
class FloatPropertySerializer : PropertySerializer<Float> {
override fun isSupported(propertyValue: PropertyValue<*>): Boolean {
return propertyValue.value is Float
}
override fun isSupported(propertyValue: PropertyValue<*>): Boolean = propertyValue.value is Float
override fun isSupported(string: String): Boolean {
return string.startsWith("float:")
}
override fun isSupported(string: String): Boolean = string.startsWith("float:")
override fun serialize(propertyValue: PropertyValue<*>): String {
return "float:" + propertyValue.value.toString()
}
override fun serialize(propertyValue: PropertyValue<*>): String = "float:" + propertyValue.value.toString()
override fun deserialize(string: String): PropertyValue<Float> {
return FloatPropertyValue(string.replace("float:", "").toFloat())
}
override fun deserialize(string: String): PropertyValue<Float> =
FloatPropertyValue(string.replace("float:", "").toFloat())
}

17
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/property/IntegerPropertyValue.kt
View File

@ -31,19 +31,12 @@ class IntegerPropertyValue(override val value: Int) : PropertyValue<Int>() {
*
*/
class IntegerPropertySerializer : PropertySerializer<Int> {
override fun isSupported(propertyValue: PropertyValue<*>): Boolean {
return propertyValue.value is Int
}
override fun isSupported(propertyValue: PropertyValue<*>): Boolean = propertyValue.value is Int
override fun isSupported(string: String): Boolean {
return string.startsWith("int32:")
}
override fun isSupported(string: String): Boolean = string.startsWith("int32:")
override fun serialize(propertyValue: PropertyValue<*>): String {
return "int32:" + propertyValue.value.toString()
}
override fun serialize(propertyValue: PropertyValue<*>): String = "int32:" + propertyValue.value.toString()
override fun deserialize(string: String): PropertyValue<Int> {
return IntegerPropertyValue(string.replace("int32:", "").toInt())
}
override fun deserialize(string: String): PropertyValue<Int> =
IntegerPropertyValue(string.replace("int32:", "").toInt())
}

17
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/property/LongPropertyValue.kt
View File

@ -27,19 +27,12 @@ class LongPropertyValue(override val value: Long) : PropertyValue<Long>() {
*
*/
class LongPropertySerializer : PropertySerializer<Long> {
override fun isSupported(propertyValue: PropertyValue<*>): Boolean {
return propertyValue.value is Long
}
override fun isSupported(propertyValue: PropertyValue<*>): Boolean = propertyValue.value is Long
override fun isSupported(string: String): Boolean {
return string.startsWith("int64:")
}
override fun isSupported(string: String): Boolean = string.startsWith("int64:")
override fun serialize(propertyValue: PropertyValue<*>): String {
return "int64:" + propertyValue.value.toString()
}
override fun serialize(propertyValue: PropertyValue<*>): String = "int64:" + propertyValue.value.toString()
override fun deserialize(string: String): PropertyValue<Long> {
return LongPropertyValue(string.replace("int64:", "").toLong())
}
override fun deserialize(string: String): PropertyValue<Long> =
LongPropertyValue(string.replace("int64:", "").toLong())
}

6
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/property/PropertyValue.kt
View File

@ -31,9 +31,5 @@ abstract class PropertyValue<T> {
* プロパティの型
*/
abstract val type: PropertyType
override fun toString(): String {
return "PropertyValue(value=$value, type=$type)"
}
override fun toString(): String = "PropertyValue(value=$value, type=$type)"
}

16
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/property/StringPropertyValue.kt
View File

@ -15,19 +15,11 @@ class StringPropertyValue(override val value: String) : PropertyValue<String>()
*
*/
class StringPropertyValueSerializer : PropertySerializer<String> {
override fun isSupported(propertyValue: PropertyValue<*>): Boolean {
return propertyValue.value is String
}
override fun isSupported(propertyValue: PropertyValue<*>): Boolean = propertyValue.value is String
override fun isSupported(string: String): Boolean {
return string.startsWith("str:")
}
override fun isSupported(string: String): Boolean = string.startsWith("str:")
override fun serialize(propertyValue: PropertyValue<*>): String {
return "str:" + propertyValue.value
}
override fun serialize(propertyValue: PropertyValue<*>): String = "str:" + propertyValue.value
override fun deserialize(string: String): PropertyValue<String> {
return StringPropertyValue(string.replace("str:", ""))
}
override fun deserialize(string: String): PropertyValue<String> = StringPropertyValue(string.replace("str:", ""))
}

1
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/retry/ExponentialRetryPolicy.kt
View File

@ -13,5 +13,4 @@ import kotlin.math.roundToLong
class ExponentialRetryPolicy(private val firstRetrySeconds: Int = 30) : RetryPolicy {
override fun nextRetry(now: Instant, attempt: Int): Instant =
now.plusSeconds(firstRetrySeconds.times((2.0).pow(attempt).roundToLong()) - firstRetrySeconds)
}

5
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/retry/RetryPolicyFactory.kt
View File

@ -16,7 +16,6 @@
package dev.usbharu.owl.common.retry
import org.slf4j.LoggerFactory
interface RetryPolicyFactory {
@ -24,9 +23,7 @@ interface RetryPolicyFactory {
}
class DefaultRetryPolicyFactory(private val map: Map<String, RetryPolicy>) : RetryPolicyFactory {
override fun factory(name: String): RetryPolicy {
return map[name] ?: throwException(name)
}
override fun factory(name: String): RetryPolicy = map[name] ?: throwException(name)
private fun throwException(name: String): Nothing {
logger.warn("RetryPolicy not found. name: {}", name)

2
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/task/PropertyDefinition.kt
View File

@ -36,6 +36,4 @@ class PropertyDefinition(val map: Map<String, PropertyType>) : Map<String, Prope
map.map { it.key + it.value.name }.joinToString("").map { hash *= it.code * 31 }
return hash
}
}

1
owl/owl-common/src/main/kotlin/dev/usbharu/owl/common/task/TaskDefinition.kt
View File

@ -107,7 +107,6 @@ interface TaskDefinition<T : Task> {
* @return デシリアライズされたタスク
*/
fun deserialize(value: Map<String, PropertyValue<*>>): T {
val task = try {
type.getDeclaredConstructor().newInstance()
} catch (e: Exception) {

1
owl/owl-consumer/src/main/kotlin/dev/usbharu/owl/consumer/AbstractTaskRunner.kt
View File

@ -29,5 +29,4 @@ abstract class AbstractTaskRunner<T : Task, D : TaskDefinition<T>>(private val t
}
abstract suspend fun typedRun(typedParam: T, taskRequest: TaskRequest): TaskResult
}

51
owl/owl-consumer/src/main/kotlin/dev/usbharu/owl/consumer/Consumer.kt
View File

@ -66,11 +66,13 @@ class Consumer(
suspend fun init(name: String, hostname: String) {
logger.info("Initialize Consumer name: {} hostname: {}", name, hostname)
logger.debug("Registered Tasks: {}", runnerMap.keys)
consumerId = subscribeTaskStub.subscribeTask(subscribeTaskRequest {
consumerId = subscribeTaskStub.subscribeTask(
subscribeTaskRequest {
this.name = name
this.hostname = hostname
this.tasks.addAll(runnerMap.keys)
}).id
}
).id
logger.info("Success initialize consumer. ConsumerID: {}", consumerId)
}
@ -84,11 +86,14 @@ class Consumer(
while (isActive) {
try {
taskResultStub
.tasKResult(flow {
.tasKResult(
flow {
assignmentTaskStub
.ready(flow {
.ready(
flow {
requestTask()
}).onEach {
}
).onEach {
logger.info("Start Task name: {} id: {}", it.name, it.id)
processing.update { it + 1 }
@ -101,7 +106,10 @@ class Consumer(
it.id.leastSignificantUuidBits
),
it.attempt,
Instant.ofEpochSecond(it.queuedAt.seconds, it.queuedAt.nanos.toLong()),
Instant.ofEpochSecond(
it.queuedAt.seconds,
it.queuedAt.nanos.toLong()
),
PropertySerializeUtils.deserialize(
propertySerializerFactory,
it.propertiesMap
@ -109,17 +117,20 @@ class Consumer(
)
)
emit(taskResult {
emit(
taskResult {
this.success = taskResult.success
this.attempt = it.attempt
this.id = it.id
this.result.putAll(
PropertySerializeUtils.serialize(
propertySerializerFactory, taskResult.result
propertySerializerFactory,
taskResult.result
)
)
this.message = taskResult.message
})
}
)
logger.info(
"Success execute task. name: {} success: {}",
it.name,
@ -128,21 +139,25 @@ class Consumer(
logger.debug("TRACE RESULT {}", taskResult)
} catch (e: CancellationException) {
logger.warn("Cancelled execute task.", e)
emit(taskResult {
emit(
taskResult {
this.success = false
this.attempt = it.attempt
this.id = it.id
this.message = e.localizedMessage
})
}
)
throw e
} catch (e: Exception) {
logger.warn("Failed execute task. name: {} id: {}", it.name, it.id, e)
emit(taskResult {
emit(
taskResult {
this.success = false
this.attempt = it.attempt
this.id = it.id
this.message = e.localizedMessage
})
}
)
} finally {
logger.debug(" Task name: {} id: {}", it.name, it.id)
processing.update { it - 1 }
@ -155,7 +170,8 @@ class Consumer(
}
}
}.flowOn(Dispatchers.Default).collect()
})
}
)
} catch (e: CancellationException) {
throw e
} catch (e: Exception) {
@ -171,14 +187,15 @@ class Consumer(
while (coroutineScope.isActive) {
val andSet = concurrent.getAndUpdate { 0 }
if (andSet != 0) {
logger.debug("Request {} tasks.", andSet)
try {
emit(readyRequest {
emit(
readyRequest {
this.consumerId = this@Consumer.consumerId
this.numberOfConcurrent = andSet
})
}
)
} catch (e: CancellationException) {
throw e
} catch (e: Exception) {

1
owl/owl-consumer/src/main/kotlin/dev/usbharu/owl/consumer/Main.kt
View File

@ -25,5 +25,4 @@ fun main() {
standaloneConsumer.init()
standaloneConsumer.start()
}
}

7
owl/owl-consumer/src/main/kotlin/dev/usbharu/owl/consumer/StandaloneConsumer.kt
View File

@ -90,9 +90,11 @@ class StandaloneConsumer(
*/
suspend fun start() {
consumer.start()
Runtime.getRuntime().addShutdownHook(Thread {
Runtime.getRuntime().addShutdownHook(
Thread {
consumer.stop()
})
}
)
}
/**
@ -102,5 +104,4 @@ class StandaloneConsumer(
fun stop() {
consumer.stop()
}
}

13
owl/owl-producer/owl-producer-default/src/main/kotlin/dev/usbharu/owl/producer/defaultimpl/DefaultOwlProducer.kt
View File

@ -36,10 +36,12 @@ class DefaultOwlProducer(private val defaultOwlProducerConfig: DefaultOwlProduce
override suspend fun start() {
producerServiceCoroutineStub =
ProducerServiceGrpcKt.ProducerServiceCoroutineStub(defaultOwlProducerConfig.channel)
producerId = producerServiceCoroutineStub.registerProducer(producer {
producerId = producerServiceCoroutineStub.registerProducer(
producer {
this.name = defaultOwlProducerConfig.name
this.hostname = defaultOwlProducerConfig.hostname
}).id
}
).id
defineTaskServiceCoroutineStub =
DefinitionTaskServiceGrpcKt.DefinitionTaskServiceCoroutineStub(defaultOwlProducerConfig.channel)
@ -48,9 +50,9 @@ class DefaultOwlProducer(private val defaultOwlProducerConfig: DefaultOwlProduce
TaskPublishServiceGrpcKt.TaskPublishServiceCoroutineStub(defaultOwlProducerConfig.channel)
}
override suspend fun <T : Task> registerTask(taskDefinition: TaskDefinition<T>) {
defineTaskServiceCoroutineStub.register(taskDefinition {
defineTaskServiceCoroutineStub.register(
taskDefinition {
this.producerId = this@DefaultOwlProducer.producerId
this.name = taskDefinition.name
this.maxRetry = taskDefinition.maxRetry
@ -58,7 +60,8 @@ class DefaultOwlProducer(private val defaultOwlProducerConfig: DefaultOwlProduce
this.retryPolicy = taskDefinition.retryPolicy
this.timeoutMilli = taskDefinition.timeoutMilli
this.propertyDefinitionHash = taskDefinition.propertyDefinition.hash()
})
}
)
}
override suspend fun <T : Task> publishTask(task: T): PublishedTask<T> {

1
owl/owl-producer/owl-producer-embedded/src/main/kotlin/dev/usbharu/owl/producer/embedded/EmbeddedOwlProducer.kt
View File

@ -96,7 +96,6 @@ class EmbeddedOwlProducer(
}
override suspend fun <T : Task> publishTask(task: T): PublishedTask<T> {
val taskDefinition = taskMap.getValue(task::class.java) as TaskDefinition<T>
val publishTask = application.get<TaskPublishService>().publishTask(

1
owl/owl-producer/owl-producer-embedded/src/main/kotlin/dev/usbharu/owl/producer/embedded/EmbeddedOwlProducerBuilder.kt
View File

@ -46,7 +46,6 @@ class EmbeddedOwlProducerBuilder : OwlProducerBuilder<EmbeddedOwlProducer, Embed
override fun apply(owlProducerConfig: EmbeddedOwlProducerConfig) {
this.config = owlProducerConfig
}
}
val EMBEDDED by lazy { EmbeddedOwlProducerBuilder() }