Skip to content

Commit

Permalink
Replace nio with multiplatform okio
Browse files Browse the repository at this point in the history
  • Loading branch information
sellmair committed Apr 14, 2024
1 parent 4035280 commit 84a8c75
Show file tree
Hide file tree
Showing 42 changed files with 440 additions and 304 deletions.
6 changes: 3 additions & 3 deletions core/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,16 @@ tasks.register<Sync>("install") {

tasks.register<JavaExec>("buildTestProject") {
dependsOn("install")
workingDir = file("testProject")
workingDir = rootDir.resolve("samples/ktorServer")
classpath = kotlin.target.compilations["main"].runtimeDependencyFiles
args = listOf("build")
args = listOf("pkg")
mainClass = "io.sellmair.okay.OkMain"
}

dependencies {
implementation(project(":okay-fs"))

implementation("io.ktor:ktor-client-cio-jvm:2.3.9")
implementation("io.ktor:ktor-client-cio:2.3.9")
implementation("io.ktor:ktor-client-core:2.3.9")
implementation("org.apache.maven:maven-model:3.9.6")
Expand All @@ -65,7 +66,6 @@ dependencies {
implementation("org.jetbrains.kotlinx:kotlinx-serialization-protobuf:1.6.3")
implementation("org.slf4j:slf4j-api:2.0.12")
implementation("org.slf4j:slf4j-jdk14:2.0.12")
implementation("io.ktor:ktor-client-cio-jvm:2.3.9")

testImplementation(kotlin("test-junit5"))
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.8.0")
Expand Down
38 changes: 14 additions & 24 deletions core/src/main/kotlin/io/sellmair/okay/OkCache.kt
Original file line number Diff line number Diff line change
@@ -1,18 +1,12 @@
package io.sellmair.okay

import io.sellmair.okay.fs.*
import io.sellmair.okay.serialization.format
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.decodeFromByteArray
import kotlinx.serialization.encodeToByteArray
import java.io.ObjectInputStream
import java.io.ObjectOutputStream
import java.nio.file.Path
import kotlin.io.path.createDirectories
import kotlin.io.path.inputStream
import kotlin.io.path.isRegularFile
import kotlin.io.path.outputStream

/**
* The dispatcher used to read/write anything to the cache.
Expand All @@ -39,16 +33,15 @@ internal val OkContext.cacheBlobsDirectory
@OptIn(ExperimentalSerializationApi::class)
internal suspend fun OkContext.readCacheRecord(key: OkHash): OkCacheRecord? = withOkContext(okCacheDispatcher) {
val file = cacheEntriesDirectory.resolve(key.value)
if (!file.system().isRegularFile()) return@withOkContext null
ObjectInputStream(file.system().inputStream()).use { stream ->
format.decodeFromByteArray<OkCacheRecord>(stream.readAllBytes())
}/*
We should not read entries that have been written during this exact session.
Otherwise, the UP-TO-DATE checks have a problem:
Yes, the recently stored cache entry is UP-TO-DATE (because it was just stored),
but it might not be UP-TO-DATE from the perspective of the previously stored
coroutine as the dependencies (or outputs) might have changed.
*/
if (!file.isRegularFile()) return@withOkContext null
format.decodeFromByteArray<OkCacheRecord>(file.readAll())
/*
We should not read entries that have been written during this exact session.
Otherwise, the UP-TO-DATE checks have a problem:
Yes, the recently stored cache entry is UP-TO-DATE (because it was just stored),
but it might not be UP-TO-DATE from the perspective of the previously stored
coroutine as the dependencies (or outputs) might have changed.
*/
.takeUnless { it.session == currentOkSessionId() }
}

Expand All @@ -57,14 +50,11 @@ internal suspend fun OkContext.readCacheRecord(key: OkHash): OkCacheRecord? = wi
*/
@OptIn(ExperimentalSerializationApi::class)
@OkUnsafe("Consider using ")
internal suspend fun OkContext.storeCacheRecord(value: OkCacheRecord): Path = withOkContext(okCacheDispatcher) {
cacheEntriesDirectory.system().createDirectories()

val file = cacheEntriesDirectory.resolve(value.inputHash.value).system()
ObjectOutputStream(file.outputStream().buffered()).use { stream ->
stream.write(format.encodeToByteArray(value))
}
internal suspend fun OkContext.storeCacheRecord(value: OkCacheRecord): OkPath = withOkContext(okCacheDispatcher) {
cacheEntriesDirectory.createDirectories()

val file = cacheEntriesDirectory.resolve(value.inputHash.value)
file.write(format.encodeToByteArray(value))
file
}

2 changes: 1 addition & 1 deletion core/src/main/kotlin/io/sellmair/okay/OkCacheRecord.kt
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.sellmair.okay

import io.sellmair.okay.fs.OkPath
import io.sellmair.okay.input.OkInput
import io.sellmair.okay.io.OkPath
import io.sellmair.okay.output.OkOutput
import io.sellmair.okay.serialization.Base64Serializer
import kotlinx.serialization.Contextual
Expand Down
16 changes: 10 additions & 6 deletions core/src/main/kotlin/io/sellmair/okay/OkCacheRestore.kt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

package io.sellmair.okay

import io.sellmair.okay.fs.copyTo
import io.sellmair.okay.fs.createParentDirectories
import io.sellmair.okay.fs.deleteRecursively
import io.sellmair.okay.fs.isRegularFile
import io.sellmair.okay.output.OkOutputDirectory
import io.sellmair.okay.output.OkOutputs
import io.sellmair.okay.utils.*
Expand All @@ -16,8 +20,8 @@ internal data class OkCacheHit(
) : OkCacheResult() {
override fun toString(): String {
return "CacheHit(" +
"upToDate=${upToDate.joinToString { it.descriptor.id }}, " +
"restored=${restored.joinToString { it.descriptor.id }})"
"upToDate=${upToDate.joinToString { it.descriptor.id }}, " +
"restored=${restored.joinToString { it.descriptor.id }})"
}
}

Expand Down Expand Up @@ -93,13 +97,13 @@ private fun OkContext.restoreFilesFromCache(
) {
entry.output.withClosure { output -> if (output is OkOutputs) output.values else emptyList() }
.filterIsInstance<OkOutputDirectory>()
.forEach { outputDirectory -> outputDirectory.path.system().deleteRecursively() }
.forEach { outputDirectory -> outputDirectory.path.deleteRecursively() }

entry.outputFiles.orEmpty().forEach { (path, hash) ->
val blob = cacheBlobsDirectory.resolve(hash.value).system()
val blob = cacheBlobsDirectory.resolve(hash.value)
if (blob.isRegularFile()) {
path.system().createParentDirectories()
blob.copyTo(path.system(), true)
path.createParentDirectories()
blob.copyTo(path)
}
}
}
Expand Down
25 changes: 10 additions & 15 deletions core/src/main/kotlin/io/sellmair/okay/OkCacheStore.kt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.sellmair.okay

import io.sellmair.okay.fs.*
import io.sellmair.okay.input.OkInput
import io.sellmair.okay.io.regularFileStateHash
import io.sellmair.okay.output.OkOutput
Expand All @@ -9,13 +10,8 @@ import io.sellmair.okay.output.OkOutputs
import io.sellmair.okay.serialization.format
import kotlinx.coroutines.Dispatchers
import kotlinx.serialization.KSerializer
import kotlinx.serialization.Serializer
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.encodeToStream
import java.nio.file.FileAlreadyExistsException
import java.nio.file.Path
import kotlin.io.path.*
import kotlin.io.path.ExperimentalPathApi


/**
Expand All @@ -37,23 +33,22 @@ suspend fun <T> OkContext.storeCachedCoroutine(
serializer: KSerializer<T>,
dependencies: Set<OkHash>
): OkCacheRecord = withOkContext(Dispatchers.IO) {
cacheBlobsDirectory.system().createDirectories()
cacheBlobsDirectory.createDirectories()

val outputFiles = output.walkFiles()
.toList()
.mapNotNull { path ->
if (!path.exists() || !path.isRegularFile()) return@mapNotNull null
if (!path.isRegularFile()) return@mapNotNull null
val fileCacheKey = path.regularFileStateHash()
val blobFile = cacheBlobsDirectory.resolve(fileCacheKey.value).system()
val blobFile = cacheBlobsDirectory.resolve(fileCacheKey.value)

try {
blobFile.createFile()
path.copyTo(blobFile, true)
path.copyTo(blobFile)
} catch (t: FileAlreadyExistsException) {
/* File exists, no need to store it */
}

path.ok() to fileCacheKey
path to fileCacheKey
}
.toMap()

Expand All @@ -74,12 +69,12 @@ suspend fun <T> OkContext.storeCachedCoroutine(
}

@OptIn(ExperimentalPathApi::class)
fun OkOutput.walkFiles(): Sequence<Path> {
fun OkOutput.walkFiles(): Sequence<OkPath> {
return sequence {
when (this@walkFiles) {
is OkOutputs -> yieldAll(values.flatMap { it.walkFiles() })
is OkOutputDirectory -> yieldAll(path.system().walk())
is OkOutputFile -> yield(path.system())
is OkOutputDirectory -> yieldAll(path.listRecursively())
is OkOutputFile -> yield(path)
}
}
}
6 changes: 3 additions & 3 deletions core/src/main/kotlin/io/sellmair/okay/OkContext.kt
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@

package io.sellmair.okay

import io.sellmair.okay.io.OkPath
import io.sellmair.okay.fs.OkPath
import io.sellmair.okay.fs.absolutePathString
import kotlinx.coroutines.*
import okio.Path.Companion.toPath
import java.nio.file.Path
import kotlin.coroutines.CoroutineContext
import kotlin.coroutines.EmptyCoroutineContext
Expand All @@ -29,8 +31,6 @@ interface OkContext {
val cs: CoroutineScope

val ctx: OkContext get() = this

fun Path.ok(): OkPath = path(this)
}

internal operator fun OkContext.plus(element: CoroutineContext.Element): OkContext {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.sellmair.okay

import io.sellmair.okay.input.OkInput
import io.sellmair.okay.io.OkPath
import io.sellmair.okay.fs.OkPath
import kotlinx.serialization.Serializable
import kotlin.reflect.typeOf

Expand Down
28 changes: 10 additions & 18 deletions core/src/main/kotlin/io/sellmair/okay/OkModuleContext.kt
Original file line number Diff line number Diff line change
@@ -1,38 +1,30 @@
package io.sellmair.okay

import io.sellmair.okay.io.OkPath
import io.sellmair.okay.fs.OkFs
import io.sellmair.okay.fs.OkPath
import io.sellmair.okay.fs.absolutePathString
import okio.Path.Companion.toPath
import java.nio.file.Path
import kotlin.coroutines.CoroutineContext
import kotlin.io.path.Path
import kotlin.io.path.name
import kotlin.io.path.relativeTo
import kotlin.io.path.pathString


@OptIn(OkUnsafe::class)
fun OkContext.path(value: String): OkPath {
val root = cs.coroutineContext[OkRoot]?.path?.toString() ?: ""
return OkPath(root, value)
}

@OptIn(OkUnsafe::class)
fun OkContext.rootPath() = path("")

fun OkContext.modulePath(): OkPath {
return cs.coroutineContext[OkModuleContext]?.path ?: rootPath()
}

fun OkContext.moduleName() = modulePath().system().toAbsolutePath()
.normalize().name
fun OkContext.moduleName() = modulePath().absolutePathString().toPath(true).name

fun OkContext.modulePath(path: String): OkPath {
return modulePath().resolve(path)
}


@OptIn(OkUnsafe::class)
fun OkContext.path(path: Path): OkPath {
val root = cs.coroutineContext[OkRoot]?.path ?: Path("")
return OkPath(root.toString(), path.relativeTo(root).toString())
fun OkContext.path(value: String): OkPath {
val root = cs.coroutineContext[OkRoot]?.path?.toString() ?: ""
return OkFs(root).path(value)
}

internal suspend fun <T> OkContext.withOkModule(path: OkPath, block: suspend OkContext.() -> T): T {
Expand All @@ -56,4 +48,4 @@ internal class OkRoot(
override val key: CoroutineContext.Key<*> = Key

companion object Key : CoroutineContext.Key<OkRoot>
}
}
4 changes: 2 additions & 2 deletions core/src/main/kotlin/io/sellmair/okay/clean/okClean.kt
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
package io.sellmair.okay.clean

import io.sellmair.okay.OkContext
import io.sellmair.okay.fs.deleteRecursively
import io.sellmair.okay.okExtensions
import io.sellmair.okay.path
import io.sellmair.okay.utils.log
import kotlin.io.path.ExperimentalPathApi
import kotlin.io.path.deleteRecursively

@OptIn(ExperimentalPathApi::class)
suspend fun OkContext.okClean() {
log("Cleaning .okay")
path(".okay").system().deleteRecursively()
path(".okay").deleteRecursively()

okExtensions<OkCleanExtension> { extension ->
extension.clean(this)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package io.sellmair.okay.dependency

import io.sellmair.okay.io.OkPath
import io.sellmair.okay.fs.OkPath

data class OkDependenciesFile(
val module: OkPath,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.sellmair.okay.dependency

import io.sellmair.okay.OkContext
import io.sellmair.okay.io.OkPath
import io.sellmair.okay.fs.OkPath
import io.sellmair.okay.path

data class OkDependencyDeclaration(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package io.sellmair.okay.dependency

import io.sellmair.okay.*
import io.sellmair.okay.fs.isRegularFile
import io.sellmair.okay.fs.readText
import io.sellmair.okay.input.asInput
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.json.*
Expand All @@ -15,11 +17,9 @@ suspend fun OkContext.parseDependenciesFile(): OkDependenciesFile? {
describeCoroutine("parseLibsFile"),
input = dependenciesFile.asInput()
) task@{
val file = dependenciesFile.system()
if (!file.exists()) return@task null
val parsedObject = file.inputStream().buffered().use { inputStream ->
Json.decodeFromStream<JsonElement>(inputStream)
}.jsonObject
if (!dependenciesFile.isRegularFile()) return@task null
val parsedObject =Json.decodeFromString<JsonElement>(dependenciesFile.readText())
.jsonObject

val declarations = parsedObject.keys.map { dependencyKey ->
OkDependencyDeclaration(
Expand Down
12 changes: 6 additions & 6 deletions core/src/main/kotlin/io/sellmair/okay/input/OkInputFile.kt
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
package io.sellmair.okay.input

import io.sellmair.okay.*
import io.sellmair.okay.io.OkPath
import io.sellmair.okay.OkContext
import io.sellmair.okay.OkHash
import io.sellmair.okay.fs.OkPath
import io.sellmair.okay.fs.isDirectory
import io.sellmair.okay.io.directoryStateHash
import io.sellmair.okay.io.regularFileStateHash
import kotlinx.serialization.Serializable
import kotlin.io.path.isDirectory

fun OkPath.asInput(): OkInputFile = OkInputFile(this)

@Serializable
data class OkInputFile(val path: OkPath) : OkInput {
override suspend fun currentHash(ctx: OkContext): OkHash {
val systemPath = path.system()
return if (systemPath.isDirectory()) systemPath.directoryStateHash()
else systemPath.regularFileStateHash()
return if (path.isDirectory()) path.directoryStateHash()
else path.regularFileStateHash()
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ data class OkInputFileCollection(val files: OkFileCollection) : OkInput {
override suspend fun currentHash(ctx: OkContext): OkHash {
return hash {
files.resolve(ctx).forEach { path ->
push(path.system().regularFileStateHash())
push(path.regularFileStateHash())
}
}
}
Expand Down
Loading

0 comments on commit 84a8c75

Please sign in to comment.