Skip to content

Commit

Permalink
Basic filesystems support for Wasm: support WASI-based filesystem for…
Browse files Browse the repository at this point in the history
… wasmWasi (#257)

Implemented basic filesystem support on top of Wasm WASI.
  • Loading branch information
fzhinkin committed Feb 19, 2024
1 parent 54914dc commit ce4e9aa
Show file tree
Hide file tree
Showing 11 changed files with 1,333 additions and 48 deletions.
43 changes: 26 additions & 17 deletions core/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
*/

import org.jetbrains.dokka.gradle.DokkaTaskPartial
import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinTargetWithNodeJsDsl

plugins {
id("kotlinx-io-multiplatform")
Expand Down Expand Up @@ -32,22 +31,6 @@ kotlin {
}
}

fun KotlinTargetWithNodeJsDsl.filterSmokeTests() {
this.nodejs {
testTask(Action {
useMocha {
timeout = "300s"
}
filter.setExcludePatterns("*SmokeFileTest*")
})
}
}

@OptIn(org.jetbrains.kotlin.gradle.targets.js.dsl.ExperimentalWasmDsl::class)
wasmWasi {
filterSmokeTests()
}

sourceSets {
commonMain {
dependencies {
Expand Down Expand Up @@ -76,6 +59,32 @@ tasks.withType<DokkaTaskPartial>().configureEach {
}
}

tasks.named("wasmWasiNodeTest") {
// TODO: remove once https://youtrack.jetbrains.com/issue/KT-65179 solved
doFirst {
val layout = project.layout
val templateFile = layout.projectDirectory.file("wasmWasi/test/test-driver.mjs.template").asFile

val driverFile = layout.buildDirectory.file(
"compileSync/wasmWasi/test/testDevelopmentExecutable/kotlin/kotlinx-io-kotlinx-io-core-wasm-wasi-test.mjs"
)

fun File.mkdirsAndEscape(): String {
mkdirs()
return absolutePath.replace("\\", "\\\\")
}

val tmpDir = temporaryDir.resolve("kotlinx-io-core-wasi-test").mkdirsAndEscape()
val tmpDir2 = temporaryDir.resolve("kotlinx-io-core-wasi-test-2").mkdirsAndEscape()

val newDriver = templateFile.readText()
.replace("<SYSTEM_TEMP_DIR>", tmpDir, false)
.replace("<SYSTEM_TEMP_DIR2>", tmpDir2, false)

driverFile.get().asFile.writeText(newDriver)
}
}

animalsniffer {
annotation = "kotlinx.io.files.AnimalSnifferIgnore"
}
3 changes: 3 additions & 0 deletions core/common/src/files/FileSystem.kt
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ public sealed interface FileSystem {
* filesystems (or different volumes, on Windows) and the operation could not be performed atomically,
* [UnsupportedOperationException] is thrown.
*
* On some platforms, like Wasm-WASI, there is no way to tell if the underlying filesystem supports atomic move.
* In such cases, the move will be performed and no [UnsupportedOperationException] will be thrown.
*
* When [destination] is an existing directory, the operation may fail on some platforms
* (on Windows, particularly).
*
Expand Down
34 changes: 30 additions & 4 deletions core/common/test/files/SmokeFileTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ class SmokeFileTest {
return f
}

private fun removeOnExit(path: Path) {
files.add(path)
}

@OptIn(ExperimentalStdlibApi::class)
@Test
fun readWriteFile() {
Expand All @@ -46,6 +50,22 @@ class SmokeFileTest {
}
}

@OptIn(ExperimentalStdlibApi::class)
@Test
fun writeFlush() {
val path = createTempPath()
SystemFileSystem.sink(path).buffered().use {
it.writeString("hello")
it.flush()
it.writeString(" world")
it.flush()
}

SystemFileSystem.source(path).buffered().use {
assertEquals("hello world", it.readLine())
}
}

@OptIn(ExperimentalStdlibApi::class)
@Test
fun readNotExistingFile() {
Expand Down Expand Up @@ -362,11 +382,17 @@ class SmokeFileTest {
}

val cwd = SystemFileSystem.resolve(Path("."))
val parentRel = Path("..")
assertEquals(cwd.parent, SystemFileSystem.resolve(parentRel))

assertEquals(cwd, SystemFileSystem.resolve(cwd),
"Absolute path resolution should not alter the path")
SystemFileSystem.createDirectories(Path("a"))
removeOnExit(Path("a"))

val childRel = Path("a", "..")
assertEquals(cwd, SystemFileSystem.resolve(childRel))

assertEquals(
cwd, SystemFileSystem.resolve(cwd),
"Absolute path resolution should not alter the path"
)

// root
// |-> a -> b
Expand Down
111 changes: 111 additions & 0 deletions core/wasmWasi/src/-WasmUtils.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
* Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
*/
@file:OptIn(UnsafeWasmMemoryApi::class)

package kotlinx.io

import kotlin.wasm.unsafe.MemoryAllocator
import kotlin.wasm.unsafe.Pointer
import kotlin.wasm.unsafe.UnsafeWasmMemoryApi

internal fun Pointer.loadInt(offset: Int): Int = (this + offset).loadInt()
internal fun Pointer.loadLong(offset: Int): Long = (this + offset).loadLong()
internal fun Pointer.loadShort(offset: Int): Short = (this + offset).loadShort()
internal fun Pointer.loadByte(offset: Int): Byte = (this + offset).loadByte()

internal fun Pointer.loadBytes(length: Int): ByteArray {
val buffer = ByteArray(length)
for (offset in 0 until length) {
buffer[offset] = this.loadByte(offset)
}
return buffer
}

internal fun Pointer.storeInt(offset: Int, value: Int): Unit = (this + offset).storeInt(value)
internal fun Pointer.storeLong(offset: Int, value: Long): Unit = (this + offset).storeLong(value)
internal fun Pointer.storeShort(offset: Int, value: Short): Unit = (this + offset).storeShort(value)
internal fun Pointer.storeByte(offset: Int, value: Byte): Unit = (this + offset).storeByte(value)

internal fun Pointer.storeBytes(bytes: ByteArray) {
for (offset in bytes.indices) {
this.storeByte(offset, bytes[offset])
}
}

@OptIn(UnsafeWasmMemoryApi::class)
internal fun Buffer.readToLinearMemory(pointer: Pointer, bytes: Int) {
checkBounds(size, 0L, bytes.toLong())
var current: Segment? = head
var remaining = bytes
var currentPtr = pointer
do {
current!!
val data = current.data
val pos = current.pos
val limit = current.limit
val read = minOf(remaining, limit - pos)
for (offset in 0 until read) {
currentPtr.storeByte(offset, data[pos + offset])
}
currentPtr += read
remaining -= read
current = current.next
} while (current != head && remaining > 0)
check(remaining == 0)
skip(bytes.toLong())
}


internal fun Buffer.writeFromLinearMemory(pointer: Pointer, bytes: Int) {
var remaining = bytes
var currentPtr = pointer
while (remaining > 0) {
val segment = writableSegment(1)
val limit = segment.limit
val data = segment.data
val toWrite = minOf(data.size - limit, remaining)

for (offset in 0 until toWrite) {
data[limit + offset] = currentPtr.loadByte(offset)
}

currentPtr += toWrite
remaining -= toWrite
segment.limit += toWrite
size += toWrite
}
}

/**
* Encoding [value] into a NULL-terminated byte sequence using UTF-8 encoding
* and writes it to a memory region allocated to fit the sequence.
* Return a pointer to the beginning of the written byte sequence and its length.
*/
@OptIn(UnsafeWasmMemoryApi::class)
internal fun MemoryAllocator.storeString(value: String): Pair<Pointer, Int> {
val bytes = value.encodeToByteArray()
val ptr = allocate(bytes.size + 1)
ptr.storeBytes(bytes)
ptr.storeByte(bytes.size, 0)
return ptr to (bytes.size + 1)
}

/**
* Encodes [value] into a NULL-terminated byte sequence using UTF-8 encoding,
* stores it in memory starting at the position this pointer points to,
* and returns the length of the stored bytes sequence.
*/
internal fun Pointer.allocateString(value: String): Int {
val bytes = value.encodeToByteArray()
storeBytes(bytes)
storeByte(bytes.size, 0)
return bytes.size + 1
}

/**
* Allocates memory to hold a single integer value.
*/
@UnsafeWasmMemoryApi
internal fun MemoryAllocator.allocateInt(): Pointer = allocate(Int.SIZE_BYTES)
Loading

0 comments on commit ce4e9aa

Please sign in to comment.