refactoring of data transformations

CHANGELOG.md

@@ -6,10 +6,12 @@
 - Wasm artifacts
 - Add automatic MetaConverter for serializeable objects
 - Add Meta and MutableMeta delegates for convertable and serializeable
+- Meta mapping for data.
 
 ### Changed
 - Descriptor `children` renamed to `nodes`
 - `MetaConverter` now inherits `MetaSpec` (former `Specifiction`). So `MetaConverter` could be used more universally.
+- Meta copy and modification now use lightweight non-observable meta builders.
 
 ### Deprecated
 - `node(key,converter)` in favor of `serializable` delegate

dataforge-context/src/jvmMain/kotlin/space/kscience/dataforge/provider/dfType.kt

@@ -12,21 +12,21 @@ import kotlin.reflect.full.findAnnotation
 
 
 @DFExperimental
-public val KClass<*>.dfId: String
+public val KClass<*>.dfType: String
     get() = findAnnotation<DfType>()?.id ?: simpleName ?: ""
 
 /**
  * Provide an object with given name inferring target from its type using [DfType] annotation
  */
 @DFExperimental
 public inline fun <reified T : Any> Provider.provideByType(name: String): T? {
-    val target = T::class.dfId
+    val target = T::class.dfType
     return provide(target, name)
 }
 
 @DFExperimental
 public inline fun <reified T : Any> Provider.top(): Map<Name, T> {
-    val target = T::class.dfId
+    val target = T::class.dfType
     return top(target)
 }
 
@@ -35,15 +35,15 @@ public inline fun <reified T : Any> Provider.top(): Map<Name, T> {
  */
 @DFExperimental
 public inline fun <reified T : Any> Context.gather(inherit: Boolean = true): Map<Name, T> =
-    gather<T>(T::class.dfId, inherit)
+    gather<T>(T::class.dfType, inherit)
 
 
 @DFExperimental
 public inline fun <reified T : Any> PluginBuilder.provides(items: Map<Name, T>) {
-    provides(T::class.dfId, items)
+    provides(T::class.dfType, items)
 }
 
 @DFExperimental
 public inline fun <reified T : Any> PluginBuilder.provides(vararg items: Named) {
-    provides(T::class.dfId, *items)
+    provides(T::class.dfType, *items)
 }

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/actions/Action.kt

@@ -7,30 +7,28 @@ import space.kscience.dataforge.misc.DFExperimental
 /**
  * A simple data transformation on a data node. Actions should avoid doing actual dependency evaluation in [execute].
  */
-public interface Action<in T : Any, out R : Any> {
+public fun interface Action<in T : Any, out R : Any> {
 
     /**
      * Transform the data in the node, producing a new node. By default, it is assumed that all calculations are lazy
      * so not actual computation is started at this moment.
      */
-    public fun execute(dataSet: DataSet<T>, meta: Meta = Meta.EMPTY): DataSet<R>
+    public fun execute(dataSet: DataSet<T>, meta: Meta): DataSet<R>
 
     public companion object
 }
 
+/**
+ * A convenience method to transform data using given [action]
+ */
+public fun <T : Any, R : Any> DataSet<T>.transform(action: Action<T, R>, meta: Meta = Meta.EMPTY): DataSet<R> =
+    action.execute(this, meta)
+
 /**
  * Action composition. The result is terminal if one of its parts is terminal
  */
-public infix fun <T : Any, I : Any, R : Any> Action<T, I>.then(action: Action<I, R>): Action<T, R> {
-    // TODO introduce composite action and add optimize by adding action to the list
-    return object : Action<T, R> {
-
-        override fun execute(
-            dataSet: DataSet<T>,
-            meta: Meta,
-        ): DataSet<R> = action.execute(this@then.execute(dataSet, meta), meta)
-    }
-}
+public infix fun <T : Any, I : Any, R : Any> Action<T, I>.then(action: Action<I, R>): Action<T, R> =
+    Action<T, R> { dataSet, meta -> action.execute(this@then.execute(dataSet, meta), meta) }
 
 @DFExperimental
 public operator fun <T : Any, R : Any> Action<T, R>.invoke(

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/Data.kt

@@ -87,7 +87,6 @@ public class StaticData<T : Any>(
 public inline fun <reified T : Any> Data(value: T, meta: Meta = Meta.EMPTY): StaticData<T> =
     StaticData(typeOf<T>(), value, meta)
 
-@Suppress("FunctionName")
 @DFInternal
 public fun <T : Any> Data(
     type: KType,
@@ -98,7 +97,6 @@ public fun <T : Any> Data(
 ): Data<T> = LazyData(type, meta, context, dependencies, block)
 
 @OptIn(DFInternal::class)
-@Suppress("FunctionName")
 public inline fun <reified T : Any> Data(
     meta: Meta = Meta.EMPTY,
     context: CoroutineContext = EmptyCoroutineContext,

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/MetaMaskData.kt

@@ -0,0 +1,23 @@
+package space.kscience.dataforge.data
+
+import space.kscience.dataforge.meta.Meta
+import space.kscience.dataforge.meta.MutableMeta
+import space.kscience.dataforge.meta.copy
+
+
+private class MetaMaskData<T>(val origin: Data<T>, override val meta: Meta) : Data<T> by origin
+
+/**
+ * A data with overriden meta. It reflects original data computed state.
+ */
+public fun <T> Data<T>.withMeta(newMeta: Meta): Data<T> = if (this is MetaMaskData) {
+    MetaMaskData(origin, newMeta)
+} else {
+    MetaMaskData(this, newMeta)
+}
+
+/**
+ * Create a new [Data] with the same computation, but different meta. The meta is created by applying [block] to
+ * the existing data meta.
+ */
+public inline fun <T> Data<T>.mapMeta(block: MutableMeta.() -> Unit): Data<T> = withMeta(meta.copy(block))

dataforge-data/src/commonMain/kotlin/space/kscience/dataforge/data/dataTransform.kt

@@ -1,11 +1,9 @@
 package space.kscience.dataforge.data
 
-import space.kscience.dataforge.meta.Meta
-import space.kscience.dataforge.meta.MutableMeta
-import space.kscience.dataforge.meta.seal
-import space.kscience.dataforge.meta.toMutableMeta
+import space.kscience.dataforge.meta.*
 import space.kscience.dataforge.misc.DFInternal
 import space.kscience.dataforge.names.Name
+import space.kscience.dataforge.names.NameToken
 import kotlin.coroutines.CoroutineContext
 import kotlin.coroutines.EmptyCoroutineContext
 import kotlin.reflect.KType
@@ -28,8 +26,8 @@ public suspend fun <T : Any> NamedData<T>.awaitWithMeta(): NamedValueWithMeta<T>
  * @param block the transformation itself
  */
 public inline fun <T : Any, reified R : Any> Data<T>.map(
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = this.meta,
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
     crossinline block: suspend (T) -> R,
 ): Data<R> = Data(meta, coroutineContext, listOf(this)) {
     block(await())
@@ -40,8 +38,8 @@ public inline fun <T : Any, reified R : Any> Data<T>.map(
  */
 public inline fun <T1 : Any, T2 : Any, reified R : Any> Data<T1>.combine(
     other: Data<T2>,
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
     meta: Meta = this.meta,
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
     crossinline block: suspend (left: T1, right: T2) -> R,
 ): Data<R> = Data(meta, coroutineContext, listOf(this, other)) {
     block(await(), other.await())
@@ -50,12 +48,22 @@ public inline fun <T1 : Any, T2 : Any, reified R : Any> Data<T1>.combine(
 
 //data collection operations
 
+@PublishedApi
+internal fun Iterable<Data<*>>.joinMeta(): Meta = Meta {
+    var counter = 0
+    forEach { data ->
+        val inputIndex = (data as? NamedData)?.name?.toString() ?: (counter++).toString()
+        val token = NameToken("data", inputIndex)
+        set(token, data.meta)
+    }
+}
+
 /**
  * Lazily reduce a collection of [Data] to a single data.
  */
 public inline fun <T : Any, reified R : Any> Collection<Data<T>>.reduceToData(
+    meta: Meta = joinMeta(),
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
     crossinline block: suspend (List<ValueWithMeta<T>>) -> R,
 ): Data<R> = Data(
     meta,
@@ -65,11 +73,19 @@ public inline fun <T : Any, reified R : Any> Collection<Data<T>>.reduceToData(
     block(map { it.awaitWithMeta() })
 }
 
+@PublishedApi
+internal fun Map<*, Data<*>>.joinMeta(): Meta = Meta {
+    forEach { (key, data) ->
+        val token = NameToken("data", key.toString())
+        set(token, data.meta)
+    }
+}
+
 @DFInternal
 public fun <K, T : Any, R : Any> Map<K, Data<T>>.reduceToData(
     outputType: KType,
+    meta: Meta = joinMeta(),
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
     block: suspend (Map<K, ValueWithMeta<T>>) -> R,
 ): Data<R> = Data(
     outputType,
@@ -87,8 +103,8 @@ public fun <K, T : Any, R : Any> Map<K, Data<T>>.reduceToData(
  * @param R type of the result goal
  */
 public inline fun <K, T : Any, reified R : Any> Map<K, Data<T>>.reduceToData(
+    meta: Meta = joinMeta(),
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
     crossinline block: suspend (Map<K, ValueWithMeta<T>>) -> R,
 ): Data<R> = Data(
     meta,
@@ -103,8 +119,8 @@ public inline fun <K, T : Any, reified R : Any> Map<K, Data<T>>.reduceToData(
 @DFInternal
 public inline fun <T : Any, R : Any> Iterable<Data<T>>.reduceToData(
     outputType: KType,
+    meta: Meta = joinMeta(),
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
     crossinline transformation: suspend (Collection<ValueWithMeta<T>>) -> R,
 ): Data<R> = Data(
     outputType,
@@ -117,20 +133,20 @@ public inline fun <T : Any, R : Any> Iterable<Data<T>>.reduceToData(
 
 @OptIn(DFInternal::class)
 public inline fun <T : Any, reified R : Any> Iterable<Data<T>>.reduceToData(
+    meta: Meta = joinMeta(),
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
     crossinline transformation: suspend (Collection<ValueWithMeta<T>>) -> R,
-): Data<R> = reduceToData(typeOf<R>(), coroutineContext, meta) {
+): Data<R> = reduceToData(typeOf<R>(), meta, coroutineContext) {
     transformation(it)
 }
 
 public inline fun <T : Any, reified R : Any> Iterable<Data<T>>.foldToData(
     initial: R,
+    meta: Meta = joinMeta(),
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
     crossinline block: suspend (result: R, data: ValueWithMeta<T>) -> R,
 ): Data<R> = reduceToData(
-    coroutineContext, meta
+    meta, coroutineContext
 ) {
     it.fold(initial) { acc, t -> block(acc, t) }
 }
@@ -141,8 +157,8 @@ public inline fun <T : Any, reified R : Any> Iterable<Data<T>>.foldToData(
 @DFInternal
 public inline fun <T : Any, R : Any> Iterable<NamedData<T>>.reduceNamedToData(
     outputType: KType,
+    meta: Meta = joinMeta(),
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
     crossinline transformation: suspend (Collection<NamedValueWithMeta<T>>) -> R,
 ): Data<R> = Data(
     outputType,
@@ -155,10 +171,10 @@ public inline fun <T : Any, R : Any> Iterable<NamedData<T>>.reduceNamedToData(
 
 @OptIn(DFInternal::class)
 public inline fun <T : Any, reified R : Any> Iterable<NamedData<T>>.reduceNamedToData(
+    meta: Meta = joinMeta(),
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
     crossinline transformation: suspend (Collection<NamedValueWithMeta<T>>) -> R,
-): Data<R> = reduceNamedToData(typeOf<R>(), coroutineContext, meta) {
+): Data<R> = reduceNamedToData(typeOf<R>(), meta, coroutineContext) {
     transformation(it)
 }
 
@@ -167,11 +183,11 @@ public inline fun <T : Any, reified R : Any> Iterable<NamedData<T>>.reduceNamedT
  */
 public inline fun <T : Any, reified R : Any> Iterable<NamedData<T>>.foldNamedToData(
     initial: R,
+    meta: Meta = joinMeta(),
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
     crossinline block: suspend (result: R, data: NamedValueWithMeta<T>) -> R,
 ): Data<R> = reduceNamedToData(
-    coroutineContext, meta
+    meta, coroutineContext
 ) {
     it.fold(initial) { acc, t -> block(acc, t) }
 }
@@ -181,8 +197,8 @@ public inline fun <T : Any, reified R : Any> Iterable<NamedData<T>>.foldNamedToD
 @DFInternal
 public suspend fun <T : Any, R : Any> DataSet<T>.map(
     outputType: KType,
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
     metaTransform: MutableMeta.() -> Unit = {},
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
     block: suspend (NamedValueWithMeta<T>) -> R,
 ): DataTree<R> = DataTree<R>(outputType) {
     forEach {
@@ -196,26 +212,36 @@ public suspend fun <T : Any, R : Any> DataSet<T>.map(
 
 @OptIn(DFInternal::class)
 public suspend inline fun <T : Any, reified R : Any> DataSet<T>.map(
-    coroutineContext: CoroutineContext = EmptyCoroutineContext,
     noinline metaTransform: MutableMeta.() -> Unit = {},
+    coroutineContext: CoroutineContext = EmptyCoroutineContext,
     noinline block: suspend (NamedValueWithMeta<T>) -> R,
-): DataTree<R> = map(typeOf<R>(), coroutineContext, metaTransform, block)
+): DataTree<R> = map(typeOf<R>(), metaTransform, coroutineContext, block)
 
 public inline fun <T : Any> DataSet<T>.forEach(block: (NamedData<T>) -> Unit) {
     for (d in this) {
         block(d)
     }
 }
 
+// DataSet reduction
+
+@PublishedApi
+internal fun DataSet<*>.joinMeta(): Meta = Meta {
+    forEach { (key, data) ->
+        val token = NameToken("data", key.toString())
+        set(token, data.meta)
+    }
+}
+
 public inline fun <T : Any, reified R : Any> DataSet<T>.reduceToData(
+    meta: Meta = joinMeta(),
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
     crossinline transformation: suspend (Iterable<NamedValueWithMeta<T>>) -> R,
-): Data<R> = asIterable().reduceNamedToData(coroutineContext, meta, transformation)
+): Data<R> = asIterable().reduceNamedToData(meta, coroutineContext, transformation)
 
 public inline fun <T : Any, reified R : Any> DataSet<T>.foldToData(
     initial: R,
+    meta: Meta = joinMeta(),
     coroutineContext: CoroutineContext = EmptyCoroutineContext,
-    meta: Meta = Meta.EMPTY,
     crossinline block: suspend (result: R, data: NamedValueWithMeta<T>) -> R,
-): Data<R> = asIterable().foldNamedToData(initial, coroutineContext, meta, block)
+): Data<R> = asIterable().foldNamedToData(initial, meta, coroutineContext, block)

dataforge-meta/src/commonMain/kotlin/space/kscience/dataforge/meta/MutableMeta.kt

@@ -370,7 +370,8 @@ public fun MutableMeta.append(key: String, value: Value): Unit = append(Name.par
 /**
  * Create a mutable copy of this meta. The copy is created even if the Meta is already mutable
  */
-public fun Meta.toMutableMeta(): ObservableMutableMeta = MutableMetaImpl(value, items)
+public fun Meta.toMutableMeta(): MutableMeta =
+    MutableMeta { update(this@toMutableMeta) } //MutableMetaImpl(value, items)
 
 public fun Meta.asMutableMeta(): MutableMeta = (this as? MutableMeta) ?: toMutableMeta()
 
@@ -385,12 +386,14 @@ public inline fun ObservableMutableMeta(builder: MutableMeta.() -> Unit = {}): O
 
 
 /**
- * Create a copy of this [Meta], optionally applying the given [block].
- * The listeners of the original Config are not retained.
+ * Create a read-only copy of this [Meta]. [modification] is an optional modification applied to [Meta] on copy.
+ *
+ *  The copy does not reflect changes of the initial Meta.
  */
-public inline fun Meta.copy(block: MutableMeta.() -> Unit = {}): Meta =
-    toMutableMeta().apply(block)
-
+public inline fun Meta.copy(modification: MutableMeta.() -> Unit = {}): Meta = Meta {
+    update(this@copy)
+    modification()
+}
 
 private class MutableMetaWithDefault(
     val source: MutableMeta, val default: MetaProvider, val rootName: Name,