Merge pull request #1084 from Kotlin/compiler-plugin-core

Prepare plugin to be moved to Kotlin repository

core/build.gradle.kts

@@ -1,8 +1,10 @@
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 import com.google.devtools.ksp.gradle.KspTask
 import com.google.devtools.ksp.gradle.KspTaskJvm
 import io.github.devcrocod.korro.KorroTask
 import nl.jolanrensen.kodex.gradle.creatingRunKodexTask
 import org.gradle.jvm.tasks.Jar
+import org.gradle.kotlin.dsl.withType
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 import xyz.ronella.gradle.plugin.simple.git.task.GitTask
 
@@ -19,6 +21,7 @@ plugins {
         alias(simpleGit)
         alias(buildconfig)
         alias(binary.compatibility.validator)
+        alias(shadow)
 
         // generates keywords using the :generator module
         alias(keywordGenerator)
@@ -173,6 +176,37 @@ tasks.withType<KorroTask> {
     dependsOn(copySamplesOutputs)
 }
 
+// region shadow
+
+tasks.withType<ShadowJar> {
+    dependencies {
+        exclude(dependency("org.jetbrains.kotlin:kotlin-reflect:.*"))
+        exclude(dependency("org.jetbrains.kotlin:kotlin-stdlib:.*"))
+        exclude(dependency("org.jetbrains.kotlinx:kotlinx-datetime-jvm:.*"))
+        exclude(dependency("commons-io:commons-io:.*"))
+        exclude(dependency("commons-io:commons-csv:.*"))
+        exclude(dependency("org.slf4j:slf4j-api:.*"))
+        exclude(dependency("io.github.microutils:kotlin-logging-jvm:.*"))
+        exclude(dependency("org.jetbrains.kotlinx:kotlinx-serialization-core-jvm:.*"))
+        exclude(dependency("org.jetbrains.kotlinx:kotlinx-serialization-json-jvm:.*"))
+        exclude(dependency("commons-codec:commons-codec:.*"))
+        exclude(dependency("com.squareup:kotlinpoet-jvm:.*"))
+        exclude(dependency("ch.randelshofer:fastdoubleparser:.*"))
+    }
+    exclude("org/jetbrains/kotlinx/dataframe/jupyter/**")
+    exclude("org/jetbrains/kotlinx/dataframe/io/**")
+    exclude("org/jetbrains/kotlinx/dataframe/documentation/**")
+    exclude("org/jetbrains/kotlinx/dataframe/impl/io/**")
+    exclude("io/github/oshai/kotlinlogging/**")
+    exclude("apache/**")
+    exclude("**.html")
+    exclude("**.js")
+    exclude("**.css")
+    minimize()
+}
+
+// endregion
+
 // region docPreprocessor
 
 val generatedSourcesFolderName = "generated-sources"
@@ -367,6 +401,7 @@ tasks.withType<KotlinCompile> {
     compilerOptions {
         optIn.addAll("kotlin.RequiresOptIn")
         freeCompilerArgs.addAll("-Xinline-classes")
+        freeCompilerArgs.addAll("-Xjvm-default=all")
     }
 }
 

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/annotations/Plugin.kt

@@ -15,16 +15,10 @@ public annotation class Interpretable(val interpreter: String)
  */
 public annotation class Refine
 
-internal annotation class OptInRefine
-
 @Retention(AnnotationRetention.SOURCE)
 @Target(AnnotationTarget.FILE, AnnotationTarget.EXPRESSION)
 public annotation class DisableInterpretation
 
-@Retention(AnnotationRetention.SOURCE)
-@Target(AnnotationTarget.EXPRESSION)
-public annotation class Import
-
 @Target(AnnotationTarget.PROPERTY)
 public annotation class Order(val order: Int)
 

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/MarkersExtractor.kt

@@ -18,17 +18,37 @@ import kotlin.reflect.jvm.jvmErasure
 import kotlin.reflect.typeOf
 
 internal fun KType.getFieldKind(): FieldKind =
-    when {
-        jvmErasure == DataFrame::class -> Frame
-        jvmErasure == List::class && (arguments[0].type?.jvmErasure?.hasAnnotation<DataSchema>() == true) -> ListToFrame
-        jvmErasure == DataRow::class -> Group
-        jvmErasure.hasAnnotation<DataSchema>() -> ObjectToGroup
-        else -> Default
+    FieldKind.of(
+        this,
+        isDataFrame = { jvmErasure == DataFrame::class },
+        isListToFrame = {
+            jvmErasure == List::class && (arguments[0].type?.jvmErasure?.hasAnnotation<DataSchema>() == true)
+        },
+        isDataRow = { jvmErasure == DataRow::class },
+        isObjectToGroup = { jvmErasure.hasAnnotation<DataSchema>() },
+    )
+
+public sealed interface FieldKind {
+    public val shouldBeConvertedToColumnGroup: Boolean get() = false
+    public val shouldBeConvertedToFrameColumn: Boolean get() = false
+
+    public companion object {
+        // Should be in sync with compiler plugin
+        public fun <T> of(
+            value: T,
+            isDataFrame: (T) -> Boolean,
+            isListToFrame: (T) -> Boolean,
+            isDataRow: (T) -> Boolean,
+            isObjectToGroup: (T) -> Boolean,
+        ): FieldKind =
+            when {
+                isDataFrame(value) -> Frame
+                isListToFrame(value) -> ListToFrame
+                isDataRow(value) -> Group
+                isObjectToGroup(value) -> ObjectToGroup
+                else -> Default
+            }
     }
-
-internal sealed interface FieldKind {
-    val shouldBeConvertedToColumnGroup: Boolean get() = false
-    val shouldBeConvertedToFrameColumn: Boolean get() = false
 }
 
 internal data object Frame : FieldKind {

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/csv.kt

@@ -10,9 +10,6 @@ import org.jetbrains.kotlinx.dataframe.AnyFrame
 import org.jetbrains.kotlinx.dataframe.AnyRow
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.DataRow
-import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
-import org.jetbrains.kotlinx.dataframe.annotations.OptInRefine
-import org.jetbrains.kotlinx.dataframe.annotations.Refine
 import org.jetbrains.kotlinx.dataframe.api.ParserOptions
 import org.jetbrains.kotlinx.dataframe.api.forEach
 import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadCsvMethod
@@ -116,8 +113,6 @@ internal fun isCompressed(url: URL) = isCompressed(url.path)
     message = APACHE_CSV,
     level = DeprecationLevel.HIDDEN, // clashes with the new readDelim
 )
-@Refine
-@Interpretable("ReadDelimStr")
 public fun DataFrame.Companion.readDelimStr(
     text: String,
     delimiter: Char = ',',
@@ -154,8 +149,6 @@ public fun DataFrame.Companion.read(
     replaceWith = ReplaceWith(READ_CSV_FILE_OR_URL_REPLACE, READ_CSV_IMPORT),
     level = DeprecationLevel.WARNING,
 )
-@OptInRefine
-@Interpretable("ReadCSV0")
 public fun DataFrame.Companion.readCSV(
     fileOrUrl: String,
     delimiter: Char = ',',

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/guess.kt

@@ -8,8 +8,6 @@ import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.DataRow
 import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
 import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema
-import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
-import org.jetbrains.kotlinx.dataframe.annotations.OptInRefine
 import org.jetbrains.kotlinx.dataframe.api.single
 import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod
 import org.jetbrains.kotlinx.jupyter.api.Code
@@ -282,8 +280,6 @@ public fun DataFrame.Companion.read(url: URL, header: List<String> = emptyList()
 public fun DataRow.Companion.read(url: URL, header: List<String> = emptyList()): AnyRow =
     DataFrame.read(url, header).single()
 
-@OptInRefine
-@Interpretable("Read0")
 public fun DataFrame.Companion.read(path: String, header: List<String> = emptyList()): AnyFrame =
     read(asUrl(path), header)
 

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/json.kt

@@ -9,9 +9,6 @@ import org.jetbrains.kotlinx.dataframe.AnyFrame
 import org.jetbrains.kotlinx.dataframe.AnyRow
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.DataRow
-import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
-import org.jetbrains.kotlinx.dataframe.annotations.OptInRefine
-import org.jetbrains.kotlinx.dataframe.annotations.Refine
 import org.jetbrains.kotlinx.dataframe.api.JsonPath
 import org.jetbrains.kotlinx.dataframe.api.KeyValueProperty
 import org.jetbrains.kotlinx.dataframe.api.single
@@ -181,8 +178,6 @@ public fun DataRow.Companion.readJson(
  * @param unifyNumbers Whether to [unify the numbers that are read][UnifyingNumbers]. `true` by default.
  * @return [DataFrame] from the given [path].
  */
-@OptInRefine
-@Interpretable("ReadJson0")
 public fun DataFrame.Companion.readJson(
     path: String,
     header: List<String> = emptyList(),
@@ -287,8 +282,6 @@ public fun DataRow.Companion.readJson(
  * @param unifyNumbers Whether to [unify the numbers that are read][UnifyingNumbers]. `true` by default.
  * @return [DataFrame] from the given [text].
  */
-@Refine
-@Interpretable("ReadJsonStr")
 public fun DataFrame.Companion.readJsonStr(
     @Language("json") text: String,
     header: List<String> = emptyList(),
@@ -306,8 +299,6 @@ public fun DataFrame.Companion.readJsonStr(
  * @param unifyNumbers Whether to [unify the numbers that are read][UnifyingNumbers]. `true` by default.
  * @return [DataRow] from the given [text].
  */
-@Refine
-@Interpretable("DataRowReadJsonStr")
 public fun DataRow.Companion.readJsonStr(
     @Language("json") text: String,
     header: List<String> = emptyList(),

dataframe-excel/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/xlsx.kt

@@ -27,8 +27,6 @@ import org.jetbrains.kotlinx.dataframe.AnyRow
 import org.jetbrains.kotlinx.dataframe.ColumnsSelector
 import org.jetbrains.kotlinx.dataframe.DataColumn
 import org.jetbrains.kotlinx.dataframe.DataFrame
-import org.jetbrains.kotlinx.dataframe.annotations.Interpretable
-import org.jetbrains.kotlinx.dataframe.annotations.Refine
 import org.jetbrains.kotlinx.dataframe.api.dataFrameOf
 import org.jetbrains.kotlinx.dataframe.api.forEach
 import org.jetbrains.kotlinx.dataframe.api.select
@@ -182,8 +180,6 @@ public fun DataFrame.Companion.readExcel(
  * ensuring unique column names will make the columns be named according to excel columns, like "A", "B", "C" etc.
  * for unstructured data.
  */
-@Refine
-@Interpretable("ReadExcel")
 public fun DataFrame.Companion.readExcel(
     fileOrUrl: String,
     sheetName: String? = null,
@@ -280,9 +276,7 @@ public fun DataFrame.Companion.readExcel(
  * @param range comma separated list of Excel column letters and column ranges (e.g. “A:E” or “A,C,E:F”)
  */
 @JvmInline
-public value class StringColumns
-    @Interpretable("StringColumns")
-    constructor(public val range: String)
+public value class StringColumns(public val range: String)
 
 public fun StringColumns.toFormattingOptions(formatter: DataFormatter = DataFormatter()): FormattingOptions =
     FormattingOptions(range, formatter)

plugins/kotlin-dataframe/build.gradle.kts

@@ -39,12 +39,9 @@ dependencies {
     testRuntimeOnly("org.jetbrains.kotlin:kotlin-script-runtime:$kotlinVersion")
     testRuntimeOnly("org.jetbrains.kotlin:kotlin-annotations-jvm:$kotlinVersion")
 
-    implementation(project(":core"))
-    implementation(project(":dataframe-excel"))
-    implementation(project(":dataframe-csv"))
-    api(libs.kotlinLogging)
-    api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.7.1")
-
+    implementation(project(":core", "shadow"))
+    testRuntimeOnly(project(":core"))
+    testRuntimeOnly(project(":dataframe-csv"))
     testImplementation("org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion")
     testImplementation("org.jetbrains.kotlin:kotlin-compiler-internal-test-framework:$kotlinVersion")
 

plugins/kotlin-dataframe/src/org/jetbrains/kotlinx/dataframe/plugin/FirDataFrameComponentRegistrar.kt

@@ -6,64 +6,22 @@
 package org.jetbrains.kotlinx.dataframe.plugin
 
 import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
-import org.jetbrains.kotlin.compiler.plugin.AbstractCliOption
-import org.jetbrains.kotlin.compiler.plugin.CliOption
-import org.jetbrains.kotlin.compiler.plugin.CliOptionProcessingException
-import org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor
 import org.jetbrains.kotlin.compiler.plugin.CompilerPluginRegistrar
 import org.jetbrains.kotlin.compiler.plugin.ExperimentalCompilerApi
 import org.jetbrains.kotlin.config.CompilerConfiguration
-import org.jetbrains.kotlin.config.CompilerConfigurationKey
 import org.jetbrains.kotlin.fir.FirSession
-import org.jetbrains.kotlin.fir.caches.FirCache
-import org.jetbrains.kotlin.fir.caches.firCachesFactory
+import org.jetbrains.kotlin.fir.extensions.FirExtensionApiInternals
+import org.jetbrains.kotlin.fir.extensions.FirExtensionRegistrar
+import org.jetbrains.kotlin.fir.extensions.FirExtensionRegistrarAdapter
 import org.jetbrains.kotlinx.dataframe.plugin.extensions.DataRowSchemaSupertype
 import org.jetbrains.kotlinx.dataframe.plugin.extensions.ExpressionAnalysisAdditionalChecker
-import org.jetbrains.kotlinx.dataframe.plugin.extensions.TopLevelExtensionsGenerator
 import org.jetbrains.kotlinx.dataframe.plugin.extensions.FunctionCallTransformer
 import org.jetbrains.kotlinx.dataframe.plugin.extensions.IrBodyFiller
-import org.jetbrains.kotlinx.dataframe.plugin.extensions.KotlinTypeFacade
 import org.jetbrains.kotlinx.dataframe.plugin.extensions.ReturnTypeBasedReceiverInjector
-import org.jetbrains.kotlin.fir.extensions.FirExtensionApiInternals
-import org.jetbrains.kotlin.fir.extensions.FirExtensionRegistrar
-import org.jetbrains.kotlin.fir.extensions.FirExtensionRegistrarAdapter
-import org.jetbrains.kotlinx.dataframe.DataFrame
-import org.jetbrains.kotlinx.dataframe.api.schema
-import org.jetbrains.kotlinx.dataframe.io.readJson
 import org.jetbrains.kotlinx.dataframe.plugin.extensions.TokenGenerator
-import org.jetbrains.kotlinx.dataframe.plugin.impl.PluginDataFrameSchema
-import org.jetbrains.kotlinx.dataframe.plugin.impl.data.toPluginDataFrameSchema
-
-val PATH: CompilerConfigurationKey<String> = CompilerConfigurationKey.create("annotation qualified name")
-val SCHEMAS: CompilerConfigurationKey<String> = CompilerConfigurationKey.create("directory to store IO schemas")
-
-// listOf("-P", "plugin:org.jetbrains.kotlinx.dataframe:path=/home/nikita/IdeaProjects/run-df")
-@OptIn(ExperimentalCompilerApi::class)
-class DataFrameCommandLineProcessor : CommandLineProcessor {
-    companion object {
-        val RESOLUTION_DIRECTORY = CliOption(
-            "path", "<path>", "", required = false, allowMultipleOccurrences = false
-        )
-        val SCHEMAS_DIRECTORY = CliOption(
-            "schemas", "<schemas>", "", required = false, allowMultipleOccurrences = false
-        )
-    }
-    override val pluginId: String = "org.jetbrains.kotlinx.dataframe"
-
-    override val pluginOptions: Collection<AbstractCliOption> = listOf(RESOLUTION_DIRECTORY, SCHEMAS_DIRECTORY)
-
-    override fun processOption(option: AbstractCliOption, value: String, configuration: CompilerConfiguration) {
-        return when (option) {
-            RESOLUTION_DIRECTORY -> configuration.put(PATH, value)
-            SCHEMAS_DIRECTORY -> configuration.put(SCHEMAS, value)
-            else -> throw CliOptionProcessingException("Unknown option: ${option.optionName}")
-        }
-    }
-}
+import org.jetbrains.kotlinx.dataframe.plugin.extensions.TopLevelExtensionsGenerator
 
 class FirDataFrameExtensionRegistrar(
-    private val path: String?,
-    val schemasDirectory: String?,
     val isTest: Boolean,
     val dumpSchemas: Boolean,
 ) : FirExtensionRegistrar() {
@@ -72,32 +30,23 @@ class FirDataFrameExtensionRegistrar(
         +::TopLevelExtensionsGenerator
         +::ReturnTypeBasedReceiverInjector
         +{ it: FirSession ->
-            FunctionCallTransformer(path, it, jsonCache(it), schemasDirectory, isTest)
+            FunctionCallTransformer(it, isTest)
         }
         +::TokenGenerator
         +::DataRowSchemaSupertype
         +{ it: FirSession ->
-            ExpressionAnalysisAdditionalChecker(it, jsonCache(it), schemasDirectory, isTest, dumpSchemas)
+            ExpressionAnalysisAdditionalChecker(it, isTest, dumpSchemas)
         }
     }
-
-    private fun jsonCache(it: FirSession): FirCache<String, PluginDataFrameSchema, KotlinTypeFacade> =
-        it.firCachesFactory.createCache { path: String, context ->
-            with(context) {
-                DataFrame.readJson(path).schema().toPluginDataFrameSchema()
-            }
-        }
 }
 
 @OptIn(ExperimentalCompilerApi::class)
 class FirDataFrameComponentRegistrar : CompilerPluginRegistrar() {
     override fun ExtensionStorage.registerExtensions(configuration: CompilerConfiguration) {
-        val schemasDirectory = configuration.get(SCHEMAS)
-        val path = configuration.get(PATH)
         FirExtensionRegistrarAdapter.registerExtension(
-            FirDataFrameExtensionRegistrar(path, schemasDirectory, isTest = false, dumpSchemas = true)
+            FirDataFrameExtensionRegistrar(isTest = false, dumpSchemas = true)
         )
-        IrGenerationExtension.registerExtension(IrBodyFiller(path, schemasDirectory))
+        IrGenerationExtension.registerExtension(IrBodyFiller())
     }
 
     override val supportsK2: Boolean = true