Fix value class encoding in various corner cases (#2242)

- Value class is located at top-level, but wraps non-primitive and thus does not fall in 'primitive on top-level' branch
- Value class is a subclass in a polymorphic hierarchy, but either is primitive or explicitly recorded without type info

Note that type info is omitted in the latter case and 'can't add type info to primitive' error is not thrown deliberately, as
there seems to be use-cases for that.

Fixes #1774
Fixes #2159

Author: sandwwraith

core/api/kotlinx-serialization-core.api

@@ -1067,7 +1067,7 @@ public abstract class kotlinx/serialization/internal/TaggedDecoder : kotlinx/ser
 	public final fun decodeEnum (Lkotlinx/serialization/descriptors/SerialDescriptor;)I
 	public final fun decodeFloat ()F
 	public final fun decodeFloatElement (Lkotlinx/serialization/descriptors/SerialDescriptor;I)F
-	public final fun decodeInline (Lkotlinx/serialization/descriptors/SerialDescriptor;)Lkotlinx/serialization/encoding/Decoder;
+	public fun decodeInline (Lkotlinx/serialization/descriptors/SerialDescriptor;)Lkotlinx/serialization/encoding/Decoder;
 	public final fun decodeInlineElement (Lkotlinx/serialization/descriptors/SerialDescriptor;I)Lkotlinx/serialization/encoding/Decoder;
 	public final fun decodeInt ()I
 	public final fun decodeIntElement (Lkotlinx/serialization/descriptors/SerialDescriptor;I)I
@@ -1123,7 +1123,7 @@ public abstract class kotlinx/serialization/internal/TaggedEncoder : kotlinx/ser
 	public final fun encodeEnum (Lkotlinx/serialization/descriptors/SerialDescriptor;I)V
 	public final fun encodeFloat (F)V
 	public final fun encodeFloatElement (Lkotlinx/serialization/descriptors/SerialDescriptor;IF)V
-	public final fun encodeInline (Lkotlinx/serialization/descriptors/SerialDescriptor;)Lkotlinx/serialization/encoding/Encoder;
+	public fun encodeInline (Lkotlinx/serialization/descriptors/SerialDescriptor;)Lkotlinx/serialization/encoding/Encoder;
 	public final fun encodeInlineElement (Lkotlinx/serialization/descriptors/SerialDescriptor;I)Lkotlinx/serialization/encoding/Encoder;
 	public final fun encodeInt (I)V
 	public final fun encodeIntElement (Lkotlinx/serialization/descriptors/SerialDescriptor;II)V

core/commonMain/src/kotlinx/serialization/internal/Tagged.kt

@@ -51,7 +51,7 @@ public abstract class TaggedEncoder<Tag : Any?> : Encoder, CompositeEncoder {
     protected open fun encodeTaggedInline(tag: Tag, inlineDescriptor: SerialDescriptor): Encoder =
         this.apply { pushTag(tag) }
 
-    final override fun encodeInline(descriptor: SerialDescriptor): Encoder =
+    override fun encodeInline(descriptor: SerialDescriptor): Encoder =
         encodeTaggedInline(popTag(), descriptor)
 
     // ---- Implementation of low-level API ----
@@ -209,7 +209,7 @@ public abstract class TaggedDecoder<Tag : Any?> : Decoder, CompositeDecoder {
 
     // ---- Implementation of low-level API ----
 
-    final override fun decodeInline(descriptor: SerialDescriptor): Decoder =
+    override fun decodeInline(descriptor: SerialDescriptor): Decoder =
         decodeTaggedInline(popTag(), descriptor)
 
     // TODO this method should be overridden by any sane format that supports top-level nulls

formats/json-tests/commonTest/src/kotlinx/serialization/features/inline/InlineClassesTest.kt

@@ -22,7 +22,7 @@ data class SimpleContainerForUInt(val i: UInt)
 @JvmInline
 value class MyUInt(val m: Int)
 
-object MyUIntSerializer: KSerializer<MyUInt> {
+object MyUIntSerializer : KSerializer<MyUInt> {
     override val descriptor = UInt.serializer().descriptor
     override fun serialize(encoder: Encoder, value: MyUInt) {
         encoder.encodeInline(descriptor).encodeInt(value.m)
@@ -73,12 +73,45 @@ value class ResourceKind(val kind: SampleEnum)
 @Serializable
 data class ResourceIdentifier(val id: ResourceId, val type: ResourceType, val type2: ValueWrapper)
 
-@Serializable @JvmInline
+@Serializable
+@JvmInline
 value class ValueWrapper(val wrapped: ResourceType)
 
+@Serializable
+@JvmInline
+value class Outer(val inner: Inner)
+
+@Serializable
+data class Inner(val n: Int)
+
+@Serializable
+data class OuterOuter(val outer: Outer)
+
+@Serializable
+@JvmInline
+value class WithList(val value: List<Int>)
+
 class InlineClassesTest : JsonTestBase() {
     private val precedent: UInt = Int.MAX_VALUE.toUInt() + 10.toUInt()
 
+    @Test
+    fun withList() = noLegacyJs {
+        val withList = WithList(listOf(1, 2, 3))
+        assertJsonFormAndRestored(WithList.serializer(), withList, """[1,2,3]""")
+    }
+
+    @Test
+    fun testOuterInner() = noLegacyJs {
+        val o = Outer(Inner(10))
+        assertJsonFormAndRestored(Outer.serializer(), o, """{"n":10}""")
+    }
+
+    @Test
+    fun testOuterOuterInner() = noLegacyJs {
+        val o = OuterOuter(Outer(Inner(10)))
+        assertJsonFormAndRestored(OuterOuter.serializer(), o, """{"outer":{"n":10}}""")
+    }
+
     @Test
     fun testTopLevel() = noLegacyJs {
         assertJsonFormAndRestored(

formats/json-tests/commonTest/src/kotlinx/serialization/features/inline/ValueClassesInSealedHierarchyTest.kt

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2017-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package kotlinx.serialization.features.inline
+
+import kotlinx.serialization.*
+import kotlinx.serialization.json.*
+import kotlinx.serialization.test.*
+import kotlin.jvm.*
+import kotlin.test.*
+
+class ValueClassesInSealedHierarchyTest : JsonTestBase() {
+    @Test
+    fun testSingle() = noLegacyJs {
+        val single = "foo"
+        assertJsonFormAndRestored(
+            AnyValue.serializer(),
+            AnyValue.Single(single),
+            "\"$single\""
+        )
+    }
+
+    @Test
+    fun testComplex() = noLegacyJs {
+        val complexJson = """{"id":"1","name":"object"}"""
+        assertJsonFormAndRestored(
+            AnyValue.serializer(),
+            AnyValue.Complex(mapOf("id" to "1", "name" to "object")),
+            complexJson
+        )
+    }
+
+    @Test
+    fun testMulti() = noLegacyJs {
+        val multiJson = """["list","of","strings"]"""
+        assertJsonFormAndRestored(AnyValue.serializer(), AnyValue.Multi(listOf("list", "of", "strings")), multiJson)
+    }
+}
+
+
+// From https://github.com/Kotlin/kotlinx.serialization/issues/2159
+@Serializable(with = AnyValue.Companion.Serializer::class)
+sealed interface AnyValue {
+
+    @JvmInline
+    @Serializable
+    value class Single(val value: String) : AnyValue
+
+    @JvmInline
+    @Serializable
+    value class Multi(val values: List<String>) : AnyValue
+
+    @JvmInline
+    @Serializable
+    value class Complex(val values: Map<String, String>) : AnyValue
+
+    @JvmInline
+    @Serializable
+    value class Unknown(val value: JsonElement) : AnyValue
+
+    companion object {
+        object Serializer : JsonContentPolymorphicSerializer<AnyValue>(AnyValue::class) {
+
+            override fun selectDeserializer(element: JsonElement): DeserializationStrategy<AnyValue> =
+                when {
+                    element is JsonArray && element.all { it is JsonPrimitive && it.isString } -> Multi.serializer()
+                    element is JsonObject && element.values.all { it is JsonPrimitive && it.isString } -> Complex.serializer()
+                    element is JsonPrimitive && element.isString -> Single.serializer()
+                    else -> Unknown.serializer()
+                }
+        }
+    }
+}

formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonDecoder.kt

@@ -165,9 +165,14 @@ private sealed class AbstractJsonTreeDecoder(
     override fun decodeTaggedInline(tag: String, inlineDescriptor: SerialDescriptor): Decoder =
         if (inlineDescriptor.isUnsignedNumber) JsonDecoderForUnsignedTypes(StringJsonLexer(getPrimitiveValue(tag).content), json)
         else super.decodeTaggedInline(tag, inlineDescriptor)
+
+    override fun decodeInline(descriptor: SerialDescriptor): Decoder {
+        return if (currentTagOrNull != null) super.decodeInline(descriptor)
+        else JsonPrimitiveDecoder(json, value).decodeInline(descriptor)
+    }
 }
 
-private class JsonPrimitiveDecoder(json: Json, override val value: JsonPrimitive) : AbstractJsonTreeDecoder(json, value) {
+private class JsonPrimitiveDecoder(json: Json, override val value: JsonElement) : AbstractJsonTreeDecoder(json, value) {
 
     init {
         pushTag(PRIMITIVE_TAG)

formats/json/commonMain/src/kotlinx/serialization/json/internal/TreeJsonEncoder.kt

@@ -25,7 +25,7 @@ public fun <T> Json.writeJson(value: T, serializer: SerializationStrategy<T>): J
 @ExperimentalSerializationApi
 private sealed class AbstractJsonTreeEncoder(
     final override val json: Json,
-    private val nodeConsumer: (JsonElement) -> Unit
+    protected val nodeConsumer: (JsonElement) -> Unit
 ) : NamedValueEncoder(), JsonEncoder {
 
     final override val serializersModule: SerializersModule
@@ -80,7 +80,6 @@ private sealed class AbstractJsonTreeEncoder(
             encodePolymorphically(serializer, value) { polymorphicDiscriminator = it }
         } else JsonPrimitiveEncoder(json, nodeConsumer).apply {
             encodeSerializableValue(serializer, value)
-            endEncode(serializer.descriptor)
         }
     }
 
@@ -112,6 +111,11 @@ private sealed class AbstractJsonTreeEncoder(
             else -> super.encodeTaggedInline(tag, inlineDescriptor)
         }
 
+    override fun encodeInline(descriptor: SerialDescriptor): Encoder {
+        return if (currentTagOrNull != null) super.encodeInline(descriptor)
+        else JsonPrimitiveEncoder(json, nodeConsumer).encodeInline(descriptor)
+    }
+
     @SuppressAnimalSniffer // Long(Integer).toUnsignedString(long)
     private fun inlineUnsignedNumberEncoder(tag: String) = object : AbstractEncoder() {
             override val serializersModule: SerializersModule = json.serializersModule
@@ -176,6 +180,7 @@ private class JsonPrimitiveEncoder(
         require(key === PRIMITIVE_TAG) { "This output can only consume primitives with '$PRIMITIVE_TAG' tag" }
         require(content == null) { "Primitive element was already recorded. Does call to .encodeXxx happen more than once?" }
         content = element
+        nodeConsumer(element)
     }
 
     override fun getCurrent(): JsonElement =