Skip to content

Commit f16ffa0

Browse files
nikolay-egorovnikolay-egorov
committed
Add special type classes in shared-compiler
1 parent dad4df7 commit f16ffa0

File tree

5 files changed

+90
-33
lines changed

5 files changed

+90
-33
lines changed

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/compiler/util/serializedCompiledScript.kt

+40-1
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,48 @@ data class SerializedCompiledScriptsData(
2020
}
2121
}
2222

23+
@Serializable
24+
data class SerializableTypeInfo(val type: Type = Type.Custom, val isPrimitive: Boolean = false, val fullType: String = "") {
25+
companion object {
26+
val ignoreSet = setOf("int", "double", "boolean", "char", "float", "byte", "string", "entry")
27+
28+
val propertyNamesForNullFilter = setOf("data", "size")
29+
30+
fun makeFromSerializedVariablesState(type: String?, isContainer: Boolean?): SerializableTypeInfo {
31+
val fullType = type.orEmpty()
32+
val enumType = fullType.toTypeEnum()
33+
val isPrimitive = !(
34+
if (fullType != "Entry") (isContainer ?: false)
35+
else true
36+
)
37+
38+
return SerializableTypeInfo(enumType, isPrimitive, fullType)
39+
}
40+
}
41+
}
42+
43+
@Serializable
44+
enum class Type {
45+
Map,
46+
Entry,
47+
Array,
48+
List,
49+
Custom
50+
}
51+
52+
fun String.toTypeEnum(): Type {
53+
return when (this) {
54+
"Map" -> Type.Map
55+
"Entry" -> Type.Entry
56+
"Array" -> Type.Array
57+
"List" -> Type.List
58+
else -> Type.Custom
59+
}
60+
}
61+
2362
@Serializable
2463
data class SerializedVariablesState(
25-
val type: String = "",
64+
val type: SerializableTypeInfo = SerializableTypeInfo(),
2665
val value: String? = null,
2766
val isContainer: Boolean = false,
2867
val stateId: String = ""

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/protocol.kt

+11-4
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,11 @@ package org.jetbrains.kotlinx.jupyter
22

33
import ch.qos.logback.classic.Level
44
import kotlinx.serialization.json.Json
5+
import kotlinx.serialization.json.JsonElement
6+
import kotlinx.serialization.json.JsonNull
57
import kotlinx.serialization.json.JsonObject
68
import kotlinx.serialization.json.encodeToJsonElement
9+
import kotlinx.serialization.json.jsonObject
710
import org.jetbrains.annotations.TestOnly
811
import org.jetbrains.kotlinx.jupyter.LoggingManagement.disableLogging
912
import org.jetbrains.kotlinx.jupyter.LoggingManagement.mainLoggerLevel
@@ -81,7 +84,6 @@ class OkResponseWithMessage(
8184
)
8285
)
8386
}
84-
8587
socket.send(
8688
makeReplyMessage(
8789
requestMsg,
@@ -91,7 +93,7 @@ class OkResponseWithMessage(
9193
"engine" to Json.encodeToJsonElement(requestMsg.data.header?.session),
9294
"status" to Json.encodeToJsonElement("ok"),
9395
"started" to Json.encodeToJsonElement(startedTime),
94-
"eval_metadata" to Json.encodeToJsonElement(metadata),
96+
"eval_metadata" to Json.encodeToJsonElement(metadata.convertToNullIfEmpty()),
9597
),
9698
content = ExecuteReply(
9799
MessageStatus.OK,
@@ -316,7 +318,7 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup
316318
if (data.isEmpty()) return sendWrapped(msg, makeReplyMessage(msg, MessageType.SERIALIZATION_REPLY))
317319
log.debug("Message data: $data")
318320
val messageContent = getVariablesDescriptorsFromJson(data)
319-
GlobalScope.launch(Dispatchers.Default) {
321+
connection.launchJob {
320322
repl.serializeVariables(
321323
messageContent.topLevelDescriptorName,
322324
messageContent.descriptorsState,
@@ -342,7 +344,7 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup
342344
}
343345
}
344346
is SerializationRequest -> {
345-
GlobalScope.launch(Dispatchers.Default) {
347+
connection.launchJob {
346348
if (content.topLevelDescriptorName.isNotEmpty()) {
347349
repl.serializeVariables(content.topLevelDescriptorName, content.descriptorsState, commID = content.commId, content.pathToDescriptor) { result ->
348350
sendWrapped(msg, makeReplyMessage(msg, MessageType.SERIALIZATION_REPLY, content = result))
@@ -539,3 +541,8 @@ fun JupyterConnection.evalWithIO(repl: ReplForJupyter, srcMessage: Message, body
539541
KernelStreams.setStreams(false, out, err)
540542
}
541543
}
544+
545+
fun EvaluatedSnippetMetadata?.convertToNullIfEmpty(): JsonElement? {
546+
val jsonNode = Json.encodeToJsonElement(this)
547+
return if (jsonNode is JsonNull || jsonNode?.jsonObject.isEmpty()) null else jsonNode
548+
}

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl.kt

+4-3
Original file line numberDiff line numberDiff line change
@@ -413,9 +413,10 @@ class ReplForJupyterImpl(
413413
val newImports: List<String>
414414
val oldDeclarations: MutableMap<String, Int> = mutableMapOf()
415415
oldDeclarations.putAll(internalEvaluator.getVariablesDeclarationInfo())
416+
val jupyterId = evalData.jupyterId
416417
val result = try {
417-
log.debug("Current cell id: ${evalData.jupyterId}")
418-
executor.execute(evalData.code, evalData.displayHandler, currentCellId = evalData.jupyterId - 1) { internalId, codeToExecute ->
418+
log.debug("Current cell id: $jupyterId")
419+
executor.execute(evalData.code, evalData.displayHandler, currentCellId = jupyterId - 1) { internalId, codeToExecute ->
419420
if (evalData.storeHistory) {
420421
cell = notebook.addCell(internalId, codeToExecute, EvalData(evalData))
421422
}
@@ -444,7 +445,7 @@ class ReplForJupyterImpl(
444445
// printVars()
445446
// printUsagesInfo(jupyterId, cellVariables[jupyterId - 1])
446447
val variablesCells: Map<String, Int> = notebook.variablesState.mapValues { internalEvaluator.findVariableCell(it.key) }
447-
val serializedData = variablesSerializer.serializeVariables(jupyterId - 1, notebook.variablesState, oldDeclarations, variablesCells, notebook.unchangedVariables())
448+
val serializedData = variablesSerializer.serializeVariables(jupyterId - 1, notebook.variablesState, oldDeclarations, variablesCells, notebook.unchangedVariables)
448449

449450
GlobalScope.launch(Dispatchers.Default) {
450451
variablesSerializer.tryValidateCache(jupyterId - 1, notebook.cellVariables)

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/serializationUtils.kt

+16-5
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import kotlinx.serialization.json.Json
55
import kotlinx.serialization.json.JsonObject
66
import kotlinx.serialization.json.decodeFromJsonElement
77
import org.jetbrains.kotlinx.jupyter.api.VariableState
8+
import org.jetbrains.kotlinx.jupyter.compiler.util.SerializableTypeInfo
89
import org.jetbrains.kotlinx.jupyter.compiler.util.SerializedVariablesState
910
import java.lang.reflect.Field
1011
import kotlin.contracts.ExperimentalContracts
@@ -32,14 +33,14 @@ enum class PropertiesType {
3233
}
3334

3435
@Serializable
35-
data class SerializedCommMessageContent(
36+
data class VariablesStateCommMessageContent(
3637
val topLevelDescriptorName: String,
3738
val descriptorsState: Map<String, SerializedVariablesState>,
3839
val pathToDescriptor: List<String> = emptyList()
3940
)
4041

41-
fun getVariablesDescriptorsFromJson(json: JsonObject): SerializedCommMessageContent {
42-
return Json.decodeFromJsonElement<SerializedCommMessageContent>(json)
42+
fun getVariablesDescriptorsFromJson(json: JsonObject): VariablesStateCommMessageContent {
43+
return Json.decodeFromJsonElement<VariablesStateCommMessageContent>(json)
4344
}
4445

4546
class ProcessedSerializedVarsState(
@@ -216,7 +217,12 @@ class VariablesSerializer(
216217
} else {
217218
""
218219
}
219-
val serializedVersion = SerializedVariablesState(simpleTypeName, stringedValue, true, varID)
220+
val serializedVersion = SerializedVariablesState(
221+
SerializableTypeInfo.makeFromSerializedVariablesState(simpleTypeName, true),
222+
stringedValue,
223+
true,
224+
varID
225+
)
220226
val descriptors = serializedVersion.fieldDescriptor
221227

222228
// only for set case
@@ -700,7 +706,12 @@ class VariablesSerializer(
700706
""
701707
}
702708

703-
val serializedVariablesState = SerializedVariablesState(type, getProperString(value), isContainer, finalID)
709+
val serializedVariablesState = SerializedVariablesState(
710+
SerializableTypeInfo.makeFromSerializedVariablesState(simpleTypeName, isContainer),
711+
getProperString(value),
712+
isContainer,
713+
finalID
714+
)
704715

705716
return ProcessedSerializedVarsState(serializedVariablesState, membersProperties?.toTypedArray())
706717
}

‎src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/ReplTests.kt

+19-20
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ import kotlin.test.assertEquals
2626
import kotlin.test.assertFails
2727
import kotlin.test.assertFailsWith
2828
import kotlin.test.assertFalse
29-
import kotlin.test.assertNotNull
3029
import kotlin.test.assertNotEquals
30+
import kotlin.test.assertNotNull
3131
import kotlin.test.fail
3232

3333
class ReplTests : AbstractSingleReplTest() {
@@ -834,14 +834,13 @@ class ReplVarsTest : AbstractSingleReplTest() {
834834
""".trimIndent(),
835835
jupyterId = 1
836836
)
837-
var state = repl.notebook.unchangedVariables()
838-
val res = eval(
837+
eval(
839838
"""
840839
l += 11111
841840
""".trimIndent(),
842841
jupyterId = 2
843842
).metadata.evaluatedVariablesState
844-
state = repl.notebook.unchangedVariables()
843+
val state: Set<String> = repl.notebook.unchangedVariables
845844
assertEquals(1, state.size)
846845
assertTrue(state.contains("m"))
847846
}
@@ -911,7 +910,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
911910
""".trimIndent(),
912911
jupyterId = 1
913912
)
914-
var state = repl.notebook.unchangedVariables()
913+
var state = repl.notebook.unchangedVariables
915914
assertEquals(3, state.size)
916915

917916
eval(
@@ -922,7 +921,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
922921
""".trimIndent(),
923922
jupyterId = 2
924923
)
925-
state = repl.notebook.unchangedVariables()
924+
state = repl.notebook.unchangedVariables
926925
assertEquals(0, state.size)
927926

928927
eval(
@@ -931,7 +930,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
931930
""".trimIndent(),
932931
jupyterId = 3
933932
)
934-
state = repl.notebook.unchangedVariables()
933+
state = repl.notebook.unchangedVariables
935934
assertEquals(1, state.size)
936935
}
937936
}
@@ -967,7 +966,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
967966
assertEquals(listOf(1, 2, 3, 4).toString().substring(1, actualContainer.value!!.length + 1), actualContainer.value)
968967

969968
val serializer = repl.variablesSerializer
970-
val newData = serializer.doIncrementalSerialization(0, "x", "data", actualContainer)
969+
serializer.doIncrementalSerialization(0, "x", "data", actualContainer)
971970
}
972971

973972
@Test
@@ -983,7 +982,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
983982
assertEquals(2, varsData.size)
984983
assertTrue(varsData.containsKey("x"))
985984
assertTrue(varsData.containsKey("f"))
986-
var unchangedVariables = repl.notebook.unchangedVariables()
985+
var unchangedVariables = repl.notebook.unchangedVariables
987986
assertTrue(unchangedVariables.isNotEmpty())
988987

989988
eval(
@@ -992,7 +991,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
992991
""".trimIndent(),
993992
jupyterId = 1
994993
)
995-
unchangedVariables = repl.notebook.unchangedVariables()
994+
unchangedVariables = repl.notebook.unchangedVariables
996995
assertTrue(unchangedVariables.contains("x"))
997996
assertTrue(unchangedVariables.contains("f"))
998997
}
@@ -1056,7 +1055,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
10561055

10571056
val serializer = repl.variablesSerializer
10581057

1059-
val newData = serializer.doIncrementalSerialization(0, "c", "i", descriptor["i"]!!)
1058+
serializer.doIncrementalSerialization(0, "c", "i", descriptor["i"]!!)
10601059
}
10611060

10621061
@Test
@@ -1345,7 +1344,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
13451344
""".trimIndent(),
13461345
jupyterId = 1
13471346
)
1348-
val state = repl.notebook.unchangedVariables()
1347+
val state = repl.notebook.unchangedVariables
13491348
val setOfCell = setOf("x", "f", "z")
13501349
assertTrue(state.isNotEmpty())
13511350
assertEquals(setOfCell, state)
@@ -1372,7 +1371,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
13721371
""".trimIndent(),
13731372
jupyterId = 1
13741373
)
1375-
var state = repl.notebook.unchangedVariables()
1374+
var state = repl.notebook.unchangedVariables
13761375
val setOfCell = setOf("x", "f", "z")
13771376
assertTrue(state.isNotEmpty())
13781377
assertEquals(setOfCell, state)
@@ -1396,9 +1395,9 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
13961395
""".trimIndent(),
13971396
jupyterId = 3
13981397
)
1399-
state = repl.notebook.unchangedVariables()
1400-
// assertTrue(state.isNotEmpty())
1401-
// assertEquals(state, setOfPrevCell)
1398+
state = repl.notebook.unchangedVariables
1399+
assertTrue(state.isEmpty())
1400+
// assertEquals(state, setOfPrevCell)
14021401

14031402
eval(
14041403
"""
@@ -1408,20 +1407,20 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
14081407
""".trimIndent(),
14091408
jupyterId = 4
14101409
)
1411-
state = repl.notebook.unchangedVariables()
1410+
state = repl.notebook.unchangedVariables
14121411
assertTrue(state.isEmpty())
14131412
}
14141413

14151414
@Test
14161415
fun testSerializationClearInfo() {
1417-
var res = eval(
1416+
eval(
14181417
"""
14191418
val x = listOf(1, 2, 3, 4)
14201419
""".trimIndent(),
14211420
jupyterId = 1
14221421
).metadata.evaluatedVariablesState
1423-
var state = repl.notebook.unchangedVariables()
1424-
res = eval(
1422+
repl.notebook.unchangedVariables
1423+
eval(
14251424
"""
14261425
val x = listOf(1, 2, 3, 4)
14271426
""".trimIndent(),

0 commit comments

Comments
 (0)