Skip to content

Commit f074bd4

Browse files
mandreanmandrean
committed
fix(java): fix async meta-shared layer jit bootstrap for ObjectStreamSerializer
1 parent 4728a0f commit f074bd4

3 files changed

Lines changed: 182 additions & 25 deletions

File tree

‎java/fory-core/src/main/java/org/apache/fory/builder/CodecUtils.java‎

Lines changed: 32 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ public class CodecUtils {
4040
// Cache key includes configHash to distinguish between xlang and non-xlang modes
4141
private static final ConcurrentHashMap<Tuple3<String, Class<?>, Integer>, Class>
4242
graalvmSerializers = new ConcurrentHashMap<>();
43+
// Generated layer serializers need the original layer metadata to bootstrap their delegate
44+
// synchronously in the constructor.
45+
private static final ConcurrentHashMap<Class<?>, MetaSharedLayerCodecContext>
46+
metaSharedLayerCodecContexts = new ConcurrentHashMap<>();
4347

4448
// TODO(chaokunyang) how to uninstall org.apache.fory.codegen/builder classes for graalvm build
4549
// time
@@ -78,16 +82,24 @@ public static <T> Class<? extends Serializer<T>> loadOrGenMetaSharedCodecClass(
7882
public static <T> Class<? extends Serializer<T>> loadOrGenMetaSharedLayerCodecClass(
7983
Class<T> cls, Fory fory, TypeDef layerTypeDef, Class<?> layerMarkerClass) {
8084
Preconditions.checkNotNull(fory);
81-
return loadSerializer(
82-
"loadOrGenMetaSharedLayerCodecClass",
83-
cls,
84-
fory,
85-
() ->
86-
loadOrGenCodecClass(
87-
cls,
88-
fory,
89-
new MetaSharedLayerCodecBuilder(
90-
TypeRef.of(cls), fory, layerTypeDef, layerMarkerClass)));
85+
Class<? extends Serializer<T>> serializerClass =
86+
loadSerializer(
87+
"loadOrGenMetaSharedLayerCodecClass",
88+
cls,
89+
fory,
90+
() ->
91+
loadOrGenCodecClass(
92+
cls,
93+
fory,
94+
new MetaSharedLayerCodecBuilder(
95+
TypeRef.of(cls), fory, layerTypeDef, layerMarkerClass)));
96+
metaSharedLayerCodecContexts.putIfAbsent(
97+
serializerClass, new MetaSharedLayerCodecContext(layerTypeDef, layerMarkerClass));
98+
return serializerClass;
99+
}
100+
101+
static MetaSharedLayerCodecContext getMetaSharedLayerCodecContext(Class<?> serializerClass) {
102+
return metaSharedLayerCodecContexts.get(serializerClass);
91103
}
92104

93105
@SuppressWarnings("unchecked")
@@ -171,4 +183,14 @@ private static <T> Class<? extends Serializer<T>> loadSerializer(
171183
throw new RuntimeException(e);
172184
}
173185
}
186+
187+
static final class MetaSharedLayerCodecContext {
188+
final TypeDef layerTypeDef;
189+
final Class<?> layerMarkerClass;
190+
191+
MetaSharedLayerCodecContext(TypeDef layerTypeDef, Class<?> layerMarkerClass) {
192+
this.layerTypeDef = layerTypeDef;
193+
this.layerMarkerClass = layerMarkerClass;
194+
}
195+
}
174196
}

‎java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedLayerCodecBuilder.java‎

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,8 @@
3232
import org.apache.fory.memory.MemoryBuffer;
3333
import org.apache.fory.meta.TypeDef;
3434
import org.apache.fory.reflect.TypeRef;
35-
import org.apache.fory.serializer.CodegenSerializer;
3635
import org.apache.fory.serializer.MetaSharedLayerSerializer;
3736
import org.apache.fory.serializer.MetaSharedLayerSerializerBase;
38-
import org.apache.fory.serializer.Serializers;
3937
import org.apache.fory.type.Descriptor;
4038
import org.apache.fory.type.DescriptorGrouper;
4139
import org.apache.fory.util.ExceptionUtils;
@@ -135,19 +133,17 @@ public static MetaSharedLayerSerializerBase setCodegenSerializer(
135133
if (GraalvmSupport.isGraalRuntime()) {
136134
return (MetaSharedLayerSerializerBase) typeResolver(fory, r -> r.getSerializer(s.getType()));
137135
}
138-
// This method hold jit lock, so create jit serializer async to avoid block serialization.
139-
// Use MetaSharedLayerSerializer as fallback since it's compatible with
140-
// MetaSharedLayerSerializerBase
141-
Class serializerClass =
142-
fory.getJITContext()
143-
.registerSerializerJITCallback(
144-
() -> MetaSharedLayerSerializer.class,
145-
() -> CodegenSerializer.loadCodegenSerializer(fory, s.getType()),
146-
c ->
147-
s.serializer =
148-
(MetaSharedLayerSerializerBase)
149-
Serializers.newSerializer(fory, s.getType(), c));
150-
return (MetaSharedLayerSerializerBase) Serializers.newSerializer(fory, cls, serializerClass);
136+
// Layer serializers don't have a generic no-arg/newSerializer construction path. The
137+
// outer ObjectStreamSerializer JIT step already resolved the layer TypeDef and marker, so
138+
// generated serializers look up that cached context here and bootstrap the interpreter
139+
// delegate synchronously in their constructor.
140+
CodecUtils.MetaSharedLayerCodecContext context =
141+
CodecUtils.getMetaSharedLayerCodecContext(s.getClass());
142+
Preconditions.checkNotNull(
143+
context, "Missing layer codec context for generated serializer " + s.getClass());
144+
s.serializer =
145+
new MetaSharedLayerSerializer(fory, cls, context.layerTypeDef, context.layerMarkerClass);
146+
return s.serializer;
151147
}
152148

153149
@Override

‎java/fory-core/src/test/java/org/apache/fory/serializer/ObjectStreamSerializerTest.java‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,18 @@
3939
import java.util.LinkedList;
4040
import java.util.List;
4141
import java.util.Map;
42+
import java.util.TreeMap;
43+
import java.util.TreeSet;
4244
import java.util.Vector;
4345
import java.util.concurrent.ConcurrentHashMap;
4446
import lombok.EqualsAndHashCode;
4547
import org.apache.fory.Fory;
4648
import org.apache.fory.ForyTestBase;
49+
import org.apache.fory.builder.Generated;
4750
import org.apache.fory.config.CompatibleMode;
4851
import org.apache.fory.config.Language;
4952
import org.apache.fory.memory.MemoryBuffer;
53+
import org.apache.fory.reflect.ReflectionUtils;
5054
import org.apache.fory.util.Preconditions;
5155
import org.testng.Assert;
5256
import org.testng.annotations.DataProvider;
@@ -1136,6 +1140,101 @@ public void testNestedObjectSerialization(CompatibleMode compatible) {
11361140
assertEquals(result.nestedList.get(1).nestedValue, "list2");
11371141
}
11381142

1143+
public static class AsyncTreeSetSubclass extends TreeSet<String> {
1144+
public AsyncTreeSetSubclass() {}
1145+
}
1146+
1147+
public static class AsyncTreeMapSubclass extends TreeMap<String, String> {
1148+
public AsyncTreeMapSubclass() {}
1149+
}
1150+
1151+
@EqualsAndHashCode
1152+
public static class AsyncLayerJitContainer implements Serializable {
1153+
private String name;
1154+
private AsyncTreeSetSubclass values;
1155+
private AsyncTreeMapSubclass attributes;
1156+
1157+
public AsyncLayerJitContainer() {}
1158+
1159+
public AsyncLayerJitContainer(
1160+
String name, AsyncTreeSetSubclass values, AsyncTreeMapSubclass attributes) {
1161+
this.name = name;
1162+
this.values = values;
1163+
this.attributes = attributes;
1164+
}
1165+
1166+
private void writeObject(ObjectOutputStream s) throws IOException {
1167+
s.defaultWriteObject();
1168+
}
1169+
1170+
private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException {
1171+
s.defaultReadObject();
1172+
}
1173+
}
1174+
1175+
@Test(timeOut = 60000)
1176+
public void testAsyncCompilationNestedTreeCollectionsCompatibleMode() throws InterruptedException {
1177+
Fory fory = newCompatibleAsyncObjectStreamFory(true);
1178+
fory.registerSerializer(
1179+
AsyncLayerJitContainer.class, new ObjectStreamSerializer(fory, AsyncLayerJitContainer.class));
1180+
fory.registerSerializer(
1181+
AsyncTreeSetSubclass.class, new ObjectStreamSerializer(fory, AsyncTreeSetSubclass.class));
1182+
fory.registerSerializer(
1183+
AsyncTreeMapSubclass.class, new ObjectStreamSerializer(fory, AsyncTreeMapSubclass.class));
1184+
1185+
AsyncTreeSetSubclass values = new AsyncTreeSetSubclass();
1186+
values.add("one");
1187+
values.add("two");
1188+
AsyncTreeMapSubclass attributes = new AsyncTreeMapSubclass();
1189+
attributes.put("alpha", "A");
1190+
attributes.put("beta", "B");
1191+
AsyncLayerJitContainer obj = new AsyncLayerJitContainer("container", values, attributes);
1192+
1193+
serDeCheckSerializer(fory, obj, "ObjectStreamSerializer");
1194+
1195+
waitForGeneratedLayerSerializer(fory, AsyncLayerJitContainer.class);
1196+
waitForGeneratedLayerSerializer(fory, AsyncTreeSetSubclass.class);
1197+
waitForGeneratedLayerSerializer(fory, AsyncTreeMapSubclass.class);
1198+
1199+
serDeCheckSerializer(fory, obj, "ObjectStreamSerializer");
1200+
}
1201+
1202+
@Test(timeOut = 60000)
1203+
public void testAsyncCompilationTreeSetSubclassObjectStreamSerializer() throws InterruptedException {
1204+
Fory fory = newCompatibleAsyncObjectStreamFory(true);
1205+
fory.registerSerializer(
1206+
AsyncTreeSetSubclass.class, new ObjectStreamSerializer(fory, AsyncTreeSetSubclass.class));
1207+
1208+
AsyncTreeSetSubclass values = new AsyncTreeSetSubclass();
1209+
values.add("one");
1210+
values.add("two");
1211+
1212+
serDeCheckSerializer(fory, values, "ObjectStreamSerializer");
1213+
waitForGeneratedLayerSerializer(fory, AsyncTreeSetSubclass.class);
1214+
serDeCheckSerializer(fory, values, "ObjectStreamSerializer");
1215+
}
1216+
1217+
@Test
1218+
public void testTreeCollectionsStillWorkWithoutAsyncCompilation() {
1219+
Fory fory = newCompatibleAsyncObjectStreamFory(false);
1220+
fory.registerSerializer(
1221+
AsyncLayerJitContainer.class, new ObjectStreamSerializer(fory, AsyncLayerJitContainer.class));
1222+
fory.registerSerializer(
1223+
AsyncTreeSetSubclass.class, new ObjectStreamSerializer(fory, AsyncTreeSetSubclass.class));
1224+
fory.registerSerializer(
1225+
AsyncTreeMapSubclass.class, new ObjectStreamSerializer(fory, AsyncTreeMapSubclass.class));
1226+
1227+
AsyncTreeSetSubclass values = new AsyncTreeSetSubclass();
1228+
values.add("one");
1229+
values.add("two");
1230+
AsyncTreeMapSubclass attributes = new AsyncTreeMapSubclass();
1231+
attributes.put("alpha", "A");
1232+
attributes.put("beta", "B");
1233+
1234+
serDeCheckSerializer(
1235+
fory, new AsyncLayerJitContainer("container", values, attributes), "ObjectStreamSerializer");
1236+
}
1237+
11391238
// ==================== Circular Reference in Custom Serialization ====================
11401239

11411240
/** Class with potential circular reference. */
@@ -1279,4 +1378,44 @@ public void testAllPrimitiveTypes(CompatibleMode compatible) {
12791378
assertEquals(result.charVal, 'A');
12801379
assertEquals(result.boolVal, true);
12811380
}
1381+
1382+
private Fory newCompatibleAsyncObjectStreamFory(boolean asyncCompilation) {
1383+
return Fory.builder()
1384+
.withLanguage(Language.JAVA)
1385+
.requireClassRegistration(false)
1386+
.withRefTracking(true)
1387+
.withCodegen(true)
1388+
.withCompatibleMode(CompatibleMode.COMPATIBLE)
1389+
.withAsyncCompilation(asyncCompilation)
1390+
.build();
1391+
}
1392+
1393+
private void waitForGeneratedLayerSerializer(Fory fory, Class<?> type) throws InterruptedException {
1394+
long deadline = System.currentTimeMillis() + 30_000;
1395+
while (System.currentTimeMillis() < deadline) {
1396+
if (hasGeneratedLayerSerializer(fory, type)) {
1397+
return;
1398+
}
1399+
Thread.sleep(10);
1400+
}
1401+
Assert.fail("Timed out waiting for generated layer serializer for " + type.getName());
1402+
}
1403+
1404+
private boolean hasGeneratedLayerSerializer(Fory fory, Class<?> type) {
1405+
Serializer<?> serializer = fory.getTypeResolver().getSerializer(type);
1406+
if (!(serializer instanceof ObjectStreamSerializer)) {
1407+
return false;
1408+
}
1409+
Object[] slotsInfos = (Object[]) ReflectionUtils.getObjectFieldValue(serializer, "slotsInfos");
1410+
if (slotsInfos.length == 0) {
1411+
return false;
1412+
}
1413+
for (Object slotsInfo : slotsInfos) {
1414+
Object slotsSerializer = ReflectionUtils.getObjectFieldValue(slotsInfo, "slotsSerializer");
1415+
if (!(slotsSerializer instanceof Generated)) {
1416+
return false;
1417+
}
1418+
}
1419+
return true;
1420+
}
12821421
}

0 commit comments

Comments
 (0)