From 44b750f8d3735489e66299955bc052b472c026a2 Mon Sep 17 00:00:00 2001 From: Ajay George Date: Mon, 8 Feb 2021 17:53:46 -0800 Subject: [PATCH 1/2] Working code checkpoint 1 --- .../drift/codec/internal/ProtocolWriter.java | 12 +++++ .../facebook/drift/codec/metadata/Any.java | 49 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 drift-codec/src/main/java/com/facebook/drift/codec/metadata/Any.java diff --git a/drift-codec/src/main/java/com/facebook/drift/codec/internal/ProtocolWriter.java b/drift-codec/src/main/java/com/facebook/drift/codec/internal/ProtocolWriter.java index ab78ec644..b84b1d77d 100644 --- a/drift-codec/src/main/java/com/facebook/drift/codec/internal/ProtocolWriter.java +++ b/drift-codec/src/main/java/com/facebook/drift/codec/internal/ProtocolWriter.java @@ -67,6 +67,18 @@ public void writeField(String name, short id, ThriftCodec codec, T value) protocol.writeFieldEnd(); } + public void writeRawField(String name, short id, ThriftCodec codec, Object value) + throws Exception + { + if (codec.isNull(value)) { + return; + } + + protocol.writeFieldBegin(new TField(name, codec.getType().getProtocolType().getType(), id)); + codec.write(value, protocol); + protocol.writeFieldEnd(); + } + public void writeBinaryField(String name, short id, ByteBuffer buf) throws TException { diff --git a/drift-codec/src/main/java/com/facebook/drift/codec/metadata/Any.java b/drift-codec/src/main/java/com/facebook/drift/codec/metadata/Any.java new file mode 100644 index 000000000..219bb0c87 --- /dev/null +++ b/drift-codec/src/main/java/com/facebook/drift/codec/metadata/Any.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2012 ${project.organization.name} + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.drift.codec.metadata; + +import com.facebook.drift.annotations.ThriftConstructor; +import com.facebook.drift.annotations.ThriftField; +import com.facebook.drift.annotations.ThriftStruct; + +@ThriftStruct +public class Any +{ + private final String id; + private final Object obj; + + @ThriftConstructor + public Any(@ThriftField(1) String id, @ThriftField(value = 2) T obj) + { + this.id = id; + this.obj = obj; + } + + public static Any pack(T obj, String id) + { + return new Any(id, obj); + } + + public Object getObj() + { + return obj; + } + + public String getId() + { + return id; + } +} From ad6be23b177015b3dcdd6445199f3f9b6482b6d2 Mon Sep 17 00:00:00 2001 From: Ajay George Date: Wed, 15 Dec 2021 18:05:34 -0800 Subject: [PATCH 2/2] Temp commit --- .../drift/annotations/ThriftField.java | 2 + .../drift/codec/ThriftCodecManager.java | 5 ++ .../codec/internal/builtin/AnyCodec.java | 72 +++++++++++++++++++ .../ReflectionThriftStructCodec.java | 14 +++- .../AbstractThriftMetadataBuilder.java | 27 +++++++ .../facebook/drift/codec/metadata/Any.java | 24 +++---- .../drift/codec/metadata/FieldMetadata.java | 15 ++++ .../drift/codec/metadata/ThriftCatalog.java | 14 +++- .../drift/codec/metadata/ThriftType.java | 1 + 9 files changed, 159 insertions(+), 15 deletions(-) create mode 100644 drift-codec/src/main/java/com/facebook/drift/codec/internal/builtin/AnyCodec.java diff --git a/drift-api/src/main/java/com/facebook/drift/annotations/ThriftField.java b/drift-api/src/main/java/com/facebook/drift/annotations/ThriftField.java index 810237e71..1a9f592c5 100644 --- a/drift-api/src/main/java/com/facebook/drift/annotations/ThriftField.java +++ b/drift-api/src/main/java/com/facebook/drift/annotations/ThriftField.java @@ -45,6 +45,8 @@ Recursiveness isRecursive() default Recursiveness.UNSPECIFIED; + Class serde() default Object.class; + ThriftIdlAnnotation[] idlAnnotations() default {}; enum Recursiveness diff --git a/drift-codec/src/main/java/com/facebook/drift/codec/ThriftCodecManager.java b/drift-codec/src/main/java/com/facebook/drift/codec/ThriftCodecManager.java index 2a13e7d05..59d5ee7f9 100644 --- a/drift-codec/src/main/java/com/facebook/drift/codec/ThriftCodecManager.java +++ b/drift-codec/src/main/java/com/facebook/drift/codec/ThriftCodecManager.java @@ -17,6 +17,7 @@ import com.facebook.drift.codec.internal.EnumThriftCodec; import com.facebook.drift.codec.internal.ThriftCodecFactory; +import com.facebook.drift.codec.internal.builtin.AnyCodec; import com.facebook.drift.codec.internal.builtin.BooleanArrayThriftCodec; import com.facebook.drift.codec.internal.builtin.BooleanThriftCodec; import com.facebook.drift.codec.internal.builtin.ByteBufferThriftCodec; @@ -41,6 +42,7 @@ import com.facebook.drift.codec.internal.builtin.VoidThriftCodec; import com.facebook.drift.codec.internal.coercion.CoercionThriftCodec; import com.facebook.drift.codec.internal.compiler.CompilerThriftCodecFactory; +import com.facebook.drift.codec.metadata.Any; import com.facebook.drift.codec.metadata.ReflectionHelper; import com.facebook.drift.codec.metadata.ThriftCatalog; import com.facebook.drift.codec.metadata.ThriftType; @@ -140,6 +142,9 @@ public ThriftCodec load(ThriftType type) switch (type.getProtocolType()) { case STRUCT: + if (type.getJavaType().equals(Any.class)) { + return new AnyCodec(); + } return factory.generateThriftTypeCodec(ThriftCodecManager.this, type.getStructMetadata()); case MAP: return new MapThriftCodec<>(type, getElementCodec(type.getKeyTypeReference()), getElementCodec(type.getValueTypeReference())); diff --git a/drift-codec/src/main/java/com/facebook/drift/codec/internal/builtin/AnyCodec.java b/drift-codec/src/main/java/com/facebook/drift/codec/internal/builtin/AnyCodec.java new file mode 100644 index 000000000..4b70d0510 --- /dev/null +++ b/drift-codec/src/main/java/com/facebook/drift/codec/internal/builtin/AnyCodec.java @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2012 ${project.organization.name} + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.drift.codec.internal.builtin; + +import com.facebook.drift.codec.ThriftCodec; +import com.facebook.drift.codec.internal.ProtocolReader; +import com.facebook.drift.codec.internal.ProtocolWriter; +import com.facebook.drift.codec.metadata.Any; +import com.facebook.drift.codec.metadata.ThriftType; +import com.facebook.drift.protocol.TProtocolReader; +import com.facebook.drift.protocol.TProtocolWriter; + +public class AnyCodec + implements ThriftCodec +{ + @Override + public ThriftType getType() + { + return ThriftType.ANY; + } + + @Override + public Any read(TProtocolReader protocol) + throws Exception + { + String id = null; + byte[] byteArr = new byte[]{}; + ProtocolReader protocolReader = new ProtocolReader(protocol); + protocolReader.readStructBegin(); + while (protocolReader.nextField()) { + if (protocolReader.getFieldId() == 1) { + id = (String) protocolReader.readField(new StringThriftCodec()); + } + /*else if (protocolReader.getFieldId() == 2) { + Class aClass = classResolver.apply(id); + ThriftCodec codec = new ThriftCodecManager(new CompilerThriftCodecFactory(true)).getCodec(aClass); + byteArr = (byte[]) protocolReader.readField(codec); + }*/ + } + protocolReader.readStructEnd(); + return new Any(id, byteArr); + } + + @Override + public void write(Any value, TProtocolWriter protocol) + throws Exception + { + ProtocolWriter writer = new ProtocolWriter(protocol); + writer.writeStructBegin("Any"); + ThriftCodec stringCodec = new StringThriftCodec(); + String id = value.getId(); + writer.writeField("id", (short) 1, stringCodec, id); + //Class aClass = classResolver.apply(id); + //ThriftCodec codec = new ThriftCodecManager(new CompilerThriftCodecFactory(true)).getCodec(aClass); + //Object cast = aClass.cast(value.getBytes()); + //writer.writeRawField("obj", (short) 2, codec, cast); + writer.writeStructEnd(); + } +} diff --git a/drift-codec/src/main/java/com/facebook/drift/codec/internal/reflection/ReflectionThriftStructCodec.java b/drift-codec/src/main/java/com/facebook/drift/codec/internal/reflection/ReflectionThriftStructCodec.java index ac6b96654..c2885d31e 100644 --- a/drift-codec/src/main/java/com/facebook/drift/codec/internal/reflection/ReflectionThriftStructCodec.java +++ b/drift-codec/src/main/java/com/facebook/drift/codec/internal/reflection/ReflectionThriftStructCodec.java @@ -20,6 +20,8 @@ import com.facebook.drift.codec.ThriftCodecManager; import com.facebook.drift.codec.internal.ProtocolReader; import com.facebook.drift.codec.internal.ProtocolWriter; +import com.facebook.drift.codec.internal.builtin.AnyCodec; +import com.facebook.drift.codec.metadata.Any; import com.facebook.drift.codec.metadata.FieldKind; import com.facebook.drift.codec.metadata.ThriftConstructorInjection; import com.facebook.drift.codec.metadata.ThriftFieldInjection; @@ -117,7 +119,13 @@ public void write(T instance, TProtocolWriter protocol) if (fieldValue != null) { @SuppressWarnings("unchecked") ThriftCodec codec = (ThriftCodec) fields.get(fieldMetadata.getId()); - writer.writeField(fieldMetadata.getName(), fieldMetadata.getId(), codec, fieldValue); + //Special handling + if (codec.getClass().equals(AnyCodec.class)) { + writer.writeField("any", fieldMetadata.getId(), codec, new Any("id", new byte[]{})); + } + else { + writer.writeField(fieldMetadata.getName(), fieldMetadata.getId(), codec, fieldValue); + } } } writer.writeStructEnd(); @@ -137,6 +145,10 @@ private T constructStruct(Map data) if (value == null) { value = metadata.getField(parameter.getId()).getThriftType().getNullValue(); } + if (value instanceof Any) { + //Special handling + value = null; + } parametersValues[parameter.getParameterIndex()] = value; } diff --git a/drift-codec/src/main/java/com/facebook/drift/codec/metadata/AbstractThriftMetadataBuilder.java b/drift-codec/src/main/java/com/facebook/drift/codec/metadata/AbstractThriftMetadataBuilder.java index 1b8586e92..0977ef1dd 100644 --- a/drift-codec/src/main/java/com/facebook/drift/codec/metadata/AbstractThriftMetadataBuilder.java +++ b/drift-codec/src/main/java/com/facebook/drift/codec/metadata/AbstractThriftMetadataBuilder.java @@ -505,6 +505,11 @@ protected final void normalizeThriftFields(ThriftCatalog catalog) field.setRequiredness(requiredness); } + Class serde = extractFieldSerde(fieldId, fields); + for (FieldMetadata field : fields) { + field.setSerde(serde); + } + // We need to do the isLegacyId check in two places. We've already done this // process for fields which had multiple `@ThriftField` annotations when we // assigned them all the same ID. It doesn't hurt to do it again. On the other @@ -633,6 +638,25 @@ protected final boolean extractFieldIsRecursiveReference(short fieldId, Collecti return isRecursiveReferences.iterator().next(); } + protected final Class extractFieldSerde(short fieldId, Collection fields) + { + Set serdeClasses = + fields.stream() + .map(FieldMetadata::getSerde) + .filter(Objects::nonNull) + .collect(toSet()); + + if (serdeClasses.size() > 1) { + metadataErrors.addError("Thrift class '%s' field '%s' has both isRecursiveReference=TRUE and isRecursiveReference=FALSE", structName, fieldId); + } + if (serdeClasses.isEmpty()) { + return Object.class; + } + else { + return serdeClasses.iterator().next(); + } + } + protected final boolean extractFieldIsLegacyId(short id, String fieldName, Collection fields) { Set isLegacyIds = fields.stream() @@ -711,6 +735,9 @@ protected final void verifyFieldType(short id, String name, Collection Any(@ThriftField(1) String id, @ThriftField(value = 2) T obj) + public Any(@ThriftField(1) String id, @ThriftField(2) byte[] bytes) { this.id = id; - this.obj = obj; - } - - public static Any pack(T obj, String id) - { - return new Any(id, obj); + this.bytes = Arrays.copyOf(bytes, bytes.length); } - public Object getObj() + public String getId() { - return obj; + return id; } - public String getId() + public byte[] getBytes() { - return id; + //TODO : Remove copying. Temporarily for satisyfing spotbugs + return Arrays.copyOf(bytes, bytes.length); } } diff --git a/drift-codec/src/main/java/com/facebook/drift/codec/metadata/FieldMetadata.java b/drift-codec/src/main/java/com/facebook/drift/codec/metadata/FieldMetadata.java index 1395d4223..16b4f3339 100644 --- a/drift-codec/src/main/java/com/facebook/drift/codec/metadata/FieldMetadata.java +++ b/drift-codec/src/main/java/com/facebook/drift/codec/metadata/FieldMetadata.java @@ -37,6 +37,8 @@ abstract class FieldMetadata private Boolean isRecursiveReference; private String name; private Requiredness requiredness; + //could be Class + private Class serde; private Map idlAnnotations; private final FieldKind type; @@ -55,6 +57,9 @@ protected FieldMetadata(ThriftField annotation, FieldKind type) name = annotation.name(); } requiredness = requireNonNull(annotation.requiredness()); + if (annotation.serde() != Object.class) { + serde = annotation.serde(); + } ImmutableMap.Builder annotationMapBuilder = ImmutableMap.builder(); for (ThriftIdlAnnotation idlAnnotation : annotation.idlAnnotations()) { @@ -187,4 +192,14 @@ public void setIsRecursiveReference(Boolean isRecursiveReference) { this.isRecursiveReference = isRecursiveReference; } + + public Class getSerde() + { + return serde; + } + + public void setSerde(Class serde) + { + this.serde = serde; + } } diff --git a/drift-codec/src/main/java/com/facebook/drift/codec/metadata/ThriftCatalog.java b/drift-codec/src/main/java/com/facebook/drift/codec/metadata/ThriftCatalog.java index ddbc0f7bd..b7dfe2352 100644 --- a/drift-codec/src/main/java/com/facebook/drift/codec/metadata/ThriftCatalog.java +++ b/drift-codec/src/main/java/com/facebook/drift/codec/metadata/ThriftCatalog.java @@ -55,6 +55,7 @@ import static com.facebook.drift.codec.metadata.ReflectionHelper.getMapValueType; import static com.facebook.drift.codec.metadata.ReflectionHelper.getOptionalType; import static com.facebook.drift.codec.metadata.ThriftEnumMetadataBuilder.thriftEnumMetadata; +import static com.facebook.drift.codec.metadata.ThriftType.ANY; import static com.facebook.drift.codec.metadata.ThriftType.BINARY; import static com.facebook.drift.codec.metadata.ThriftType.BOOL; import static com.facebook.drift.codec.metadata.ThriftType.BYTE; @@ -321,6 +322,9 @@ private ThriftType buildThriftTypeInternal(Type javaType) if (void.class.isAssignableFrom(rawType) || Void.class.isAssignableFrom(rawType)) { return VOID; } + if (Any.class.isAssignableFrom(rawType)) { + return ANY; + } if (isStructType(rawType)) { ThriftStructMetadata structMetadata = getThriftStructMetadata(javaType); // Unions are covered because a union looks like a struct with a single field. @@ -351,8 +355,16 @@ public ThriftTypeReference getFieldThriftTypeReference(FieldMetadata fieldMetada "Field normalization should have set a non-null value for isRecursiveReference"); } + Class serde = fieldMetadata.getSerde(); + + Type javaType = fieldMetadata.getJavaType(); + + if (!Object.class.equals(serde)) { + javaType = TypeToken.of(Any.class).getType(); + } + return getThriftTypeReference( - fieldMetadata.getJavaType(), + javaType, isRecursive ? Recursiveness.FORCED : Recursiveness.NOT_ALLOWED); } diff --git a/drift-codec/src/main/java/com/facebook/drift/codec/metadata/ThriftType.java b/drift-codec/src/main/java/com/facebook/drift/codec/metadata/ThriftType.java index 3baa19979..6dd225bd5 100644 --- a/drift-codec/src/main/java/com/facebook/drift/codec/metadata/ThriftType.java +++ b/drift-codec/src/main/java/com/facebook/drift/codec/metadata/ThriftType.java @@ -48,6 +48,7 @@ public class ThriftType public static final ThriftType STRING = new ThriftType(ThriftProtocolType.STRING, String.class); public static final ThriftType BINARY = new ThriftType(ThriftProtocolType.BINARY, ByteBuffer.class); public static final ThriftType VOID = new ThriftType(ThriftProtocolType.STRUCT, void.class); + public static final ThriftType ANY = new ThriftType(ThriftProtocolType.STRUCT, Any.class); public static final ThriftTypeReference BOOL_REF = new DefaultThriftTypeReference(BOOL); public static final ThriftTypeReference BYTE_REF = new DefaultThriftTypeReference(BYTE);