Support UDF in planner

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java

@@ -27,6 +27,7 @@
 import com.apple.foundationdb.record.metadata.JoinedRecordType;
 import com.apple.foundationdb.record.metadata.MetaDataException;
 import com.apple.foundationdb.record.metadata.RecordType;
+import com.apple.foundationdb.record.metadata.SerializableFunction;
 import com.apple.foundationdb.record.metadata.SyntheticRecordType;
 import com.apple.foundationdb.record.metadata.UnnestedRecordType;
 import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
@@ -48,6 +49,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
@@ -83,6 +85,8 @@ public class RecordMetaData implements RecordMetaDataProvider {
     @Nonnull
     private final Map<Object, SyntheticRecordType<?>> recordTypeKeyToSyntheticTypeMap;
     @Nonnull
+    private final Set<SerializableFunction> serializableFunctions;
+    @Nonnull
     private final Map<String, Index> indexes;
     @Nonnull
     private final Map<String, Index> universalIndexes;
@@ -112,6 +116,7 @@ protected RecordMetaData(@Nonnull RecordMetaData orig) {
                 Collections.unmodifiableMap(orig.indexes),
                 Collections.unmodifiableMap(orig.universalIndexes),
                 Collections.unmodifiableList(orig.formerIndexes),
+                Collections.unmodifiableSet(orig.serializableFunctions),
                 orig.splitLongRecords,
                 orig.storeRecordVersions,
                 orig.version,
@@ -131,6 +136,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor,
                              @Nonnull Map<String, Index> indexes,
                              @Nonnull Map<String, Index> universalIndexes,
                              @Nonnull List<FormerIndex> formerIndexes,
+                             @Nonnull Set<SerializableFunction> serializableFunctions,
                              boolean splitLongRecords,
                              boolean storeRecordVersions,
                              int version,
@@ -147,6 +153,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor,
         this.indexes = indexes;
         this.universalIndexes = universalIndexes;
         this.formerIndexes = formerIndexes;
+        this.serializableFunctions = serializableFunctions;
         this.splitLongRecords = splitLongRecords;
         this.storeRecordVersions = storeRecordVersions;
         this.version = version;
@@ -342,6 +349,11 @@ public List<FormerIndex> getFormerIndexes() {
         return formerIndexes;
     }
 
+    @Nonnull
+    public Collection<SerializableFunction> getAllSerializableFunctions() {
+        return serializableFunctions;
+    }
+
     public boolean isSplitLongRecords() {
         return splitLongRecords;
     }
@@ -692,7 +704,7 @@ public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor
             builder.addFormerIndexes(formerIndex.toProto());
         }
 
-        // Add in the final options.
+        builder.addAllSerializableFunction(serializableFunctions.stream().map(SerializableFunction::toProto).collect(Collectors.toList()));
         builder.setSplitLongRecords(splitLongRecords);
         builder.setStoreRecordVersions(storeRecordVersions);
         builder.setVersion(version);

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java

@@ -33,6 +33,7 @@
 import com.apple.foundationdb.record.metadata.RecordType;
 import com.apple.foundationdb.record.metadata.RecordTypeBuilder;
 import com.apple.foundationdb.record.metadata.RecordTypeIndexesBuilder;
+import com.apple.foundationdb.record.metadata.SerializableFunction;
 import com.apple.foundationdb.record.metadata.SyntheticRecordType;
 import com.apple.foundationdb.record.metadata.SyntheticRecordTypeBuilder;
 import com.apple.foundationdb.record.metadata.UnnestedRecordTypeBuilder;
@@ -42,6 +43,7 @@
 import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry;
 import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl;
 import com.apple.foundationdb.record.provider.foundationdb.MetaDataProtoEditor;
+import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry;
 import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
@@ -110,6 +112,8 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider {
     @Nonnull
     private final Map<String, SyntheticRecordTypeBuilder<?>> syntheticRecordTypes;
     @Nonnull
+    private final Set<SerializableFunction> serializableFunctions;
+    @Nonnull
     private final Map<String, Index> indexes;
     @Nonnull
     private final Map<String, Index> universalIndexes;
@@ -144,6 +148,7 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider {
         indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance();
         evolutionValidator = MetaDataEvolutionValidator.getDefaultInstance();
         syntheticRecordTypes = new HashMap<>();
+        serializableFunctions = new HashSet<>();
     }
 
     private void processSchemaOptions(boolean processExtensionOptions) {
@@ -222,6 +227,14 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa
                 typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey()));
             }
         }
+        PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE,
+                PlanHashable.CURRENT_FOR_CONTINUATION);
+        for (RecordMetaDataProto.SerializableFunction serializableFunction: metaDataProto.getSerializableFunctionList()) {
+            SerializableFunction func = SerializableFunction.fromProto(serializationContext, serializableFunction);
+            if (func != null) {
+                serializableFunctions.add(func);
+            }
+        }
         if (metaDataProto.hasSplitLongRecords()) {
             splitLongRecords = metaDataProto.getSplitLongRecords();
         }
@@ -1179,6 +1192,14 @@ public void addFormerIndex(@Nonnull FormerIndex formerIndex) {
         formerIndexes.add(formerIndex);
     }
 
+    public void addSerializableFunction(@Nonnull SerializableFunction serializableFunction) {
+        serializableFunctions.add(serializableFunction);
+    }
+
+    public void addSerializableFunctions(@Nonnull Iterable<? extends SerializableFunction> functions) {
+        functions.forEach(this.serializableFunctions::add);
+    }
+
     public boolean isSplitLongRecords() {
         return splitLongRecords;
     }
@@ -1420,7 +1441,7 @@ public RecordMetaData build(boolean validate) {
         Map<Object, SyntheticRecordType<?>> recordTypeKeyToSyntheticRecordTypeMap = Maps.newHashMapWithExpectedSize(syntheticRecordTypes.size());
         RecordMetaData metaData = new RecordMetaData(recordsDescriptor, getUnionDescriptor(), unionFields,
                 builtRecordTypes, builtSyntheticRecordTypes, recordTypeKeyToSyntheticRecordTypeMap,
-                indexes, universalIndexes, formerIndexes,
+                indexes, universalIndexes, formerIndexes, serializableFunctions,
                 splitLongRecords, storeRecordVersions, version, subspaceKeyCounter, usesSubspaceKeyCounter, recordCountKey, localFileDescriptor != null);
         for (RecordTypeBuilder recordTypeBuilder : recordTypes.values()) {
             KeyExpression primaryKey = recordTypeBuilder.getPrimaryKey();

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java

@@ -22,7 +22,7 @@
 
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression;
 import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression;
@@ -392,21 +392,21 @@ public static class Evaluated {
          * Values used in index keys in place of missing fields.
          */
         public enum NullStandin {
-            NULL(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE), // Missing field here skips uniqueness checks.
-            NULL_UNIQUE(RecordMetaDataProto.Field.NullInterpretation.UNIQUE), // Missing field here like ordinary value, but null, for uniqueness.
-            NOT_NULL(RecordMetaDataProto.Field.NullInterpretation.NOT_NULL); // Missing field has type's ordinary default value.
+            NULL(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE), // Missing field here skips uniqueness checks.
+            NULL_UNIQUE(RecordKeyExpressionProto.Field.NullInterpretation.UNIQUE), // Missing field here like ordinary value, but null, for uniqueness.
+            NOT_NULL(RecordKeyExpressionProto.Field.NullInterpretation.NOT_NULL); // Missing field has type's ordinary default value.
 
-            private RecordMetaDataProto.Field.NullInterpretation proto;
+            private RecordKeyExpressionProto.Field.NullInterpretation proto;
 
-            NullStandin(RecordMetaDataProto.Field.NullInterpretation nullInterpretation) {
+            NullStandin(RecordKeyExpressionProto.Field.NullInterpretation nullInterpretation) {
                 proto = nullInterpretation;
             }
 
-            public RecordMetaDataProto.Field.NullInterpretation toProto() {
+            public RecordKeyExpressionProto.Field.NullInterpretation toProto() {
                 return proto;
             }
 
-            public static NullStandin valueOf(RecordMetaDataProto.Field.NullInterpretation nullInterpretation) throws KeyExpression.DeserializationException {
+            public static NullStandin valueOf(RecordKeyExpressionProto.Field.NullInterpretation nullInterpretation) throws KeyExpression.DeserializationException {
                 switch (nullInterpretation) {
                     case NOT_UNIQUE:
                         return NULL;

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SerializableFunction.java

@@ -0,0 +1,42 @@
+/*
+ * SerializableFunction.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb.record.metadata;
+
+import com.apple.foundationdb.record.PlanSerializationContext;
+import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.query.plan.cascades.MacroFunction;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+public interface SerializableFunction {
+    @Nonnull
+    RecordMetaDataProto.SerializableFunction toProto();
+
+    @Nullable
+    static SerializableFunction fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final RecordMetaDataProto.SerializableFunction proto) {
+        if (proto.hasMacroFunction()) {
+            return MacroFunction.fromProto(serializationContext, proto.getMacroFunction());
+        } else {
+            return null;
+        }
+    }
+}

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java

@@ -23,7 +23,7 @@
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.record.ObjectPlanHash;
 import com.apple.foundationdb.record.PlanHashable;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
 import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
@@ -63,7 +63,7 @@ private DimensionsKeyExpression(@Nonnull final KeyExpression wholeKey,
         this.dimensionsSize = dimensionsSize;
     }
 
-    DimensionsKeyExpression(@Nonnull final RecordMetaDataProto.Dimensions dimensions) throws DeserializationException {
+    DimensionsKeyExpression(@Nonnull final RecordKeyExpressionProto.Dimensions dimensions) throws DeserializationException {
         this(KeyExpression.fromProto(dimensions.getWholeKey()), dimensions.getPrefixSize(), dimensions.getDimensionsSize());
     }
 
@@ -114,8 +114,8 @@ public boolean needsCopyingToPartialRecord() {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.Dimensions toProto() throws SerializationException {
-        final RecordMetaDataProto.Dimensions.Builder builder = RecordMetaDataProto.Dimensions.newBuilder();
+    public RecordKeyExpressionProto.Dimensions toProto() throws SerializationException {
+        final RecordKeyExpressionProto.Dimensions.Builder builder = RecordKeyExpressionProto.Dimensions.newBuilder();
         builder.setWholeKey(getWholeKey().toKeyExpression());
         builder.setPrefixSize(prefixSize);
         builder.setDimensionsSize(dimensionsSize);
@@ -124,8 +124,8 @@ public RecordMetaDataProto.Dimensions toProto() throws SerializationException {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.KeyExpression toKeyExpression() {
-        return RecordMetaDataProto.KeyExpression.newBuilder().setDimensions(toProto()).build();
+    public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
+        return RecordKeyExpressionProto.KeyExpression.newBuilder().setDimensions(toProto()).build();
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java

@@ -23,7 +23,7 @@
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.record.ObjectPlanHash;
 import com.apple.foundationdb.record.PlanHashable;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
 import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
@@ -45,8 +45,8 @@ public class EmptyKeyExpression extends BaseKeyExpression implements KeyExpressi
     private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Empty-Key-Expression");
 
     public static final EmptyKeyExpression EMPTY = new EmptyKeyExpression();
-    public static final RecordMetaDataProto.KeyExpression EMPTY_PROTO =
-            RecordMetaDataProto.KeyExpression.newBuilder().setEmpty(EMPTY.toProto()).build();
+    public static final RecordKeyExpressionProto.KeyExpression EMPTY_PROTO =
+            RecordKeyExpressionProto.KeyExpression.newBuilder().setEmpty(EMPTY.toProto()).build();
 
     private EmptyKeyExpression() {
         // nothing to initialize
@@ -75,8 +75,8 @@ public int getColumnSize() {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.Empty toProto() throws SerializationException {
-        return RecordMetaDataProto.Empty.getDefaultInstance();
+    public RecordKeyExpressionProto.Empty toProto() throws SerializationException {
+        return RecordKeyExpressionProto.Empty.getDefaultInstance();
     }
 
     @Nonnull
@@ -87,7 +87,7 @@ public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpr
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.KeyExpression toKeyExpression() {
+    public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
         return EMPTY_PROTO;
     }
 

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java

@@ -24,7 +24,7 @@
 import com.apple.foundationdb.record.ObjectPlanHash;
 import com.apple.foundationdb.record.PlanHashable;
 import com.apple.foundationdb.record.RecordCoreException;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
@@ -71,7 +71,7 @@ public FieldKeyExpression(@Nonnull String fieldName, @Nonnull FanType fanType, @
         this.nullStandin = nullStandin;
     }
 
-    public FieldKeyExpression(@Nonnull RecordMetaDataProto.Field field) throws DeserializationException {
+    public FieldKeyExpression(@Nonnull RecordKeyExpressionProto.Field field) throws DeserializationException {
         if (!field.hasFieldName()) {
             throw new DeserializationException("Serialized Field is missing field name");
         }
@@ -193,8 +193,8 @@ public int getColumnSize() {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.Field toProto() throws SerializationException {
-        return RecordMetaDataProto.Field.newBuilder()
+    public RecordKeyExpressionProto.Field toProto() throws SerializationException {
+        return RecordKeyExpressionProto.Field.newBuilder()
                 .setFieldName(fieldName)
                 .setFanType(fanType.toProto())
                 .setNullInterpretation(nullStandin.toProto())
@@ -203,8 +203,8 @@ public RecordMetaDataProto.Field toProto() throws SerializationException {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.KeyExpression toKeyExpression() {
-        return RecordMetaDataProto.KeyExpression.newBuilder().setField(toProto()).build();
+    public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
+        return RecordKeyExpressionProto.KeyExpression.newBuilder().setField(toProto()).build();
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java

@@ -25,7 +25,7 @@
 import com.apple.foundationdb.record.PlanHashable;
 import com.apple.foundationdb.record.RecordCoreArgumentException;
 import com.apple.foundationdb.record.RecordCoreException;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
@@ -232,7 +232,7 @@ public boolean equalsAtomic(AtomKeyExpression other) {
      *   implementation, or the arguments provided are not suitable for the function.
      */
     @Nonnull
-    public static FunctionKeyExpression fromProto(RecordMetaDataProto.Function function) throws DeserializationException {
+    public static FunctionKeyExpression fromProto(RecordKeyExpressionProto.Function function) throws DeserializationException {
         try {
             return create(function.getName(), KeyExpression.fromProto(function.getArguments()));
         } catch (RecordCoreException e) {
@@ -242,17 +242,17 @@ public static FunctionKeyExpression fromProto(RecordMetaDataProto.Function funct
 
     @Nonnull
     @Override
-    public final RecordMetaDataProto.Function toProto() throws SerializationException {
-        RecordMetaDataProto.Function.Builder builder = RecordMetaDataProto.Function.newBuilder()
+    public final RecordKeyExpressionProto.Function toProto() throws SerializationException {
+        RecordKeyExpressionProto.Function.Builder builder = RecordKeyExpressionProto.Function.newBuilder()
                 .setName(getName());
         builder.setArguments(getArguments().toKeyExpression());
         return builder.build();
     }
 
     @Nonnull
     @Override
-    public final RecordMetaDataProto.KeyExpression toKeyExpression() {
-        return RecordMetaDataProto.KeyExpression.newBuilder().setFunction(toProto()).build();
+    public final RecordKeyExpressionProto.KeyExpression toKeyExpression() {
+        return RecordKeyExpressionProto.KeyExpression.newBuilder().setFunction(toProto()).build();
     }
 
     @Nonnull