From 19175c7b8cea9c4ec74e1d0977e8fc157f1f63ab Mon Sep 17 00:00:00 2001 From: martin <2710952048@qq.com> Date: Fri, 21 Feb 2025 14:27:36 +0800 Subject: [PATCH] support tryEval and staticInvoke(readSidePadding function) --- .../org/apache/spark/sql/util/ModifyUtil.scala | 17 +++++++++++++++-- .../org/apache/spark/sql/util/ModifyUtil.scala | 17 +++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/omnioperator/omniop-spark-extension/spark-extension-shims/spark34-modify/src/main/scala/org/apache/spark/sql/util/ModifyUtil.scala b/omnioperator/omniop-spark-extension/spark-extension-shims/spark34-modify/src/main/scala/org/apache/spark/sql/util/ModifyUtil.scala index ba1cbda83..562869920 100644 --- a/omnioperator/omniop-spark-extension/spark-extension-shims/spark34-modify/src/main/scala/org/apache/spark/sql/util/ModifyUtil.scala +++ b/omnioperator/omniop-spark-extension/spark-extension-shims/spark34-modify/src/main/scala/org/apache/spark/sql/util/ModifyUtil.scala @@ -22,7 +22,8 @@ import com.huawei.boostkit.spark.expression.OmniExpressionAdaptor.{JsonObjectExt import com.huawei.boostkit.spark.util.ModifyUtilAdaptor import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSessionExtensions -import org.apache.spark.sql.catalyst.expressions.{BloomFilterMightContain, ExprId, Expression} +import org.apache.spark.sql.catalyst.expressions.{BloomFilterMightContain, ExprId, Expression, TryEval} +import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke import org.apache.spark.sql.catalyst.optimizer.{CombineJoinedAggregates, MergeSubqueryFilters} import org.apache.spark.sql.execution.adaptive.QueryStageExec import org.apache.spark.sql.execution.{BroadcastExchangeExecProxy, ColumnarBloomFilterSubquery, ColumnarToRowExec, OmniColumnarToRowExec, SparkPlan} @@ -60,7 +61,19 @@ object ModifyUtil extends Logging { .put("isNull", bfAddress == 0L) .put("dataType", 2) .put("value", bfAddress) - + case tryEval: TryEval => + func(tryEval.child, exprsIndexMap, returnDatatype) + case staticInvoke: StaticInvoke if staticInvoke.functionName == "readSidePadding" => + { + val arg0 = staticInvoke.arguments(0) + val arg1 = staticInvoke.arguments(1) + new JsonObject().put("exprType", "FUNCTION") + .put("function_name", "StaticInvokeCharReadPadding") + .addOmniExpJsonType("returnType", staticInvoke.dataType) + .put("arguments", + new JsonArray().put(func(arg0, exprsIndexMap, arg0.dataType)) + .put(func(arg1, exprsIndexMap, arg1.dataType))) + } case _ => null } diff --git a/omnioperator/omniop-spark-extension/spark-extension-shims/spark35-modify/src/main/scala/org/apache/spark/sql/util/ModifyUtil.scala b/omnioperator/omniop-spark-extension/spark-extension-shims/spark35-modify/src/main/scala/org/apache/spark/sql/util/ModifyUtil.scala index ba1cbda83..562869920 100644 --- a/omnioperator/omniop-spark-extension/spark-extension-shims/spark35-modify/src/main/scala/org/apache/spark/sql/util/ModifyUtil.scala +++ b/omnioperator/omniop-spark-extension/spark-extension-shims/spark35-modify/src/main/scala/org/apache/spark/sql/util/ModifyUtil.scala @@ -22,7 +22,8 @@ import com.huawei.boostkit.spark.expression.OmniExpressionAdaptor.{JsonObjectExt import com.huawei.boostkit.spark.util.ModifyUtilAdaptor import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSessionExtensions -import org.apache.spark.sql.catalyst.expressions.{BloomFilterMightContain, ExprId, Expression} +import org.apache.spark.sql.catalyst.expressions.{BloomFilterMightContain, ExprId, Expression, TryEval} +import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke import org.apache.spark.sql.catalyst.optimizer.{CombineJoinedAggregates, MergeSubqueryFilters} import org.apache.spark.sql.execution.adaptive.QueryStageExec import org.apache.spark.sql.execution.{BroadcastExchangeExecProxy, ColumnarBloomFilterSubquery, ColumnarToRowExec, OmniColumnarToRowExec, SparkPlan} @@ -60,7 +61,19 @@ object ModifyUtil extends Logging { .put("isNull", bfAddress == 0L) .put("dataType", 2) .put("value", bfAddress) - + case tryEval: TryEval => + func(tryEval.child, exprsIndexMap, returnDatatype) + case staticInvoke: StaticInvoke if staticInvoke.functionName == "readSidePadding" => + { + val arg0 = staticInvoke.arguments(0) + val arg1 = staticInvoke.arguments(1) + new JsonObject().put("exprType", "FUNCTION") + .put("function_name", "StaticInvokeCharReadPadding") + .addOmniExpJsonType("returnType", staticInvoke.dataType) + .put("arguments", + new JsonArray().put(func(arg0, exprsIndexMap, arg0.dataType)) + .put(func(arg1, exprsIndexMap, arg1.dataType))) + } case _ => null } -- Gitee