std::shared_ptr<compute::FunctionOptions> options) {

Expression::Call call;

call.function_name = std::move(function);

call.arguments = std::move(arguments);

call.options = std::move(options);

return Expression(std::move(call));

if (datum.is_scalar()) {

if (!datum.scalar()->is_valid) return "null[" + datum.type()->ToString() + "]";

switch (datum.type()->id()) {

case Type::STRING:

case Type::LARGE_STRING:

return '"' +

Escape(std::string_view(*datum.scalar_as<BaseBinaryScalar>().value)) + '"';

case Type::BINARY:

case Type::FIXED_SIZE_BINARY:

case Type::LARGE_BINARY:

return '"' + datum.scalar_as<BaseBinaryScalar>().value->ToHexString() + '"';

default:

break;

}

return datum.scalar()->ToString();

} else if (datum.is_array()) {

return "Array[" + datum.type()->ToString() + "]";

}

return datum.ToString();

if (auto lit = literal()) {

return PrintDatum(*lit);

}

if (auto ref = field_ref()) {

if (auto name = ref->name()) {

return *name;

}

if (auto path = ref->field_path()) {

return path->ToString();

}

return ref->ToString();

}

auto call = CallNotNull(*this);

auto binary = [&](std::string op) {

return "(" + call->arguments[0].ToString() + " " + op + " " +

call->arguments[1].ToString() + ")";

};

if (auto cmp = Comparison::Get(call->function_name)) {

return binary(Comparison::GetOp(*cmp));

}

constexpr std::string_view kleene = "_kleene";

if (EndsWith(call->function_name, kleene)) {

auto op = call->function_name.substr(0, call->function_name.size() - kleene.size());

return binary(std::move(op));

}

if (auto options = GetMakeStructOptions(*call)) {

std::string out = "{";

auto argument = call->arguments.begin();

for (const auto& field_name : options->field_names) {

out += field_name + "=" + argument++->ToString() + ", ";

}

out.resize(out.size() - 1);

out.back() = '}';

return out;

}

std::string out = call->function_name + "(";

for (const auto& arg : call->arguments) {

out += arg.ToString() + ", ";

}

if (call->options) {

out += call->options->ToString();

} else if (call->arguments.size()) {

out.resize(out.size() - 2);

}

out += ')';

return out;

*os << expr.ToString();

if (expr.IsBound()) {

*os << "[bound]";

}

if (Identical(*this, other)) return true;

if (impl_->index() != other.impl_->index()) {

return false;

}

if (auto lit = literal()) {

return lit->Equals(*other.literal());

}

if (auto ref = field_ref()) {

return ref->Equals(*other.field_ref());

}

auto call = CallNotNull(*this);

auto other_call = CallNotNull(other);

if (call->function_name != other_call->function_name ||

call->kernel != other_call->kernel) {

return false;

}

for (size_t i = 0; i < call->arguments.size(); ++i) {

if (!call->arguments[i].Equals(other_call->arguments[i])) {

return false;

}

}

if (call->options == other_call->options) return true;

if (call->options && other_call->options) {

return call->options->Equals(other_call->options);

}

return false;

if (auto lit = literal()) {

if (lit->is_scalar()) {

return lit->scalar()->hash();

}

return 0;

}

if (auto ref = field_ref()) {

return ref->hash();

}

return CallNotNull(*this)->hash;

if (type() == nullptr) return false;

if (const Call* call = this->call()) {

if (call->kernel == nullptr) return false;

for (const Expression& arg : call->arguments) {

if (!arg.IsBound()) return false;

}

}

return true;

if (auto lit = literal()) {

return lit->is_scalar();

}

if (field_ref()) return true;

auto call = CallNotNull(*this);

for (const Expression& arg : call->arguments) {

if (!arg.IsScalarExpression()) return false;

}

if (call->function) {

return call->function->kind() == compute::Function::SCALAR;

}

// this expression is not bound; make a best guess based on

// the default function registry

if (auto function = compute::GetFunctionRegistry()

->GetFunction(call->function_name)

.ValueOr(nullptr)) {

return function->kind() == compute::Function::SCALAR;

}

// unknown function or other error; conservatively return false

return false;

if (auto lit = literal()) {

if (lit->null_count() == lit->length()) {

return true;

}

}

return false;

DCHECK_NE(call.function, nullptr);

if (call.function->kind() == compute::Function::SCALAR) {

return static_cast<const compute::ScalarKernel*>(call.kernel)->null_handling;

}

return std::nullopt;

if (type() == nullptr) return true;

if (type()->id() != Type::BOOL) return true;

if (auto lit = literal()) {

if (lit->null_count() == lit->length()) {

return false;

}

if (lit->is_scalar()) {

return lit->scalar_as<BooleanScalar>().value;

}

return true;

}

if (field_ref()) return true;

auto call = CallNotNull(*this);

// invert(true_unless_null(x)) is always false or null by definition

// true_unless_null arises in simplification of inequalities below

if (call->function_name == "invert") {

if (auto nested_call = call->arguments[0].call()) {

if (nested_call->function_name == "true_unless_null") return false;

}

}

if (call->function_name == "and_kleene" || call->function_name == "and") {

for (const Expression& arg : call->arguments) {

if (!arg.IsSatisfiable()) return false;

}

}

return true;

compute::ExecContext* exec_context) {

DCHECK(std::all_of(call.arguments.begin(), call.arguments.end(),

[](const Expression& argument) { return argument.IsBound(); }));

std::vector<TypeHolder> types = GetTypes(call.arguments);

ARROW_ASSIGN_OR_RAISE(call.function, GetFunction(call, exec_context));

if (!insert_implicit_casts) {

ARROW_ASSIGN_OR_RAISE(call.kernel, call.function->DispatchExact(types));

} else {

ARROW_ASSIGN_OR_RAISE(call.kernel, call.function->DispatchBest(&types));

for (size_t i = 0; i < types.size(); ++i) {

if (types[i] == call.arguments[i].type()) continue;

if (const Datum* lit = call.arguments[i].literal()) {

ARROW_ASSIGN_OR_RAISE(Datum new_lit,

compute::Cast(*lit, types[i].GetSharedPtr()));

call.arguments[i] = literal(std::move(new_lit));

continue;

}

// construct an implicit cast Expression with which to replace this argument

Expression::Call implicit_cast;

implicit_cast.function_name = "cast";

implicit_cast.arguments = {std::move(call.arguments[i])};

// TODO(wesm): Use TypeHolder in options

implicit_cast.options = std::make_shared<compute::CastOptions>(

compute::CastOptions::Safe(types[i].GetSharedPtr()));

ARROW_ASSIGN_OR_RAISE(

call.arguments[i],

BindNonRecursive(std::move(implicit_cast),

/*insert_implicit_casts=*/false, exec_context));

}

}

compute::KernelContext kernel_context(exec_context, call.kernel);

if (call.kernel->init) {

const FunctionOptions* options =

call.options ? call.options.get() : call.function->default_options();

ARROW_ASSIGN_OR_RAISE(

call.kernel_state,

call.kernel->init(&kernel_context, {call.kernel, types, options}));

kernel_context.SetState(call.kernel_state.get());

}

ARROW_ASSIGN_OR_RAISE(

call.type, call.kernel->signature->out_type().Resolve(&kernel_context, types));

return Expression(std::move(call));

compute::ExecContext* exec_context) {

if (exec_context == nullptr) {

compute::ExecContext exec_context;

return BindImpl(std::move(expr), in, &exec_context);

}

if (expr.literal()) return expr;

if (const FieldRef* ref = expr.field_ref()) {

ARROW_ASSIGN_OR_RAISE(FieldPath path, ref->FindOne(in));

Expression::Parameter param = *expr.parameter();

param.indices.resize(path.indices().size());

std::copy(path.indices().begin(), path.indices().end(), param.indices.begin());

ARROW_ASSIGN_OR_RAISE(auto field, path.Get(in));

param.type = field->type();

return Expression{std::move(param)};

}

auto call = *CallNotNull(expr);

for (auto& argument : call.arguments) {

ARROW_ASSIGN_OR_RAISE(argument, BindImpl(std::move(argument), in, exec_context));

}

return BindNonRecursive(std::move(call),

/*insert_implicit_casts=*/true, exec_context);

Expression guarantee) {

ExecBatch out;

if (partial.kind() == Datum::RECORD_BATCH) {

const auto& partial_batch = *partial.record_batch();

out.guarantee = std::move(guarantee);

out.length = partial_batch.num_rows();

ARROW_ASSIGN_OR_RAISE(auto known_field_values,

ExtractKnownFieldValues(out.guarantee));

for (const auto& field : full_schema.fields()) {

auto field_ref = FieldRef(field->name());

// If we know what the value must be from the guarantee, prefer to use that value

// than the data from the record batch (if it exists at all -- probably it doesn't),

// because this way it will be a scalar.

auto known_field_value = known_field_values.map.find(field_ref);

if (known_field_value != known_field_values.map.end()) {

out.values.emplace_back(known_field_value->second);

continue;

}

ARROW_ASSIGN_OR_RAISE(auto column, field_ref.GetOneOrNone(partial_batch));

if (column) {

if (!column->type()->Equals(field->type())) {

// Referenced field was present but didn't have the expected type.

// This *should* be handled by readers, and will just be an error in the future.

ARROW_ASSIGN_OR_RAISE(

auto converted,

compute::Cast(column, field->type(), compute::CastOptions::Safe()));

column = converted.make_array();

}

out.values.emplace_back(std::move(column));

} else {

out.values.emplace_back(MakeNullScalar(field->type()));

}

}

return out;

}

// wasteful but useful for testing:

if (partial.type()->id() == Type::STRUCT) {

if (partial.is_array()) {

ARROW_ASSIGN_OR_RAISE(auto partial_batch,

RecordBatch::FromStructArray(partial.make_array()));

return MakeExecBatch(full_schema, partial_batch, std::move(guarantee));

}

if (partial.is_scalar()) {

ARROW_ASSIGN_OR_RAISE(auto partial_array,

MakeArrayFromScalar(*partial.scalar(), 1));

ARROW_ASSIGN_OR_RAISE(

auto out, MakeExecBatch(full_schema, partial_array, std::move(guarantee)));

for (Datum& value : out.values) {

if (value.is_scalar()) continue;

ARROW_ASSIGN_OR_RAISE(value, value.make_array()->GetScalar(0));

}

return out;

}

}

return Status::NotImplemented("MakeExecBatch from ", PrintDatum(partial));

const Datum& partial_input,

compute::ExecContext* exec_context) {

ARROW_ASSIGN_OR_RAISE(auto input, MakeExecBatch(full_schema, partial_input));

return ExecuteScalarExpression(expr, input, exec_context);

compute::ExecContext* exec_context) {

if (exec_context == nullptr) {

compute::ExecContext exec_context;

return ExecuteScalarExpression(expr, input, &exec_context);

}

if (!expr.IsBound()) {

return Status::Invalid("Cannot Execute unbound expression.");

}

if (!expr.IsScalarExpression()) {

return Status::Invalid(

"ExecuteScalarExpression cannot Execute non-scalar expression ", expr.ToString());

}

if (auto lit = expr.literal()) return *lit;

if (auto param = expr.parameter()) {

if (param->type.id() == Type::NA) {

return MakeNullScalar(null());

}

Datum field = input[param->indices[0]];

if (param->indices.size() > 1) {

std::vector<int> indices(param->indices.begin() + 1, param->indices.end());

compute::StructFieldOptions options(std::move(indices));

ARROW_ASSIGN_OR_RAISE(

field, compute::CallFunction("struct_field", {std::move(field)}, &options));

}

if (!field.type()->Equals(*param->type.type)) {

return Status::Invalid("Referenced field ", expr.ToString(), " was ",

field.type()->ToString(), " but should have been ",

param->type.ToString());

}

return field;

}

auto call = CallNotNull(expr);

std::vector<Datum> arguments(call->arguments.size());

bool all_scalar = true;

for (size_t i = 0; i < arguments.size(); ++i) {

ARROW_ASSIGN_OR_RAISE(

arguments[i], ExecuteScalarExpression(call->arguments[i], input, exec_context));

if (arguments[i].is_array()) {

all_scalar = false;

}

}

auto executor = compute::detail::KernelExecutor::MakeScalar();

compute::KernelContext kernel_context(exec_context, call->kernel);

kernel_context.SetState(call->kernel_state.get());

const Kernel* kernel = call->kernel;

std::vector<TypeHolder> types = GetTypes(arguments);

auto options = call->options.get();

RETURN_NOT_OK(executor->Init(&kernel_context, {kernel, types, options}));

compute::detail::DatumAccumulator listener;

RETURN_NOT_OK(executor->Execute(

ExecBatch(std::move(arguments), all_scalar ? 1 : input.length), &listener));

const auto out = executor->WrapResults(arguments, listener.values());

#ifndef NDEBUG

DCHECK_OK(executor->CheckResultType(out, call->function_name.c_str()));

#endif

return out;

DCHECK_EQ(call.arguments.size(), 2);

return {std::pair<const Expression&, const Expression&>{call.arguments[0],

call.arguments[1]},

std::pair<const Expression&, const Expression&>{call.arguments[1],

call.arguments[0]}};

if (expr.literal()) return {};

if (auto ref = expr.field_ref()) {

return {*ref};

}

std::vector<FieldRef> fields;

for (const Expression& arg : CallNotNull(expr)->arguments) {

auto argument_fields = FieldsInExpression(arg);

std::move(argument_fields.begin(), argument_fields.end(), std::back_inserter(fields));

}

return fields;

if (expr.literal()) return false;

if (expr.field_ref()) return true;

for (const Expression& arg : CallNotNull(expr)->arguments) {

if (ExpressionHasFieldRefs(arg)) return true;

}

return false;

if (!expr.IsBound()) {

return Status::Invalid("Cannot fold constants in unbound expression.");

}

return ModifyExpression(

std::move(expr), [](Expression expr) { return expr; },

[](Expression expr, ...) -> Result<Expression> {

auto call = CallNotNull(expr);

if (std::all_of(call->arguments.begin(), call->arguments.end(),

[](const Expression& argument) { return argument.literal(); })) {

// all arguments are literal; we can evaluate this subexpression *now*

static const ExecBatch ignored_input = ExecBatch({}, 1);

ARROW_ASSIGN_OR_RAISE(Datum constant,

ExecuteScalarExpression(expr, ignored_input));

return literal(std::move(constant));

}

// XXX the following should probably be in a registry of passes instead

// of inline

if (GetNullHandling(*call) == compute::NullHandling::INTERSECTION) {

// kernels which always produce intersected validity can be resolved

// to null *now* if any of their inputs is a null literal

if (!call->type.type) {

return Status::Invalid("Cannot fold constants for unbound expression ",

expr.ToString());

}

for (const Expression& argument : call->arguments) {

if (argument.IsNullLiteral()) {

if (argument.type()->Equals(*call->type.type)) {

return argument;

} else {

return literal(MakeNullScalar(call->type.GetSharedPtr()));

}

}

}

}

if (call->function_name == "and_kleene") {

for (auto args : ArgumentsAndFlippedArguments(*call)) {

// true and x == x

if (args.first == literal(true)) return args.second;

// false and x == false

if (args.first == literal(false)) return args.first;

// x and x == x

if (args.first == args.second) return args.first;

}

return expr;

}

if (call->function_name == "or_kleene") {

for (auto args : ArgumentsAndFlippedArguments(*call)) {

// false or x == x

if (args.first == literal(false)) return args.second;

// true or x == true

if (args.first == literal(true)) return args.first;

// x or x == x

if (args.first == args.second) return args.first;

}

return expr;

}

return expr;

});

const Expression& guaranteed_true_predicate) {

auto guarantee = guaranteed_true_predicate.call();

if (!guarantee || guarantee->function_name != "and_kleene") {

return {guaranteed_true_predicate};

}

return FlattenedAssociativeChain(guaranteed_true_predicate).fringe;

const Expression& guarantee) {

auto call = guarantee.call();

if (!call) return std::nullopt;

// search for an equality conditions between a field and a literal

if (call->function_name == "equal") {

auto ref = call->arguments[0].field_ref();

if (!ref) return std::nullopt;

auto lit = call->arguments[1].literal();

if (!lit) return std::nullopt;

return std::make_pair(*ref, *lit);

}

// ... or a known null field

if (call->function_name == "is_null") {

auto ref = call->arguments[0].field_ref();

if (!ref) return std::nullopt;

return std::make_pair(*ref, Datum(std::make_shared<NullScalar>()));

}

return std::nullopt;

KnownFieldValues* known_values) {

// filter out consumed conjunction members, leaving only unconsumed

*conjunction_members = arrow::internal::FilterVector(

std::move(*conjunction_members),

[known_values](const Expression& guarantee) -> bool {

if (auto known_value = ExtractOneFieldValue(guarantee)) {

known_values->map.insert(std::move(*known_value));

return false;

}

return true;

});

return Status::OK();

const Expression& guaranteed_true_predicate) {

KnownFieldValues known_values;

auto conjunction_members = GuaranteeConjunctionMembers(guaranteed_true_predicate);

RETURN_NOT_OK(ExtractKnownFieldValues(&conjunction_members, &known_values));

return known_values;

Expression expr) {

if (!expr.IsBound()) {

return Status::Invalid(

"ReplaceFieldsWithKnownValues called on an unbound Expression");

}

return ModifyExpression(

std::move(expr),

[&known_values](Expression expr) -> Result<Expression> {

if (auto ref = expr.field_ref()) {

auto it = known_values.map.find(*ref);

if (it != known_values.map.end()) {

Datum lit = it->second;

if (lit.type()->Equals(*expr.type())) return literal(std::move(lit));

// type mismatch, try casting the known value to the correct type

if (expr.type()->id() == Type::DICTIONARY &&

lit.type()->id() != Type::DICTIONARY) {

// the known value must be dictionary encoded

const auto& dict_type = checked_cast<const DictionaryType&>(*expr.type());

if (!lit.type()->Equals(dict_type.value_type())) {

ARROW_ASSIGN_OR_RAISE(lit, compute::Cast(lit, dict_type.value_type()));

}

if (lit.is_scalar()) {

ARROW_ASSIGN_OR_RAISE(auto dictionary,

MakeArrayFromScalar(*lit.scalar(), 1));

lit = Datum{DictionaryScalar::Make(MakeScalar<int32_t>(0),

std::move(dictionary))};

}

}

ARROW_ASSIGN_OR_RAISE(lit, compute::Cast(lit, expr.type()->GetSharedPtr()));

return literal(std::move(lit));

}

}

return expr;

},

[](Expression expr, ...) { return expr; });

static std::unordered_set<std::string> binary_associative_commutative{

"and", "or", "and_kleene", "or_kleene", "xor",

"multiply", "add", "multiply_checked", "add_checked"};

auto it = binary_associative_commutative.find(call.function_name);

return it != binary_associative_commutative.end();

compute::ExecContext* exec_context) {

// ARROW-18334: due to reordering of arguments, the call may have

// inconsistent argument types. For example, the call's kernel may

// correspond to `timestamp + duration` but the arguments happen to

// be `duration, timestamp`. The addition itself is still commutative,

// but the mismatch in declared argument types is potentially problematic

// if we ever start using the Expression::Call::kernel field more than

// we do currently. Check and rebind if necessary.

//

// The more correct fix for this problem is to ensure that all kernels of

// functions which are commutative be commutative as well, which would

// obviate rebinding like this. In the context of ARROW-18334, this

// would require rewriting KernelSignature so that a single kernel can

// handle both `timestamp + duration` and `duration + timestamp`.

if (call.kernel->signature->MatchesInputs(GetTypes(call.arguments))) {

return Expression(std::move(call));

}

return BindNonRecursive(std::move(call), /*insert_implicit_casts=*/false, exec_context);

if (!expr.IsBound()) {

return Status::Invalid("Cannot canonicalize an unbound expression.");

}

if (exec_context == nullptr) {

compute::ExecContext exec_context;

return Canonicalize(std::move(expr), &exec_context);

}

// If potentially reconstructing more deeply than a call's immediate arguments

// (for example, when reorganizing an associative chain), add expressions to this set to

// avoid unnecessary work

struct {

std::unordered_set<Expression, Expression::Hash> set_;

bool operator()(const Expression& expr) const {

return set_.find(expr) != set_.end();

}

void Add(std::vector<Expression> exprs) {

std::move(exprs.begin(), exprs.end(), std::inserter(set_, set_.end()));

}

} AlreadyCanonicalized;

return ModifyExpression(

std::move(expr),

[&AlreadyCanonicalized, exec_context](Expression expr) -> Result<Expression> {

auto call = expr.call();

if (!call) return expr;

if (AlreadyCanonicalized(expr)) return expr;

if (IsBinaryAssociativeCommutative(*call)) {

struct {

int Priority(const Expression& operand) const {

// order literals first, starting with nulls

if (operand.IsNullLiteral()) return 0;

if (operand.literal()) return 1;

return 2;

}

bool operator()(const Expression& l, const Expression& r) const {

return Priority(l) < Priority(r);

}

} CanonicalOrdering;

FlattenedAssociativeChain chain(expr);

if (chain.was_left_folded &&

std::is_sorted(chain.fringe.begin(), chain.fringe.end(),

CanonicalOrdering)) {

// fast path for expressions which happen to have arrived in an

// already-canonical form

AlreadyCanonicalized.Add(std::move(chain.exprs));

return expr;

}

std::stable_sort(chain.fringe.begin(), chain.fringe.end(), CanonicalOrdering);

// fold the chain back up

Expression folded = std::move(chain.fringe.front());

for (auto it = chain.fringe.begin() + 1; it != chain.fringe.end(); ++it) {

auto canonicalized_call = *call;

canonicalized_call.arguments = {std::move(folded), std::move(*it)};

ARROW_ASSIGN_OR_RAISE(

folded,

HandleInconsistentTypes(std::move(canonicalized_call), exec_context));

AlreadyCanonicalized.Add({expr});

}

return folded;

}

if (auto cmp = Comparison::Get(call->function_name)) {

if (call->arguments[0].literal() && !call->arguments[1].literal()) {

// ensure that literals are on comparisons' RHS

auto flipped_call = *call;

std::swap(flipped_call.arguments[0], flipped_call.arguments[1]);

flipped_call.function_name =

Comparison::GetName(Comparison::GetFlipped(*cmp));

return BindNonRecursive(flipped_call,

/*insert_implicit_casts=*/false, exec_context);

}

}

return expr;

},

[](Expression expr, ...) { return expr; });

// The inequality type

Comparison::type cmp;

// The LHS of the inequality

const FieldRef& target;

// The RHS of the inequality

const Datum& bound;

// Whether target can be null

bool nullable;

// Extract an Inequality if possible, derived from "less",

// "greater", "less_equal", and "greater_equal" expressions,

// possibly disjuncted with an "is_null" Expression.

// cmp(a, 2)

// cmp(a, 2) or is_null(a)

static std::optional<Inequality> ExtractOne(const Expression& guarantee) {

auto call = guarantee.call();

if (!call) return std::nullopt;

if (call->function_name == "or_kleene") {

// expect the LHS to be a usable field inequality

auto out = ExtractOneFromComparison(call->arguments[0]);

if (!out) return std::nullopt;

// expect the RHS to be an is_null expression

auto call_rhs = call->arguments[1].call();

if (!call_rhs) return std::nullopt;

if (call_rhs->function_name != "is_null") return std::nullopt;