forked from apache/arrow
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_nodes.cc
More file actions
95 lines (84 loc) · 3.33 KB
/
Copy pathtest_nodes.cc
File metadata and controls
95 lines (84 loc) · 3.33 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "arrow/compute/exec/test_nodes.h"
#include <deque>
#include <iostream>
#include <mutex>
#include <string>
#include <vector>
#include "arrow/api.h"
#include "arrow/compute/exec/exec_plan.h"
#include "arrow/compute/exec/util.h"
#include "arrow/io/interfaces.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/iterator.h"
namespace arrow {
using internal::checked_cast;
namespace compute {
AsyncGenerator<std::optional<ExecBatch>> MakeDelayedGen(
Iterator<std::optional<ExecBatch>> src, std::string label, double delay_sec,
bool noisy) {
struct DelayedIoGenState {
DelayedIoGenState(Iterator<std::optional<ExecBatch>> batch_it, double delay_sec,
std::string label, bool noisy)
: batch_it(std::move(batch_it)),
delay_sec(delay_sec),
label(std::move(label)),
noisy(noisy) {}
std::optional<ExecBatch> Next() {
Result<std::optional<ExecBatch>> opt_batch_res = batch_it.Next();
if (!opt_batch_res.ok()) {
return std::nullopt;
}
std::optional<ExecBatch> opt_batch = opt_batch_res.ValueOrDie();
if (!opt_batch) {
return std::nullopt;
}
if (noisy) {
std::cout << label + ": asking for batch(" + std::to_string(index) + ")\n";
}
SleepFor(delay_sec);
++index;
return *opt_batch;
}
Iterator<std::optional<ExecBatch>> batch_it;
double delay_sec;
std::string label;
bool noisy;
std::size_t index = 0;
};
auto state = std::make_shared<DelayedIoGenState>(std::move(src), delay_sec,
std::move(label), noisy);
return [state]() {
return DeferNotOk(::arrow::io::default_io_context().executor()->Submit(
[state]() { return state->Next(); }));
};
}
AsyncGenerator<std::optional<ExecBatch>> MakeDelayedGen(
AsyncGenerator<std::optional<ExecBatch>> src, std::string label, double delay_sec,
bool noisy) {
return MakeDelayedGen(MakeGeneratorIterator(src), label, delay_sec, noisy);
}
AsyncGenerator<std::optional<ExecBatch>> MakeDelayedGen(BatchesWithSchema src,
std::string label,
double delay_sec, bool noisy) {
std::vector<std::optional<ExecBatch>> opt_batches = ::arrow::internal::MapVector(
[](ExecBatch batch) { return std::make_optional(std::move(batch)); }, src.batches);
return MakeDelayedGen(MakeVectorIterator(opt_batches), label, delay_sec, noisy);
}
} // namespace compute
} // namespace arrow