-
Notifications
You must be signed in to change notification settings - Fork 507
Expand file tree
/
Copy pathPhysicalWriter.java
More file actions
153 lines (133 loc) · 4.79 KB
/
PhysicalWriter.java
File metadata and controls
153 lines (133 loc) · 4.79 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.orc;
import org.apache.orc.impl.StreamName;
import org.apache.orc.impl.writer.StreamOptions;
import org.apache.orc.impl.writer.WriterEncryptionVariant;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* This interface separates the physical layout of ORC files from the higher
* level details.
* <p>
* This API is limited to being used by LLAP.
* @since 1.3.0
*/
public interface PhysicalWriter {
/**
* The target of an output stream.
*/
interface OutputReceiver {
/**
* Output the given buffer to the final destination
*
* @param buffer the buffer to output
*/
void output(ByteBuffer buffer) throws IOException;
/**
* Suppress this stream from being written to the stripe.
*/
void suppress();
}
/**
* Writes the header of the file, which consists of the magic "ORC" bytes.
*/
void writeHeader() throws IOException;
/**
* Create an OutputReceiver for the given name.
* @param name the name of the stream
*/
OutputReceiver createDataStream(StreamName name) throws IOException;
/**
* Write an index in the given stream name.
* @param name the name of the stream
* @param index the bloom filter to write
*/
void writeIndex(StreamName name,
OrcProto.RowIndex.Builder index) throws IOException;
/**
* Write a bloom filter index in the given stream name.
* @param name the name of the stream
* @param bloom the bloom filter to write
*/
void writeBloomFilter(StreamName name,
OrcProto.BloomFilterIndex.Builder bloom) throws IOException;
/**
* Flushes the data in all the streams, spills them to disk, write out stripe
* footer.
* @param footer Stripe footer to be updated with relevant data and written out.
* @param dirEntry File metadata entry for the stripe, to be updated with
* relevant data.
*/
void finalizeStripe(OrcProto.StripeFooter.Builder footer,
OrcProto.StripeInformation.Builder dirEntry) throws IOException;
/**
* Write a stripe or file statistics to the file.
* @param name the name of the stream
* @param statistics the statistics to write
*/
void writeStatistics(StreamName name,
OrcProto.ColumnStatistics.Builder statistics
) throws IOException;
/**
* Writes out the file metadata.
* @param builder Metadata builder to finalize and write.
*/
void writeFileMetadata(OrcProto.Metadata.Builder builder) throws IOException;
/**
* Writes out the file footer.
* @param builder Footer builder to finalize and write.
*/
void writeFileFooter(OrcProto.Footer.Builder builder) throws IOException;
/**
* Writes out the postscript (including the size byte if needed).
* @param builder Postscript builder to finalize and write.
*/
long writePostScript(OrcProto.PostScript.Builder builder) throws IOException;
/**
* Closes the writer.
*/
void close() throws IOException;
/**
* Flushes the writer so that readers can see the preceding postscripts.
*/
void flush() throws IOException;
/**
* Appends raw stripe data (e.g. for file merger).
* @param stripe Stripe data buffer.
* @param dirEntry File metadata entry for the stripe, to be updated with
* relevant data.
*/
void appendRawStripe(ByteBuffer stripe,
OrcProto.StripeInformation.Builder dirEntry
) throws IOException;
/**
* Get the number of bytes for a file in a given column.
* @param column column from which to get file size
* @param variant the encryption variant to check
* @return number of bytes for the given column
*/
long getFileBytes(int column, WriterEncryptionVariant variant);
/**
* Get the unencrypted stream options for this file. This class needs the
* stream options to write the indexes and footers.
*
* Additionally, the LLAP CacheWriter wants to disable the generic compression.
*/
StreamOptions getStreamOptions();
}