private static final Logger LOG = LoggerFactory.getLogger(OrcFile.class);

public static final String MAGIC = "ORC";

/**

public enum Version {

V_0_11("0.11", 0, 11),

V_0_12("0.12", 0, 12),

/**

UNSTABLE_PRE_2_0("UNSTABLE-PRE-2.0", 1, 9999),

/**

FUTURE("future", Integer.MAX_VALUE, Integer.MAX_VALUE);

public static final Version CURRENT = V_0_12;

private final String name;

private final int major;

private final int minor;

Version(String name, int major, int minor) {

this.name = name;

this.major = major;

this.minor = minor;

}

public static Version byName(String name) {

for(Version version: values()) {

if (version.name.equals(name)) {

return version;

}

}

throw new IllegalArgumentException("Unknown ORC version " + name);

}

/**

public String getName() {

return name;

}

/**

public int getMajor() {

return major;

}

/**

public int getMinor() {

return minor;

}

}

public enum WriterImplementation {

ORC_JAVA(0), // ORC Java writer

ORC_CPP(1), // ORC C++ writer

PRESTO(2), // Presto writer

SCRITCHLEY_GO(3), // Go writer from https://github.com/scritchley/orc

TRINO(4), // Trino writer

CUDF(5), // CUDF writer

UNKNOWN(Integer.MAX_VALUE);

private final int id;

WriterImplementation(int id) {

this.id = id;

}

public int getId() {

return id;

}

public static WriterImplementation from(int id) {

WriterImplementation[] values = values();

if (id >= 0 && id < values.length - 1) {

return values[id];

}

return UNKNOWN;

}

}

/**

public enum WriterVersion {

// Java ORC Writer

ORIGINAL(WriterImplementation.ORC_JAVA, 0),

HIVE_8732(WriterImplementation.ORC_JAVA, 1), /**

HIVE_4243(WriterImplementation.ORC_JAVA, 2), // use real column names from Hive tables

HIVE_12055(WriterImplementation.ORC_JAVA, 3), // vectorized writer

HIVE_13083(WriterImplementation.ORC_JAVA, 4), // decimals write present stream correctly

ORC_101(WriterImplementation.ORC_JAVA, 5), // bloom filters use utf8

ORC_135(WriterImplementation.ORC_JAVA, 6), // timestamp stats use utc

ORC_517(WriterImplementation.ORC_JAVA, 7), // decimal64 min/max are fixed

ORC_203(WriterImplementation.ORC_JAVA, 8), // trim long strings & record they were trimmed

ORC_14(WriterImplementation.ORC_JAVA, 9), // column encryption added

// C++ ORC Writer

ORC_CPP_ORIGINAL(WriterImplementation.ORC_CPP, 6),

// Presto Writer

PRESTO_ORIGINAL(WriterImplementation.PRESTO, 6),

// Scritchley Go Writer

SCRITCHLEY_GO_ORIGINAL(WriterImplementation.SCRITCHLEY_GO, 6),

// Trino Writer

TRINO_ORIGINAL(WriterImplementation.TRINO, 6),

// CUDF Writer

CUDF_ORIGINAL(WriterImplementation.CUDF, 6),

// Don't use any magic numbers here except for the below:

FUTURE(WriterImplementation.UNKNOWN, Integer.MAX_VALUE); // a version from a future writer

private final int id;

private final WriterImplementation writer;

public WriterImplementation getWriterImplementation() {

return writer;

}

public int getId() {

return id;

}

WriterVersion(WriterImplementation writer, int id) {

this.writer = writer;

this.id = id;

}

private static final WriterVersion[][] values =

new WriterVersion[WriterImplementation.values().length][];

static {

for(WriterVersion v: WriterVersion.values()) {

WriterImplementation writer = v.writer;

if (writer != WriterImplementation.UNKNOWN) {

if (values[writer.id] == null) {

values[writer.id] = new WriterVersion[WriterVersion.values().length];

}

if (values[writer.id][v.id] != null) {

throw new IllegalArgumentException("Duplicate WriterVersion id " + v);

}

values[writer.id][v.id] = v;

}

}

}

/**

public static WriterVersion from(WriterImplementation writer, int val) {

if (writer == WriterImplementation.UNKNOWN) {

return FUTURE;

}

if (writer != WriterImplementation.ORC_JAVA && val < 6) {

throw new IllegalArgumentException("ORC File with illegal version " +

val + " for writer " + writer);

}

WriterVersion[] versions = values[writer.id];

if (val < 0 || versions.length <= val) {

return FUTURE;

}

WriterVersion result = versions[val];

return result == null ? FUTURE : result;

}

/**

public boolean includes(WriterVersion fix) {

return writer != fix.writer || id >= fix.id;

}

}

/**

public static final WriterVersion CURRENT_WRITER = WriterVersion.ORC_14;

public enum EncodingStrategy {

SPEED, COMPRESSION

}

public enum CompressionStrategy {

SPEED, COMPRESSION

}

// unused

protected OrcFile() {}

public static class ReaderOptions {

private final Configuration conf;

private FileSystem filesystem;

private long maxLength = Long.MAX_VALUE;

private OrcTail orcTail;

private KeyProvider keyProvider;

// TODO: We can generalize FileMetadata interface. Make OrcTail implement FileMetadata interface

// and remove this class altogether. Both footer caching and llap caching just needs OrcTail.

// For now keeping this around to avoid complex surgery

private FileMetadata fileMetadata;

private boolean useUTCTimestamp;

private boolean useProlepticGregorian;

public ReaderOptions(Configuration conf) {

this.conf = conf;

this.useProlepticGregorian = OrcConf.PROLEPTIC_GREGORIAN.getBoolean(conf);

}

public ReaderOptions filesystem(FileSystem fs) {

this.filesystem = fs;

return this;

}

public ReaderOptions maxLength(long val) {

maxLength = val;

return this;

}

public ReaderOptions orcTail(OrcTail tail) {

this.orcTail = tail;

return this;

}

/**

public ReaderOptions setKeyProvider(KeyProvider provider) {

this.keyProvider = provider;

return this;

}

/**

public ReaderOptions convertToProlepticGregorian(boolean newValue) {

this.useProlepticGregorian = newValue;

return this;

}

public Configuration getConfiguration() {

return conf;

}

public FileSystem getFilesystem() {

return filesystem;

}

public long getMaxLength() {

return maxLength;

}

public OrcTail getOrcTail() {

return orcTail;

}

public KeyProvider getKeyProvider() {

return keyProvider;

}

/**

@Deprecated

public ReaderOptions fileMetadata(final FileMetadata metadata) {

fileMetadata = metadata;

return this;

}

public FileMetadata getFileMetadata() {

return fileMetadata;

}

public ReaderOptions useUTCTimestamp(boolean value) {

useUTCTimestamp = value;

return this;

}

public boolean getUseUTCTimestamp() {

return useUTCTimestamp;

}

public boolean getConvertToProlepticGregorian() {

return useProlepticGregorian;

}

}

public static ReaderOptions readerOptions(Configuration conf) {

return new ReaderOptions(conf);

}

public static Reader createReader(Path path,

ReaderOptions options) throws IOException {

return new ReaderImpl(path, options);

}

public interface WriterContext {

/**

Writer getWriter();

}

public interface WriterCallback {

/**

void preStripeWrite(WriterContext context) throws IOException;

/**

void preFooterWrite(WriterContext context) throws IOException;

}

public enum BloomFilterVersion {

// Include both the BLOOM_FILTER and BLOOM_FILTER_UTF8 streams to support

// both old and new readers.

ORIGINAL("original"),

// Only include the BLOOM_FILTER_UTF8 streams that consistently use UTF8.

// See ORC-101

UTF8("utf8");

private final String id;

BloomFilterVersion(String id) {

this.id = id;

}

@Override

public String toString() {

return id;

}

public static BloomFilterVersion fromString(String s) {

for (BloomFilterVersion version: values()) {

if (version.id.equals(s)) {

return version;

}

}

throw new IllegalArgumentException("Unknown BloomFilterVersion " + s);

}

}

public static class ZstdCompressOptions {

private int compressionZstdLevel;

private int compressionZstdWindowLog;

private int compressionZstdStrategy;

public int getCompressionZstdLevel() {

return compressionZstdLevel;

}

public void setCompressionZstdLevel(int compressionZstdLevel) {

this.compressionZstdLevel = compressionZstdLevel;

}

public int getCompressionZstdWindowLog() {

return compressionZstdWindowLog;

}

public void setCompressionZstdWindowLog(int compressionZstdWindowLog) {

this.compressionZstdWindowLog = compressionZstdWindowLog;

}

public int getCompressionZstdStrategy() {

return compressionZstdStrategy;

}

public void setCompressionZstdStrategy(int compressionZstdStrategy) {

this.compressionZstdStrategy = compressionZstdStrategy;

}

}

/**

public static class WriterOptions implements Cloneable {

private final Configuration configuration;

private FileSystem fileSystemValue = null;

private TypeDescription schema = null;

private long stripeSizeValue;

private long stripeRowCountValue;

private long blockSizeValue;

private boolean buildIndex;

private int rowIndexStrideValue;

private int bufferSizeValue;

private boolean enforceBufferSize = false;

private boolean blockPaddingValue;

private CompressionKind compressValue;

private MemoryManager memoryManagerValue;

private Version versionValue;

private WriterCallback callback;

private EncodingStrategy encodingStrategy;

private CompressionStrategy compressionStrategy;

private ZstdCompressOptions zstdCompressOptions;

private double paddingTolerance;

private String bloomFilterColumns;

private double bloomFilterFpp;

private BloomFilterVersion bloomFilterVersion;

private PhysicalWriter physicalWriter;

private WriterVersion writerVersion = CURRENT_WRITER;

private boolean useUTCTimestamp;

private boolean overwrite;

private boolean writeVariableLengthBlocks;

private HadoopShims shims;

private String directEncodingColumns;

private String encryption;

private String masks;

private KeyProvider provider;

private boolean useProlepticGregorian;

private Map<String, HadoopShims.KeyMetadata> keyOverrides = new HashMap<>();

protected WriterOptions(Properties tableProperties, Configuration conf) {

configuration = conf;

memoryManagerValue = getStaticMemoryManager(conf);

overwrite = OrcConf.OVERWRITE_OUTPUT_FILE.getBoolean(tableProperties, conf);

stripeSizeValue = OrcConf.STRIPE_SIZE.getLong(tableProperties, conf);

stripeRowCountValue = OrcConf.STRIPE_ROW_COUNT.getLong(tableProperties, conf);

blockSizeValue = OrcConf.BLOCK_SIZE.getLong(tableProperties, conf);

buildIndex = OrcConf.ENABLE_INDEXES.getBoolean(tableProperties, conf);

rowIndexStrideValue =

(int) OrcConf.ROW_INDEX_STRIDE.getLong(tableProperties, conf);

bufferSizeValue = (int) OrcConf.BUFFER_SIZE.getLong(tableProperties,

conf);

blockPaddingValue =

OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf);

compressValue =

CompressionKind.valueOf(OrcConf.COMPRESS.getString(tableProperties,

conf).toUpperCase());

enforceBufferSize = OrcConf.ENFORCE_COMPRESSION_BUFFER_SIZE.getBoolean(tableProperties, conf);

String versionName = OrcConf.WRITE_FORMAT.getString(tableProperties,

conf);

versionValue = Version.byName(versionName);

String enString = OrcConf.ENCODING_STRATEGY.getString(tableProperties,

conf);

encodingStrategy = EncodingStrategy.valueOf(enString);

String compString =

OrcConf.COMPRESSION_STRATEGY.getString(tableProperties, conf);

compressionStrategy = CompressionStrategy.valueOf(compString);

zstdCompressOptions = new ZstdCompressOptions();

zstdCompressOptions.setCompressionZstdLevel(

OrcConf.COMPRESSION_ZSTD_LEVEL.getInt(tableProperties, conf));

zstdCompressOptions.setCompressionZstdWindowLog(

OrcConf.COMPRESSION_ZSTD_WINDOWLOG.getInt(tableProperties, conf));

zstdCompressOptions.setCompressionZstdStrategy(

OrcConf.COMPRESSION_ZSTD_STRATEGY.getInt(tableProperties, conf));

paddingTolerance =

OrcConf.BLOCK_PADDING_TOLERANCE.getDouble(tableProperties, conf);

bloomFilterColumns = OrcConf.BLOOM_FILTER_COLUMNS.getString(tableProperties,

conf);

bloomFilterFpp = OrcConf.BLOOM_FILTER_FPP.getDouble(tableProperties,

conf);

bloomFilterVersion =

BloomFilterVersion.fromString(

OrcConf.BLOOM_FILTER_WRITE_VERSION.getString(tableProperties,

conf));

shims = HadoopShimsFactory.get();

writeVariableLengthBlocks =

OrcConf.WRITE_VARIABLE_LENGTH_BLOCKS.getBoolean(tableProperties,conf);

directEncodingColumns = OrcConf.DIRECT_ENCODING_COLUMNS.getString(

tableProperties, conf);

useProlepticGregorian = OrcConf.PROLEPTIC_GREGORIAN.getBoolean(conf);

}

/**

@Override

public WriterOptions clone() {

try {

return (WriterOptions) super.clone();

} catch (CloneNotSupportedException ex) {

throw new AssertionError("Expected super.clone() to work");

}

}

/**

public WriterOptions fileSystem(FileSystem value) {

fileSystemValue = value;

return this;

}

/**

public WriterOptions overwrite(boolean value) {

overwrite = value;

return this;

}

/**

public WriterOptions stripeSize(long value) {

stripeSizeValue = value;

return this;

}

/**

public WriterOptions blockSize(long value) {

blockSizeValue = value;

return this;

}

/**

public WriterOptions rowIndexStride(int value) {

rowIndexStrideValue = value;

if (rowIndexStrideValue <= 0) {

buildIndex = false;

}

return this;

}

/**

public WriterOptions buildIndex(boolean value) {

buildIndex = value;

if (!buildIndex) {

rowIndexStrideValue = 0;

}

return this;

}

/**

public WriterOptions bufferSize(int value) {

bufferSizeValue = value;

return this;

}

/**

public WriterOptions enforceBufferSize() {

enforceBufferSize = true;

return this;

}

/**

public WriterOptions blockPadding(boolean value) {

blockPaddingValue = value;

return this;

}

/**

public WriterOptions encodingStrategy(EncodingStrategy strategy) {

encodingStrategy = strategy;

return this;

}

/**

public WriterOptions paddingTolerance(double value) {

paddingTolerance = value;

return this;

}

/**

public WriterOptions bloomFilterColumns(String columns) {

bloomFilterColumns = columns;

return this;

}

/**

public WriterOptions bloomFilterFpp(double fpp) {

bloomFilterFpp = fpp;

return this;

}

/**

public WriterOptions compress(CompressionKind value) {

compressValue = value;

return this;

}

/**

public WriterOptions setSchema(TypeDescription schema) {

this.schema = schema;

return this;

}

/**

public WriterOptions version(Version value) {

versionValue = value;

return this;

}

/**

public WriterOptions callback(WriterCallback callback) {

this.callback = callback;

return this;

}

/**

@Deprecated

public WriterOptions bloomFilterVersion(BloomFilterVersion version) {

this.bloomFilterVersion = version;

return this;

}

/**

public WriterOptions physicalWriter(PhysicalWriter writer) {

this.physicalWriter = writer;

return this;

}

/**

public WriterOptions memory(MemoryManager value) {

memoryManagerValue = value;

return this;

}

/**

public WriterOptions writeVariableLengthBlocks(boolean value) {

writeVariableLengthBlocks = value;

return this;

}

/**

public WriterOptions setShims(HadoopShims value) {

this.shims = value;

return this;

}

/**

protected WriterOptions writerVersion(WriterVersion version) {

if (version == WriterVersion.FUTURE) {

throw new IllegalArgumentException("Can't write a future version.");

}

this.writerVersion = version;

return this;

}

/**

public WriterOptions useUTCTimestamp(boolean value) {

useUTCTimestamp = value;

return this;

}

/**

public WriterOptions directEncodingColumns(String value) {

directEncodingColumns = value;

return this;

}

/**

public WriterOptions encrypt(String value) {

encryption = value;

return this;

}

/**

public WriterOptions masks(String value) {

masks = value;

return this;

}

/**

public WriterOptions setKeyVersion(String keyName, int version,

EncryptionAlgorithm algorithm) {

HadoopShims.KeyMetadata meta = new HadoopShims.KeyMetadata(keyName,

version, algorithm);

keyOverrides.put(keyName, meta);

return this;

}

/**

public WriterOptions setKeyProvider(KeyProvider provider) {

this.provider = provider;

return this;

}

/**

public WriterOptions setProlepticGregorian(boolean newValue) {

this.useProlepticGregorian = newValue;

return this;

}

public KeyProvider getKeyProvider() {

return provider;

}

public boolean getBlockPadding() {

return blockPaddingValue;

}

public long getBlockSize() {

return blockSizeValue;

}

public String getBloomFilterColumns() {

return bloomFilterColumns;

}

public boolean getOverwrite() {

return overwrite;

}

public FileSystem getFileSystem() {

return fileSystemValue;

}

public Configuration getConfiguration() {

return configuration;

}

public TypeDescription getSchema() {

return schema;

}

public long getStripeSize() {

return stripeSizeValue;

}

public long getStripeRowCountValue() {

return stripeRowCountValue;

}

public CompressionKind getCompress() {

return compressValue;

}

public WriterCallback getCallback() {

return callback;

}

public Version getVersion() {

return versionValue;

}

public MemoryManager getMemoryManager() {

return memoryManagerValue;

}

public int getBufferSize() {

return bufferSizeValue;

}

public boolean isEnforceBufferSize() {

return enforceBufferSize;

}

public int getRowIndexStride() {

return rowIndexStrideValue;

}

public boolean isBuildIndex() {

return buildIndex;

}

public CompressionStrategy getCompressionStrategy() {

return compressionStrategy;

}

public EncodingStrategy getEncodingStrategy() {

return encodingStrategy;

}

public ZstdCompressOptions getZstdCompressOptions() {

return zstdCompressOptions;

}