org.apache.iceberg.spark.data.vectorized

Class IcebergArrowColumnVector

java.lang.Object

org.apache.spark.sql.vectorized.ColumnVector

org.apache.iceberg.spark.data.vectorized.IcebergArrowColumnVector

All Implemented Interfaces:

java.lang.AutoCloseable

Direct Known Subclasses:

ColumnVectorWithFilter

public class IcebergArrowColumnVector
extends org.apache.spark.sql.vectorized.ColumnVector

Implementation of Spark's ColumnVector interface. The code for this class is heavily inspired from Spark's ArrowColumnVector The main difference is in how nullability checks are made in this class by relying on NullabilityHolder instead of the validity vector in the Arrow vector.

Field Summary

Fields inherited from class org.apache.spark.sql.vectorized.ColumnVector

type

Constructor Summary

Constructors

Constructor and Description
IcebergArrowColumnVector(VectorHolder holder)

Method Summary

Modifier and Type	Method and Description
protected ArrowVectorAccessor<org.apache.spark.sql.types.Decimal,org.apache.spark.unsafe.types.UTF8String,org.apache.spark.sql.vectorized.ColumnarArray,org.apache.spark.sql.vectorized.ArrowColumnVector>	accessor()
void	close()
org.apache.spark.sql.vectorized.ColumnarArray	getArray(int rowId)
byte[]	getBinary(int rowId)
boolean	getBoolean(int rowId)
byte	getByte(int rowId)
org.apache.spark.sql.vectorized.ArrowColumnVector	getChild(int ordinal)
org.apache.spark.sql.types.Decimal	getDecimal(int rowId, int precision, int scale)
double	getDouble(int rowId)
float	getFloat(int rowId)
int	getInt(int rowId)
long	getLong(int rowId)
org.apache.spark.sql.vectorized.ColumnarMap	getMap(int rowId)
short	getShort(int rowId)
org.apache.spark.unsafe.types.UTF8String	getUTF8String(int rowId)
boolean	hasNull()
boolean	isNullAt(int rowId)
protected NullabilityHolder	nullabilityHolder()
int	numNulls()
ArrowVectorAccessor<org.apache.spark.sql.types.Decimal,org.apache.spark.unsafe.types.UTF8String,org.apache.spark.sql.vectorized.ColumnarArray,org.apache.spark.sql.vectorized.ArrowColumnVector>	vectorAccessor()

Methods inherited from class org.apache.spark.sql.vectorized.ColumnVector

dataType, getBooleans, getBytes, getDoubles, getFloats, getInterval, getInts, getLongs, getShorts, getStruct

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

IcebergArrowColumnVector

public IcebergArrowColumnVector(VectorHolder holder)

Method Detail

accessor

protected ArrowVectorAccessor<org.apache.spark.sql.types.Decimal,org.apache.spark.unsafe.types.UTF8String,org.apache.spark.sql.vectorized.ColumnarArray,org.apache.spark.sql.vectorized.ArrowColumnVector> accessor()

nullabilityHolder

protected NullabilityHolder nullabilityHolder()

close

public void close()

Specified by:

close in interface java.lang.AutoCloseable

Specified by:

close in class org.apache.spark.sql.vectorized.ColumnVector

hasNull

public boolean hasNull()

Specified by:

hasNull in class org.apache.spark.sql.vectorized.ColumnVector

numNulls

public int numNulls()

Specified by:

numNulls in class org.apache.spark.sql.vectorized.ColumnVector

isNullAt

public boolean isNullAt(int rowId)

Specified by:

isNullAt in class org.apache.spark.sql.vectorized.ColumnVector

getBoolean

public boolean getBoolean(int rowId)

Specified by:

getBoolean in class org.apache.spark.sql.vectorized.ColumnVector

getByte

public byte getByte(int rowId)

Specified by:

getByte in class org.apache.spark.sql.vectorized.ColumnVector

getShort

public short getShort(int rowId)

Specified by:

getShort in class org.apache.spark.sql.vectorized.ColumnVector

getInt

public int getInt(int rowId)

Specified by:

getInt in class org.apache.spark.sql.vectorized.ColumnVector

getLong

public long getLong(int rowId)

Specified by:

getLong in class org.apache.spark.sql.vectorized.ColumnVector

getFloat

public float getFloat(int rowId)

Specified by:

getFloat in class org.apache.spark.sql.vectorized.ColumnVector

getDouble

public double getDouble(int rowId)

Specified by:

getDouble in class org.apache.spark.sql.vectorized.ColumnVector

getArray

public org.apache.spark.sql.vectorized.ColumnarArray getArray(int rowId)

Specified by:

getArray in class org.apache.spark.sql.vectorized.ColumnVector

getMap

public org.apache.spark.sql.vectorized.ColumnarMap getMap(int rowId)

Specified by:

getMap in class org.apache.spark.sql.vectorized.ColumnVector

getDecimal

public org.apache.spark.sql.types.Decimal getDecimal(int rowId,
                                                     int precision,
                                                     int scale)

Specified by:

getDecimal in class org.apache.spark.sql.vectorized.ColumnVector

getUTF8String

public org.apache.spark.unsafe.types.UTF8String getUTF8String(int rowId)

Specified by:

getUTF8String in class org.apache.spark.sql.vectorized.ColumnVector

getBinary

public byte[] getBinary(int rowId)

Specified by:

getBinary in class org.apache.spark.sql.vectorized.ColumnVector

getChild

public org.apache.spark.sql.vectorized.ArrowColumnVector getChild(int ordinal)

Specified by:

getChild in class org.apache.spark.sql.vectorized.ColumnVector

vectorAccessor

public ArrowVectorAccessor<org.apache.spark.sql.types.Decimal,org.apache.spark.unsafe.types.UTF8String,org.apache.spark.sql.vectorized.ColumnarArray,org.apache.spark.sql.vectorized.ArrowColumnVector> vectorAccessor()