Package org.apache.iceberg.spark

Class SparkWriteConf

java.lang.Object

org.apache.iceberg.spark.SparkWriteConf

public class SparkWriteConf
extends java.lang.Object

A class for common Iceberg configs for Spark writes.

If a config is set at multiple levels, the following order of precedence is used (top to bottom):

1. Write options
2. Session configuration
3. Table metadata

The most specific value is set in write options and takes precedence over all other configs. If no write option is provided, this class checks the session configuration for any overrides. If no applicable value is found in the session configuration, this class uses the table metadata.

Note this class is NOT meant to be serialized and sent to executors.

Constructor Summary

Constructors

Constructor	Description
SparkWriteConf(org.apache.spark.sql.SparkSession spark, Table table, java.lang.String branch, java.util.Map<java.lang.String,java.lang.String> writeOptions)
SparkWriteConf(org.apache.spark.sql.SparkSession spark, Table table, java.util.Map<java.lang.String,java.lang.String> writeOptions)

Method Summary

Modifier and Type	Method	Description
java.lang.String	branch()
boolean	caseSensitive()
boolean	checkNullability()
boolean	checkOrdering()
SparkWriteRequirements	copyOnWriteRequirements(org.apache.spark.sql.connector.write.RowLevelOperation.Command command)
FileFormat	dataFileFormat()
FileFormat	deleteFileFormat()
java.util.Map<java.lang.String,java.lang.String>	extraSnapshotMetadata()
IsolationLevel	isolationLevel()
boolean	mergeSchema()
int	outputSpecId()
java.lang.String	overwriteMode()
SparkWriteRequirements	positionDeltaRequirements(org.apache.spark.sql.connector.write.RowLevelOperation.Command command)
java.lang.String	rewrittenFileSetId()
long	targetDataFileSize()
long	targetDeleteFileSize()
boolean	useFanoutWriter(SparkWriteRequirements writeRequirements)
java.lang.Long	validateFromSnapshotId()
boolean	wapEnabled()
java.lang.String	wapId()
java.util.Map<java.lang.String,java.lang.String>	writeProperties()
SparkWriteRequirements	writeRequirements()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

SparkWriteConf

public SparkWriteConf(org.apache.spark.sql.SparkSession spark,
                      Table table,
                      java.util.Map<java.lang.String,java.lang.String> writeOptions)

SparkWriteConf

public SparkWriteConf(org.apache.spark.sql.SparkSession spark,
                      Table table,
                      java.lang.String branch,
                      java.util.Map<java.lang.String,java.lang.String> writeOptions)

Method Detail

checkNullability

public boolean checkNullability()

checkOrdering

public boolean checkOrdering()

overwriteMode

public java.lang.String overwriteMode()

wapEnabled

public boolean wapEnabled()

wapId

public java.lang.String wapId()

mergeSchema

public boolean mergeSchema()

outputSpecId

public int outputSpecId()

dataFileFormat

public FileFormat dataFileFormat()

targetDataFileSize

public long targetDataFileSize()

useFanoutWriter

public boolean useFanoutWriter(SparkWriteRequirements writeRequirements)

deleteFileFormat

public FileFormat deleteFileFormat()

targetDeleteFileSize

public long targetDeleteFileSize()

extraSnapshotMetadata

public java.util.Map<java.lang.String,java.lang.String> extraSnapshotMetadata()

rewrittenFileSetId

public java.lang.String rewrittenFileSetId()

writeRequirements

public SparkWriteRequirements writeRequirements()

copyOnWriteRequirements

public SparkWriteRequirements copyOnWriteRequirements(org.apache.spark.sql.connector.write.RowLevelOperation.Command command)

positionDeltaRequirements

public SparkWriteRequirements positionDeltaRequirements(org.apache.spark.sql.connector.write.RowLevelOperation.Command command)

validateFromSnapshotId

public java.lang.Long validateFromSnapshotId()

isolationLevel

public IsolationLevel isolationLevel()

caseSensitive

public boolean caseSensitive()

branch

public java.lang.String branch()

writeProperties

public java.util.Map<java.lang.String,java.lang.String> writeProperties()