Package org.apache.iceberg.spark
Class SparkWriteConf
- java.lang.Object
-
- org.apache.iceberg.spark.SparkWriteConf
-
public class SparkWriteConf extends java.lang.Object
A class for common Iceberg configs for Spark writes.If a config is set at multiple levels, the following order of precedence is used (top to bottom):
- Write options
- Session configuration
- Table metadata
Note this class is NOT meant to be serialized and sent to executors.
-
-
Constructor Summary
Constructors Constructor Description SparkWriteConf(org.apache.spark.sql.SparkSession spark, Table table, java.lang.String branch, java.util.Map<java.lang.String,java.lang.String> writeOptions)
SparkWriteConf(org.apache.spark.sql.SparkSession spark, Table table, java.util.Map<java.lang.String,java.lang.String> writeOptions)
-
Method Summary
All Methods Instance Methods Concrete Methods Modifier and Type Method Description java.lang.String
branch()
boolean
caseSensitive()
boolean
checkNullability()
boolean
checkOrdering()
SparkWriteRequirements
copyOnWriteRequirements(org.apache.spark.sql.connector.write.RowLevelOperation.Command command)
FileFormat
dataFileFormat()
FileFormat
deleteFileFormat()
DeleteGranularity
deleteGranularity()
java.util.Map<java.lang.String,java.lang.String>
extraSnapshotMetadata()
IsolationLevel
isolationLevel()
boolean
mergeSchema()
int
outputSpecId()
java.lang.String
overwriteMode()
SparkWriteRequirements
positionDeltaRequirements(org.apache.spark.sql.connector.write.RowLevelOperation.Command command)
java.lang.String
rewrittenFileSetId()
long
targetDataFileSize()
long
targetDeleteFileSize()
boolean
useFanoutWriter(SparkWriteRequirements writeRequirements)
java.lang.Long
validateFromSnapshotId()
boolean
wapEnabled()
java.lang.String
wapId()
java.util.Map<java.lang.String,java.lang.String>
writeProperties()
SparkWriteRequirements
writeRequirements()
-
-
-
Constructor Detail
-
SparkWriteConf
public SparkWriteConf(org.apache.spark.sql.SparkSession spark, Table table, java.util.Map<java.lang.String,java.lang.String> writeOptions)
-
SparkWriteConf
public SparkWriteConf(org.apache.spark.sql.SparkSession spark, Table table, java.lang.String branch, java.util.Map<java.lang.String,java.lang.String> writeOptions)
-
-
Method Detail
-
checkNullability
public boolean checkNullability()
-
checkOrdering
public boolean checkOrdering()
-
overwriteMode
public java.lang.String overwriteMode()
-
wapEnabled
public boolean wapEnabled()
-
wapId
public java.lang.String wapId()
-
mergeSchema
public boolean mergeSchema()
-
outputSpecId
public int outputSpecId()
-
dataFileFormat
public FileFormat dataFileFormat()
-
targetDataFileSize
public long targetDataFileSize()
-
useFanoutWriter
public boolean useFanoutWriter(SparkWriteRequirements writeRequirements)
-
deleteFileFormat
public FileFormat deleteFileFormat()
-
targetDeleteFileSize
public long targetDeleteFileSize()
-
extraSnapshotMetadata
public java.util.Map<java.lang.String,java.lang.String> extraSnapshotMetadata()
-
rewrittenFileSetId
public java.lang.String rewrittenFileSetId()
-
writeRequirements
public SparkWriteRequirements writeRequirements()
-
copyOnWriteRequirements
public SparkWriteRequirements copyOnWriteRequirements(org.apache.spark.sql.connector.write.RowLevelOperation.Command command)
-
positionDeltaRequirements
public SparkWriteRequirements positionDeltaRequirements(org.apache.spark.sql.connector.write.RowLevelOperation.Command command)
-
validateFromSnapshotId
public java.lang.Long validateFromSnapshotId()
-
isolationLevel
public IsolationLevel isolationLevel()
-
caseSensitive
public boolean caseSensitive()
-
branch
public java.lang.String branch()
-
writeProperties
public java.util.Map<java.lang.String,java.lang.String> writeProperties()
-
deleteGranularity
public DeleteGranularity deleteGranularity()
-
-