org.apache.iceberg.spark.source

Class IcebergSource

java.lang.Object

org.apache.iceberg.spark.source.IcebergSource

All Implemented Interfaces:

org.apache.spark.sql.execution.streaming.BaseStreamingSink, org.apache.spark.sql.sources.DataSourceRegister, org.apache.spark.sql.sources.v2.DataSourceV2, org.apache.spark.sql.sources.v2.ReadSupport, org.apache.spark.sql.sources.v2.StreamWriteSupport, org.apache.spark.sql.sources.v2.WriteSupport

public class IcebergSource
extends java.lang.Object
implements org.apache.spark.sql.sources.v2.DataSourceV2, org.apache.spark.sql.sources.v2.ReadSupport, org.apache.spark.sql.sources.v2.WriteSupport, org.apache.spark.sql.sources.DataSourceRegister, org.apache.spark.sql.sources.v2.StreamWriteSupport

Constructor Summary

Constructors

Constructor and Description
IcebergSource()

Method Summary

Modifier and Type	Method and Description
org.apache.spark.sql.sources.v2.reader.DataSourceReader	createReader(org.apache.spark.sql.sources.v2.DataSourceOptions options)
org.apache.spark.sql.sources.v2.reader.DataSourceReader	createReader(org.apache.spark.sql.types.StructType readSchema, org.apache.spark.sql.sources.v2.DataSourceOptions options)
org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter	createStreamWriter(java.lang.String runId, org.apache.spark.sql.types.StructType dsStruct, org.apache.spark.sql.streaming.OutputMode mode, org.apache.spark.sql.sources.v2.DataSourceOptions options)
java.util.Optional<org.apache.spark.sql.sources.v2.writer.DataSourceWriter>	createWriter(java.lang.String jobId, org.apache.spark.sql.types.StructType dsStruct, org.apache.spark.sql.SaveMode mode, org.apache.spark.sql.sources.v2.DataSourceOptions options)
protected Table	findTable(org.apache.spark.sql.sources.v2.DataSourceOptions options, org.apache.hadoop.conf.Configuration conf)
java.lang.String	shortName()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

IcebergSource

public IcebergSource()

Method Detail

shortName

public java.lang.String shortName()

Specified by:

shortName in interface org.apache.spark.sql.sources.DataSourceRegister

createReader

public org.apache.spark.sql.sources.v2.reader.DataSourceReader createReader(org.apache.spark.sql.sources.v2.DataSourceOptions options)

Specified by:

createReader in interface org.apache.spark.sql.sources.v2.ReadSupport

createReader

public org.apache.spark.sql.sources.v2.reader.DataSourceReader createReader(org.apache.spark.sql.types.StructType readSchema,
                                                                            org.apache.spark.sql.sources.v2.DataSourceOptions options)

Specified by:

createReader in interface org.apache.spark.sql.sources.v2.ReadSupport

createWriter

public java.util.Optional<org.apache.spark.sql.sources.v2.writer.DataSourceWriter> createWriter(java.lang.String jobId,
                                                                                                org.apache.spark.sql.types.StructType dsStruct,
                                                                                                org.apache.spark.sql.SaveMode mode,
                                                                                                org.apache.spark.sql.sources.v2.DataSourceOptions options)

Specified by:

createWriter in interface org.apache.spark.sql.sources.v2.WriteSupport

createStreamWriter

public org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter createStreamWriter(java.lang.String runId,
                                                                                        org.apache.spark.sql.types.StructType dsStruct,
                                                                                        org.apache.spark.sql.streaming.OutputMode mode,
                                                                                        org.apache.spark.sql.sources.v2.DataSourceOptions options)

Specified by:

createStreamWriter in interface org.apache.spark.sql.sources.v2.StreamWriteSupport

findTable

protected Table findTable(org.apache.spark.sql.sources.v2.DataSourceOptions options,
                          org.apache.hadoop.conf.Configuration conf)