Package org.apache.iceberg.spark

Class SparkSessionCatalog<T extends org.apache.spark.sql.connector.catalog.TableCatalog & org.apache.spark.sql.connector.catalog.FunctionCatalog & org.apache.spark.sql.connector.catalog.SupportsNamespaces>

  • Type Parameters:
    T - CatalogPlugin class to avoid casting to TableCatalog, FunctionCatalog and SupportsNamespaces.
    All Implemented Interfaces:
    HasIcebergCatalog, org.apache.spark.sql.connector.catalog.CatalogExtension, org.apache.spark.sql.connector.catalog.CatalogPlugin, org.apache.spark.sql.connector.catalog.FunctionCatalog, org.apache.spark.sql.connector.catalog.StagingTableCatalog, org.apache.spark.sql.connector.catalog.SupportsNamespaces, org.apache.spark.sql.connector.catalog.TableCatalog, ProcedureCatalog
    public class SparkSessionCatalog<T extends org.apache.spark.sql.connector.catalog.TableCatalog & org.apache.spark.sql.connector.catalog.FunctionCatalog & org.apache.spark.sql.connector.catalog.SupportsNamespaces>
extends java.lang.Object
implements org.apache.spark.sql.connector.catalog.CatalogExtension
    A Spark catalog that can also load non-Iceberg tables.

    • Field Summary

      • Fields inherited from interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

        PROP_COMMENT, PROP_LOCATION, PROP_OWNER

      • Fields inherited from interface org.apache.spark.sql.connector.catalog.TableCatalog

        OPTION_PREFIX, PROP_COMMENT, PROP_EXTERNAL, PROP_IS_MANAGED_LOCATION, PROP_LOCATION, PROP_OWNER, PROP_PROVIDER

    • Method Summary

      All Methods Instance Methods Concrete Methods Default Methods 
      Modifier and Type Method Description
      void alterNamespace​(java.lang.String[] namespace, org.apache.spark.sql.connector.catalog.NamespaceChange... changes)  
      org.apache.spark.sql.connector.catalog.Table alterTable​(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.connector.catalog.TableChange... changes)  
      protected org.apache.spark.sql.connector.catalog.TableCatalog buildSparkCatalog​(java.lang.String name, org.apache.spark.sql.util.CaseInsensitiveStringMap options)
      Build a SparkCatalog to be used for Iceberg operations.
      void createNamespace​(java.lang.String[] namespace, java.util.Map<java.lang.String,​java.lang.String> metadata)  
      org.apache.spark.sql.connector.catalog.Table createTable​(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] partitions, java.util.Map<java.lang.String,​java.lang.String> properties)  
      java.lang.String[] defaultNamespace()  
      boolean dropNamespace​(java.lang.String[] namespace, boolean cascade)  
      boolean dropTable​(org.apache.spark.sql.connector.catalog.Identifier ident)  
      Catalog icebergCatalog()
      Returns the underlying Catalog backing this Spark Catalog
      void initialize​(java.lang.String name, org.apache.spark.sql.util.CaseInsensitiveStringMap options)  
      void invalidateTable​(org.apache.spark.sql.connector.catalog.Identifier ident)  
      boolean isExistingNamespace​(java.lang.String[] namespace)  
      boolean isFunctionNamespace​(java.lang.String[] namespace)  
      default org.apache.spark.sql.connector.catalog.Identifier[] listFunctions​(java.lang.String[] namespace)  
      java.lang.String[][] listNamespaces()  
      java.lang.String[][] listNamespaces​(java.lang.String[] namespace)  
      org.apache.spark.sql.connector.catalog.Identifier[] listTables​(java.lang.String[] namespace)  
      org.apache.spark.sql.connector.catalog.functions.UnboundFunction loadFunction​(org.apache.spark.sql.connector.catalog.Identifier ident)  
      java.util.Map<java.lang.String,​java.lang.String> loadNamespaceMetadata​(java.lang.String[] namespace)  
      Procedure loadProcedure​(org.apache.spark.sql.connector.catalog.Identifier ident)
      Load a stored procedure by identifier.
      org.apache.spark.sql.connector.catalog.Table loadTable​(org.apache.spark.sql.connector.catalog.Identifier ident)  
      org.apache.spark.sql.connector.catalog.Table loadTable​(org.apache.spark.sql.connector.catalog.Identifier ident, long timestamp)  
      org.apache.spark.sql.connector.catalog.Table loadTable​(org.apache.spark.sql.connector.catalog.Identifier ident, java.lang.String version)  
      java.lang.String name()  
      boolean namespaceExists​(java.lang.String[] namespace)  
      boolean purgeTable​(org.apache.spark.sql.connector.catalog.Identifier ident)  
      void renameTable​(org.apache.spark.sql.connector.catalog.Identifier from, org.apache.spark.sql.connector.catalog.Identifier to)  
      void setDelegateCatalog​(org.apache.spark.sql.connector.catalog.CatalogPlugin sparkSessionCatalog)  
      org.apache.spark.sql.connector.catalog.StagedTable stageCreate​(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] partitions, java.util.Map<java.lang.String,​java.lang.String> properties)  
      org.apache.spark.sql.connector.catalog.StagedTable stageCreateOrReplace​(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] partitions, java.util.Map<java.lang.String,​java.lang.String> properties)  
      org.apache.spark.sql.connector.catalog.StagedTable stageReplace​(org.apache.spark.sql.connector.catalog.Identifier ident, org.apache.spark.sql.types.StructType schema, org.apache.spark.sql.connector.expressions.Transform[] partitions, java.util.Map<java.lang.String,​java.lang.String> properties)  
      boolean useNullableQuerySchema()  

      • Methods inherited from class java.lang.Object

        clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

      • Methods inherited from interface org.apache.spark.sql.connector.catalog.FunctionCatalog

        functionExists

      • Methods inherited from interface org.apache.spark.sql.connector.catalog.StagingTableCatalog

        stageCreate, stageCreateOrReplace, stageReplace

      • Methods inherited from interface org.apache.spark.sql.connector.catalog.TableCatalog

        capabilities, createTable, tableExists, useNullableQuerySchema

    • Constructor Detail

      • SparkSessionCatalog

        public SparkSessionCatalog()

    • Method Detail

      • buildSparkCatalog

        protected org.apache.spark.sql.connector.catalog.TableCatalog buildSparkCatalog​(java.lang.String name,
                                                                                org.apache.spark.sql.util.CaseInsensitiveStringMap options)
        Build a SparkCatalog to be used for Iceberg operations.

        The default implementation creates a new SparkCatalog with the session catalog's name and options.

        Parameters:
        name - catalog name
        options - catalog options
        Returns:
        a SparkCatalog to be used for Iceberg tables

      • defaultNamespace

        public java.lang.String[] defaultNamespace()
        Specified by:
        defaultNamespace in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

      • listNamespaces

        public java.lang.String[][] listNamespaces()
                                    throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
        Specified by:
        listNamespaces in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

      • listNamespaces

        public java.lang.String[][] listNamespaces​(java.lang.String[] namespace)
                                    throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
        Specified by:
        listNamespaces in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

      • namespaceExists

        public boolean namespaceExists​(java.lang.String[] namespace)
        Specified by:
        namespaceExists in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces

      • loadNamespaceMetadata

        public java.util.Map<java.lang.String,​java.lang.String> loadNamespaceMetadata​(java.lang.String[] namespace)
                                                                             throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
        Specified by:
        loadNamespaceMetadata in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

      • createNamespace

        public void createNamespace​(java.lang.String[] namespace,
                            java.util.Map<java.lang.String,​java.lang.String> metadata)
                     throws org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException
        Specified by:
        createNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
        Throws:
        org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException

      • alterNamespace

        public void alterNamespace​(java.lang.String[] namespace,
                           org.apache.spark.sql.connector.catalog.NamespaceChange... changes)
                    throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
        Specified by:
        alterNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

      • dropNamespace

        public boolean dropNamespace​(java.lang.String[] namespace,
                             boolean cascade)
                      throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException,
                             org.apache.spark.sql.catalyst.analysis.NonEmptyNamespaceException
        Specified by:
        dropNamespace in interface org.apache.spark.sql.connector.catalog.SupportsNamespaces
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
        org.apache.spark.sql.catalyst.analysis.NonEmptyNamespaceException

      • listTables

        public org.apache.spark.sql.connector.catalog.Identifier[] listTables​(java.lang.String[] namespace)
                                                               throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
        Specified by:
        listTables in interface org.apache.spark.sql.connector.catalog.TableCatalog
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

      • loadTable

        public org.apache.spark.sql.connector.catalog.Table loadTable​(org.apache.spark.sql.connector.catalog.Identifier ident)
                                                       throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException
        Specified by:
        loadTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchTableException

      • loadTable

        public org.apache.spark.sql.connector.catalog.Table loadTable​(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                              java.lang.String version)
                                                       throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException
        Specified by:
        loadTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchTableException

      • loadTable

        public org.apache.spark.sql.connector.catalog.Table loadTable​(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                              long timestamp)
                                                       throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException
        Specified by:
        loadTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchTableException

      • invalidateTable

        public void invalidateTable​(org.apache.spark.sql.connector.catalog.Identifier ident)
        Specified by:
        invalidateTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

      • createTable

        public org.apache.spark.sql.connector.catalog.Table createTable​(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                                org.apache.spark.sql.types.StructType schema,
                                                                org.apache.spark.sql.connector.expressions.Transform[] partitions,
                                                                java.util.Map<java.lang.String,​java.lang.String> properties)
                                                         throws org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException,
                                                                org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
        Specified by:
        createTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
        Throws:
        org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
        org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

      • stageCreate

        public org.apache.spark.sql.connector.catalog.StagedTable stageCreate​(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                                      org.apache.spark.sql.types.StructType schema,
                                                                      org.apache.spark.sql.connector.expressions.Transform[] partitions,
                                                                      java.util.Map<java.lang.String,​java.lang.String> properties)
                                                               throws org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException,
                                                                      org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
        Specified by:
        stageCreate in interface org.apache.spark.sql.connector.catalog.StagingTableCatalog
        Throws:
        org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
        org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

      • stageReplace

        public org.apache.spark.sql.connector.catalog.StagedTable stageReplace​(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                                       org.apache.spark.sql.types.StructType schema,
                                                                       org.apache.spark.sql.connector.expressions.Transform[] partitions,
                                                                       java.util.Map<java.lang.String,​java.lang.String> properties)
                                                                throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException,
                                                                       org.apache.spark.sql.catalyst.analysis.NoSuchTableException
        Specified by:
        stageReplace in interface org.apache.spark.sql.connector.catalog.StagingTableCatalog
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
        org.apache.spark.sql.catalyst.analysis.NoSuchTableException

      • stageCreateOrReplace

        public org.apache.spark.sql.connector.catalog.StagedTable stageCreateOrReplace​(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                                               org.apache.spark.sql.types.StructType schema,
                                                                               org.apache.spark.sql.connector.expressions.Transform[] partitions,
                                                                               java.util.Map<java.lang.String,​java.lang.String> properties)
                                                                        throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
        Specified by:
        stageCreateOrReplace in interface org.apache.spark.sql.connector.catalog.StagingTableCatalog
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException

      • alterTable

        public org.apache.spark.sql.connector.catalog.Table alterTable​(org.apache.spark.sql.connector.catalog.Identifier ident,
                                                               org.apache.spark.sql.connector.catalog.TableChange... changes)
                                                        throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException
        Specified by:
        alterTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchTableException

      • dropTable

        public boolean dropTable​(org.apache.spark.sql.connector.catalog.Identifier ident)
        Specified by:
        dropTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

      • purgeTable

        public boolean purgeTable​(org.apache.spark.sql.connector.catalog.Identifier ident)
        Specified by:
        purgeTable in interface org.apache.spark.sql.connector.catalog.TableCatalog

      • renameTable

        public void renameTable​(org.apache.spark.sql.connector.catalog.Identifier from,
                        org.apache.spark.sql.connector.catalog.Identifier to)
                 throws org.apache.spark.sql.catalyst.analysis.NoSuchTableException,
                        org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
        Specified by:
        renameTable in interface org.apache.spark.sql.connector.catalog.TableCatalog
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchTableException
        org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException

      • initialize

        public final void initialize​(java.lang.String name,
                             org.apache.spark.sql.util.CaseInsensitiveStringMap options)
        Specified by:
        initialize in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

      • setDelegateCatalog

        public void setDelegateCatalog​(org.apache.spark.sql.connector.catalog.CatalogPlugin sparkSessionCatalog)
        Specified by:
        setDelegateCatalog in interface org.apache.spark.sql.connector.catalog.CatalogExtension

      • name

        public java.lang.String name()
        Specified by:
        name in interface org.apache.spark.sql.connector.catalog.CatalogPlugin

      • loadFunction

        public org.apache.spark.sql.connector.catalog.functions.UnboundFunction loadFunction​(org.apache.spark.sql.connector.catalog.Identifier ident)
                                                                              throws org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException
        Specified by:
        loadFunction in interface org.apache.spark.sql.connector.catalog.FunctionCatalog
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException

      • isFunctionNamespace

        public boolean isFunctionNamespace​(java.lang.String[] namespace)

      • isExistingNamespace

        public boolean isExistingNamespace​(java.lang.String[] namespace)

      • useNullableQuerySchema

        public boolean useNullableQuerySchema()
        Specified by:
        useNullableQuerySchema in interface org.apache.spark.sql.connector.catalog.TableCatalog

      • listFunctions

        public default org.apache.spark.sql.connector.catalog.Identifier[] listFunctions​(java.lang.String[] namespace)
                                                                          throws org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
        Specified by:
        listFunctions in interface org.apache.spark.sql.connector.catalog.FunctionCatalog
        Throws:
        org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException