public class Spark3Util
extends java.lang.Object
Modifier and Type | Class and Description |
---|---|
static class |
Spark3Util.DescribeSchemaVisitor |
Modifier and Type | Method and Description |
---|---|
static UpdateProperties |
applyPropertyChanges(UpdateProperties pendingUpdate,
java.util.List<org.apache.spark.sql.connector.catalog.TableChange> changes)
Applies a list of Spark table changes to an
UpdateProperties operation. |
static UpdateSchema |
applySchemaChanges(UpdateSchema pendingUpdate,
java.util.List<org.apache.spark.sql.connector.catalog.TableChange> changes)
Applies a list of Spark table changes to an
UpdateSchema operation. |
static int |
batchSize(java.util.Map<java.lang.String,java.lang.String> properties,
org.apache.spark.sql.util.CaseInsensitiveStringMap readOptions) |
static java.lang.String |
describe(Expression expr) |
static java.lang.String |
describe(Schema schema) |
static java.lang.String |
describe(Type type) |
static boolean |
isLocalityEnabled(FileIO io,
java.lang.String location,
org.apache.spark.sql.util.CaseInsensitiveStringMap readOptions) |
static boolean |
isVectorizationEnabled(java.util.Map<java.lang.String,java.lang.String> properties,
org.apache.spark.sql.util.CaseInsensitiveStringMap readOptions) |
static java.lang.Integer |
propertyAsInt(org.apache.spark.sql.util.CaseInsensitiveStringMap options,
java.lang.String property,
java.lang.Integer defaultValue) |
static java.lang.Long |
propertyAsLong(org.apache.spark.sql.util.CaseInsensitiveStringMap options,
java.lang.String property,
java.lang.Long defaultValue) |
static java.util.Map<java.lang.String,java.lang.String> |
rebuildCreateProperties(java.util.Map<java.lang.String,java.lang.String> createProperties) |
static PartitionSpec |
toPartitionSpec(Schema schema,
org.apache.spark.sql.connector.expressions.Transform[] partitioning)
Converts Spark transforms into a
PartitionSpec . |
static org.apache.spark.sql.connector.expressions.Transform[] |
toTransforms(PartitionSpec spec)
Converts a PartitionSpec to Spark transforms.
|
public static java.util.Map<java.lang.String,java.lang.String> rebuildCreateProperties(java.util.Map<java.lang.String,java.lang.String> createProperties)
public static UpdateProperties applyPropertyChanges(UpdateProperties pendingUpdate, java.util.List<org.apache.spark.sql.connector.catalog.TableChange> changes)
UpdateProperties
operation.pendingUpdate
- an uncommitted UpdateProperties operation to configurechanges
- a list of Spark table changespublic static UpdateSchema applySchemaChanges(UpdateSchema pendingUpdate, java.util.List<org.apache.spark.sql.connector.catalog.TableChange> changes)
UpdateSchema
operation.pendingUpdate
- an uncommitted UpdateSchema operation to configurechanges
- a list of Spark table changespublic static org.apache.spark.sql.connector.expressions.Transform[] toTransforms(PartitionSpec spec)
spec
- a PartitionSpecpublic static PartitionSpec toPartitionSpec(Schema schema, org.apache.spark.sql.connector.expressions.Transform[] partitioning)
PartitionSpec
.schema
- the table schemapartitioning
- Spark Transformspublic static java.lang.String describe(Expression expr)
public static java.lang.String describe(Schema schema)
public static java.lang.String describe(Type type)
public static boolean isLocalityEnabled(FileIO io, java.lang.String location, org.apache.spark.sql.util.CaseInsensitiveStringMap readOptions)
public static boolean isVectorizationEnabled(java.util.Map<java.lang.String,java.lang.String> properties, org.apache.spark.sql.util.CaseInsensitiveStringMap readOptions)
public static int batchSize(java.util.Map<java.lang.String,java.lang.String> properties, org.apache.spark.sql.util.CaseInsensitiveStringMap readOptions)
public static java.lang.Long propertyAsLong(org.apache.spark.sql.util.CaseInsensitiveStringMap options, java.lang.String property, java.lang.Long defaultValue)
public static java.lang.Integer propertyAsInt(org.apache.spark.sql.util.CaseInsensitiveStringMap options, java.lang.String property, java.lang.Integer defaultValue)