Package org.apache.iceberg.deletes

Class Deletes

java.lang.Object

org.apache.iceberg.deletes.Deletes

public class Deletes
extends java.lang.Object

Method Summary

Modifier and Type	Method	Description
static <T extends StructLike> CloseableIterable<java.lang.Long>	deletePositions(java.lang.CharSequence dataLocation, java.util.List<CloseableIterable<T>> deleteFiles)
static CloseableIterable<java.lang.Long>	deletePositions(java.lang.CharSequence dataLocation, CloseableIterable<StructLike> deleteFile)
static <T> CloseableIterable<T>	filter(CloseableIterable<T> rows, java.util.function.Function<T,StructLike> rowToDeleteKey, StructLikeSet deleteSet)
static <T> CloseableIterable<T>	filterDeleted(CloseableIterable<T> rows, java.util.function.Predicate<T> isDeleted, DeleteCounter counter)	Returns the remaining rows (the ones that are not deleted), while counting the deleted ones.
static <T> CloseableIterable<T>	markDeleted(CloseableIterable<T> rows, java.util.function.Predicate<T> isDeleted, java.util.function.Consumer<T> deleteMarker)	Returns the same rows that are input, while marking the deleted ones.
static <T> CloseableIterable<T>	streamingFilter(CloseableIterable<T> rows, java.util.function.Function<T,java.lang.Long> rowToPosition, CloseableIterable<java.lang.Long> posDeletes)
static <T> CloseableIterable<T>	streamingFilter(CloseableIterable<T> rows, java.util.function.Function<T,java.lang.Long> rowToPosition, CloseableIterable<java.lang.Long> posDeletes, DeleteCounter counter)
static <T> CloseableIterable<T>	streamingMarker(CloseableIterable<T> rows, java.util.function.Function<T,java.lang.Long> rowToPosition, CloseableIterable<java.lang.Long> posDeletes, java.util.function.Consumer<T> markDeleted)
static StructLikeSet	toEqualitySet(CloseableIterable<StructLike> eqDeletes, Types.StructType eqType)
static <T extends StructLike> PositionDeleteIndex	toPositionIndex(java.lang.CharSequence dataLocation, java.util.List<CloseableIterable<T>> deleteFiles)
static <T extends StructLike> PositionDeleteIndex	toPositionIndex(java.lang.CharSequence dataLocation, java.util.List<CloseableIterable<T>> deleteFiles, java.util.concurrent.ExecutorService deleteWorkerPool)
static PositionDeleteIndex	toPositionIndex(CloseableIterable<java.lang.Long> posDeletes)
static <T extends StructLike> CharSequenceMap<PositionDeleteIndex>	toPositionIndexes(CloseableIterable<T> posDeletes)	Builds a map of position delete indexes by path.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Method Detail

filter

public static <T> CloseableIterable<T> filter(CloseableIterable<T> rows,
                                              java.util.function.Function<T,StructLike> rowToDeleteKey,
                                              StructLikeSet deleteSet)

markDeleted

public static <T> CloseableIterable<T> markDeleted(CloseableIterable<T> rows,
                                                   java.util.function.Predicate<T> isDeleted,
                                                   java.util.function.Consumer<T> deleteMarker)

Returns the same rows that are input, while marking the deleted ones.

Parameters:

rows - the rows to process

isDeleted - a predicate that determines if a row is deleted

deleteMarker - a function that marks a row as deleted

Returns:

the processed rows

filterDeleted

public static <T> CloseableIterable<T> filterDeleted(CloseableIterable<T> rows,
                                                     java.util.function.Predicate<T> isDeleted,
                                                     DeleteCounter counter)

Returns the remaining rows (the ones that are not deleted), while counting the deleted ones.

Parameters:

rows - the rows to process

isDeleted - a predicate that determines if a row is deleted

counter - a counter that counts deleted rows

Returns:

the processed rows

toEqualitySet

public static StructLikeSet toEqualitySet(CloseableIterable<StructLike> eqDeletes,
                                          Types.StructType eqType)

toPositionIndexes

public static <T extends StructLike> CharSequenceMap<PositionDeleteIndex> toPositionIndexes(CloseableIterable<T> posDeletes)

Builds a map of position delete indexes by path.

Unlike toPositionIndex(CharSequence, List), this method builds a position delete index for each referenced data file and does not filter deletes. This can be useful when the entire delete file content is needed (e.g. caching).

Parameters:

posDeletes - position deletes

Returns:

the map of position delete indexes by path

toPositionIndex

public static <T extends StructLike> PositionDeleteIndex toPositionIndex(java.lang.CharSequence dataLocation,
                                                                         java.util.List<CloseableIterable<T>> deleteFiles)

toPositionIndex

public static <T extends StructLike> PositionDeleteIndex toPositionIndex(java.lang.CharSequence dataLocation,
                                                                         java.util.List<CloseableIterable<T>> deleteFiles,
                                                                         java.util.concurrent.ExecutorService deleteWorkerPool)

toPositionIndex

public static PositionDeleteIndex toPositionIndex(CloseableIterable<java.lang.Long> posDeletes)

streamingFilter

public static <T> CloseableIterable<T> streamingFilter(CloseableIterable<T> rows,
                                                       java.util.function.Function<T,java.lang.Long> rowToPosition,
                                                       CloseableIterable<java.lang.Long> posDeletes)

streamingFilter

public static <T> CloseableIterable<T> streamingFilter(CloseableIterable<T> rows,
                                                       java.util.function.Function<T,java.lang.Long> rowToPosition,
                                                       CloseableIterable<java.lang.Long> posDeletes,
                                                       DeleteCounter counter)

streamingMarker

public static <T> CloseableIterable<T> streamingMarker(CloseableIterable<T> rows,
                                                       java.util.function.Function<T,java.lang.Long> rowToPosition,
                                                       CloseableIterable<java.lang.Long> posDeletes,
                                                       java.util.function.Consumer<T> markDeleted)

deletePositions

public static CloseableIterable<java.lang.Long> deletePositions(java.lang.CharSequence dataLocation,
                                                                CloseableIterable<StructLike> deleteFile)

deletePositions

public static <T extends StructLike> CloseableIterable<java.lang.Long> deletePositions(java.lang.CharSequence dataLocation,
                                                                                       java.util.List<CloseableIterable<T>> deleteFiles)