com.logicalclocks.hsfs.engine

Class FeatureGroupEngine

java.lang.Object

com.logicalclocks.hsfs.engine.FeatureGroupEngine

public class FeatureGroupEngine
extends Object

Constructor Summary

Constructors

Constructor and Description
FeatureGroupEngine()

Method Summary

Modifier and Type	Method and Description
void	insert(FeatureGroup featureGroup, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData, Storage storage, HudiOperationType operation, org.apache.spark.sql.SaveMode saveMode, List<String> partitionKeys, String hudiPrecombineKey, Map<String,String> writeOptions)
org.apache.spark.sql.streaming.StreamingQuery	insertStream(FeatureGroup featureGroup, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData, String queryName, String outputMode, boolean awaitTermination, Long timeout, String checkpointLocation, List<String> partitionKeys, String hudiPrecombineKey, Map<String,String> writeOptions) Deprecated.
FeatureGroup	save(FeatureGroup featureGroup, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset, List<String> partitionKeys, String hudiPrecombineKey, Map<String,String> writeOptions) Create the metadata and write the data to the online/offline feature store.
void	saveDataframe(FeatureGroup featureGroup, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset, Storage storage, HudiOperationType operation, Map<String,String> offlineWriteOptions, Map<String,String> onlineWriteOptions, Integer validationId)
FeatureGroup	saveFeatureGroupMetaData(FeatureGroup featureGroup, List<String> partitionKeys, String hudiPrecombineKey, org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData, boolean saveEmpty)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

FeatureGroupEngine

public FeatureGroupEngine()

Method Detail

save

public FeatureGroup save(FeatureGroup featureGroup,
                         org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
                         List<String> partitionKeys,
                         String hudiPrecombineKey,
                         Map<String,String> writeOptions)
                  throws FeatureStoreException,
                         IOException,
                         ParseException

Create the metadata and write the data to the online/offline feature store.

Parameters:

featureGroup - Feature Group metadata object.

dataset - Spark DataFrame or RDD.

partitionKeys - A list of feature names to be used as partition key when writing the feature data to the offline storage, defaults to empty list.

hudiPrecombineKey - A feature name to be used as a precombine key for the `TimeTravelFormat.HUDI` feature group. If feature group has `TimeTravelFormat.HUDI` and hudi precombine key was not specified then the first primary key of the feature group will be used as hudi precombine key.

writeOptions - Additional write options as key-value pairs, defaults to empty Map.

Returns:

Feature Group metadata object

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

ParseException - ParseException

insert

public void insert(FeatureGroup featureGroup,
                   org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
                   Storage storage,
                   HudiOperationType operation,
                   org.apache.spark.sql.SaveMode saveMode,
                   List<String> partitionKeys,
                   String hudiPrecombineKey,
                   Map<String,String> writeOptions)
            throws FeatureStoreException,
                   IOException,
                   ParseException

Throws:

FeatureStoreException

IOException

ParseException

insertStream

@Deprecated
public org.apache.spark.sql.streaming.StreamingQuery insertStream(FeatureGroup featureGroup,
                                                                              org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
                                                                              String queryName,
                                                                              String outputMode,
                                                                              boolean awaitTermination,
                                                                              Long timeout,
                                                                              String checkpointLocation,
                                                                              List<String> partitionKeys,
                                                                              String hudiPrecombineKey,
                                                                              Map<String,String> writeOptions)
                                                                       throws FeatureStoreException,
                                                                              IOException,
                                                                              org.apache.spark.sql.streaming.StreamingQueryException,
                                                                              TimeoutException,
                                                                              ParseException

Deprecated.

Throws:

FeatureStoreException

IOException

org.apache.spark.sql.streaming.StreamingQueryException

TimeoutException

ParseException

saveDataframe

public void saveDataframe(FeatureGroup featureGroup,
                          org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset,
                          Storage storage,
                          HudiOperationType operation,
                          Map<String,String> offlineWriteOptions,
                          Map<String,String> onlineWriteOptions,
                          Integer validationId)
                   throws IOException,
                          FeatureStoreException,
                          ParseException

Throws:

IOException

FeatureStoreException

ParseException

saveFeatureGroupMetaData

public FeatureGroup saveFeatureGroupMetaData(FeatureGroup featureGroup,
                                             List<String> partitionKeys,
                                             String hudiPrecombineKey,
                                             org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> featureData,
                                             boolean saveEmpty)
                                      throws FeatureStoreException,
                                             IOException,
                                             ParseException

Throws:

FeatureStoreException

IOException

ParseException