com.logicalclocks.hsfs

Class StreamFeatureGroup

java.lang.Object

com.logicalclocks.hsfs.metadata.FeatureGroupBase

com.logicalclocks.hsfs.StreamFeatureGroup

public class StreamFeatureGroup
extends FeatureGroupBase

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	StreamFeatureGroup.StreamFeatureGroupBuilder

Field Summary

Fields inherited from class com.logicalclocks.hsfs.metadata.FeatureGroupBase

created, creator, description, eventTime, expectationsNames, featureGroupBaseEngine, features, featureStore, id, location, name, primaryKeys, statisticsConfig, statisticsEngine, subject, timeTravelFormat, type, utils, version

Constructor Summary

Constructors

Constructor and Description
StreamFeatureGroup()
StreamFeatureGroup(Boolean onlineEnabled, StorageConnector onlineStorageConnector, StorageConnector offlineStorageConnector, List<String> statisticColumns, List<String> partitionKeys, String hudiPrecombineKey, String onlineTopicName, DeltaStreamerJobConf deltaStreamerJobConf)
StreamFeatureGroup(FeatureStore featureStore, int id)
StreamFeatureGroup(FeatureStore featureStore, @NonNull String name, Integer version, String description, List<String> primaryKeys, List<String> partitionKeys, String hudiPrecombineKey, boolean onlineEnabled, List<Feature> features, StatisticsConfig statisticsConfig, String onlineTopicName, String eventTime)
StreamFeatureGroup(Integer id, String description, List<Feature> features)

Method Summary

Modifier and Type	Method and Description
Query	asOf(String wallclockTime) Get Query object to retrieve all features of the group at a point in the past.
Query	asOf(String wallclockTime, String excludeUntil) Get Query object to retrieve all features of the group at a point in the past.
static StreamFeatureGroup.StreamFeatureGroupBuilder	builder()
<S> void	commitDeleteRecord(S featureData)
<S> void	commitDeleteRecord(S featureData, Map<String,String> writeOptions)
Map<Long,Map<String,String>>	commitDetails() Return commit details.
Map<Long,Map<String,String>>	commitDetails(Integer limit) Return commit details.
Map<Long,Map<String,String>>	commitDetails(String wallclockTime) Return commit details.
Map<Long,Map<String,String>>	commitDetails(String wallclockTime, Integer limit) Return commit details.
String	getAvroSchema()
List<String>	getComplexFeatures()
org.apache.avro.Schema	getDeserializedAvroSchema()
String	getEncodedAvroSchema()
String	getFeatureAvroSchema(String featureName)
StorageConnector	getOfflineStorageConnector()
Boolean	getOnlineEnabled()
StorageConnector	getOnlineStorageConnector()
String	getOnlineTopicName()
List<String>	getStatisticColumns()
<S> void	insert(S featureData)
<S> void	insert(S featureData, boolean overwrite, SaveMode saveMode, JobConfiguration jobConfiguration)
<S> void	insert(S featureData, boolean overwrite, SaveMode saveMode, Map<String,String> writeOptions)
<S> void	insert(S featureData, boolean overwrite, SaveMode saveMode, Map<String,String> writeOptions, JobConfiguration jobConfiguration)
<S> void	insert(S featureData, JobConfiguration jobConfiguration)
<S> void	insert(S featureData, Map<String,String> writeOptions)
<S> Object	insertStream(S featureData)
<S> Object	insertStream(S featureData, Map<String,String> writeOptions)
<S> Object	insertStream(S featureData, String queryName)
<S> Object	insertStream(S featureData, String queryName, Map<String,String> writeOptions)
<S> Object	insertStream(S featureData, String queryName, String outputMode)
<S> Object	insertStream(S featureData, String queryName, String outputMode, boolean awaitTermination, Long timeout)
<S> Object	insertStream(S featureData, String queryName, String outputMode, boolean awaitTermination, Long timeout, String checkpointLocation)
<S> Object	insertStream(S featureData, String queryName, String outputMode, boolean awaitTermination, Long timeout, String checkpointLocation, Map<String,String> writeOptions)
<S> Object	insertStream(S featureData, String queryName, String outputMode, boolean awaitTermination, Long timeout, String checkpointLocation, Map<String,String> writeOptions, JobConfiguration jobConfiguration)
<S> Object	insertStream(S featureData, String queryName, String outputMode, String checkpointLocation)
Object	read() Reads Feature group data.
Object	read(boolean online)
Object	read(boolean online, Map<String,String> readOptions)
Object	read(String wallclockTime) Reads Feature group data at a specific point in time.
Object	read(String wallclockTime, Map<String,String> readOptions)
Object	readChanges(String wallclockStartTime, String wallclockEndTime) Deprecated. `readChanges` method is deprecated. Use `asOf(wallclockEndTime, wallclockStartTime).read()` instead.
Object	readChanges(String wallclockStartTime, String wallclockEndTime, Map<String,String> readOptions) Deprecated.
<S> void	save(S featureData, Map<String,String> writeOptions) Deprecated.
<S> void	save(S featureData, Map<String,String> writeOptions, JobConfiguration jobConfiguration) Deprecated.
void	setDeltaStreamerJobConf(DeltaStreamerJobConf deltaStreamerJobConf)
void	setOfflineStorageConnector(StorageConnector offlineStorageConnector)
void	setOnlineEnabled(Boolean onlineEnabled)
void	setOnlineStorageConnector(StorageConnector onlineStorageConnector)
void	setOnlineTopicName(String onlineTopicName)
void	setStatisticColumns(List<String> statisticColumns)

Methods inherited from class com.logicalclocks.hsfs.metadata.FeatureGroupBase

addTag, appendFeatures, appendFeatures, computeStatistics, delete, deleteTag, filter, filter, getCreated, getCreator, getDescription, getEventTime, getExpectationsNames, getFeature, getFeatures, getFeatureStore, getId, getLocation, getName, getPrimaryKeys, getStatistics, getStatistics, getStatisticsConfig, getSubject, getTag, getTags, getTimeTravelFormat, getType, getVersion, select, selectAll, selectAll, selectExcept, selectExceptFeatures, selectFeatures, setDescription, setEventTime, setExpectationsNames, setFeatures, setFeatureStore, setId, setLocation, setName, setStatisticsConfig, setTimeTravelFormat, setType, setVersion, unloadSubject, updateDescription, updateFeatureDescription, updateFeatures, updateFeatures, updateStatisticsConfig

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

StreamFeatureGroup

public StreamFeatureGroup(FeatureStore featureStore,
                          @NonNull
                          @NonNull String name,
                          Integer version,
                          String description,
                          List<String> primaryKeys,
                          List<String> partitionKeys,
                          String hudiPrecombineKey,
                          boolean onlineEnabled,
                          List<Feature> features,
                          StatisticsConfig statisticsConfig,
                          String onlineTopicName,
                          String eventTime)

StreamFeatureGroup

public StreamFeatureGroup()

StreamFeatureGroup

public StreamFeatureGroup(Integer id,
                          String description,
                          List<Feature> features)

StreamFeatureGroup

public StreamFeatureGroup(FeatureStore featureStore,
                          int id)

StreamFeatureGroup

public StreamFeatureGroup(Boolean onlineEnabled,
                          StorageConnector onlineStorageConnector,
                          StorageConnector offlineStorageConnector,
                          List<String> statisticColumns,
                          List<String> partitionKeys,
                          String hudiPrecombineKey,
                          String onlineTopicName,
                          DeltaStreamerJobConf deltaStreamerJobConf)

Method Detail

read

public Object read()
            throws FeatureStoreException,
                   IOException

Reads Feature group data.

Overrides:

read in class FeatureGroupBase

Returns:

DataFrame.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

read

public Object read(boolean online)
            throws FeatureStoreException,
                   IOException

Throws:

FeatureStoreException

IOException

read

public Object read(boolean online,
                   Map<String,String> readOptions)
            throws FeatureStoreException,
                   IOException

Throws:

FeatureStoreException

IOException

read

public Object read(String wallclockTime)
            throws FeatureStoreException,
                   IOException,
                   ParseException

Reads Feature group data at a specific point in time.

Parameters:

wallclockTime - point in time.

Returns:

DataFrame.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

ParseException - ParseException

read

public Object read(String wallclockTime,
                   Map<String,String> readOptions)
            throws FeatureStoreException,
                   IOException,
                   ParseException

Throws:

FeatureStoreException

IOException

ParseException

readChanges

@Deprecated
public Object readChanges(String wallclockStartTime,
                                      String wallclockEndTime)
                               throws FeatureStoreException,
                                      IOException,
                                      ParseException

Deprecated. `readChanges` method is deprecated. Use `asOf(wallclockEndTime, wallclockStartTime).read()` instead.

Reads changes that occurred between specified points in time.

Parameters:

wallclockStartTime - start date.

wallclockEndTime - end date.

Returns:

DataFrame.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

ParseException - ParseException

readChanges

@Deprecated
public Object readChanges(String wallclockStartTime,
                                      String wallclockEndTime,
                                      Map<String,String> readOptions)
                               throws FeatureStoreException,
                                      IOException,
                                      ParseException

Deprecated.

Throws:

FeatureStoreException

IOException

ParseException

asOf

public Query asOf(String wallclockTime)
           throws FeatureStoreException,
                  ParseException

Get Query object to retrieve all features of the group at a point in the past. This method selects all features in the feature group and returns a Query object at the specified point in time. This can then either be read into a Dataframe or used further to perform joins or construct a training dataset.

Parameters:

wallclockTime - Datetime string. The String should be formatted in one of the following formats `%Y%m%d`, `%Y%m%d%H`, `%Y%m%d%H%M`, or `%Y%m%d%H%M%S`.

Returns:

Query. The query object with the applied time travel condition

Throws:

FeatureStoreException - FeatureStoreException

ParseException - ParseException

asOf

public Query asOf(String wallclockTime,
                  String excludeUntil)
           throws FeatureStoreException,
                  ParseException

Get Query object to retrieve all features of the group at a point in the past. This method selects all features in the feature group and returns a Query object at the specified point in time. This can then either be read into a Dataframe or used further to perform joins or construct a training dataset.

Parameters:

wallclockTime - Datetime string. The String should be formatted in one of the following formats `%Y%m%d`, `%Y%m%d%H`, `%Y%m%d%H%M`, or `%Y%m%d%H%M%S`.

excludeUntil - Datetime string. The String should be formatted in one of the following formats `%Y%m%d`, `%Y%m%d%H`, `%Y%m%d%H%M`, or `%Y%m%d%H%M%S`.

Returns:

Query. The query object with the applied time travel condition

Throws:

FeatureStoreException - FeatureStoreException

ParseException - ParseException

save

@Deprecated
public <S> void save(S featureData,
                                 Map<String,String> writeOptions)
                          throws FeatureStoreException,
                                 IOException,
                                 ParseException

Deprecated.

Throws:

FeatureStoreException

IOException

ParseException

save

@Deprecated
public <S> void save(S featureData,
                                 Map<String,String> writeOptions,
                                 JobConfiguration jobConfiguration)
                          throws FeatureStoreException,
                                 IOException,
                                 ParseException

Deprecated.

Throws:

FeatureStoreException

IOException

ParseException

insert

public <S> void insert(S featureData)
                throws FeatureStoreException,
                       IOException,
                       ParseException

Throws:

FeatureStoreException

IOException

ParseException

insert

public <S> void insert(S featureData,
                       Map<String,String> writeOptions)
                throws FeatureStoreException,
                       IOException,
                       ParseException

Throws:

FeatureStoreException

IOException

ParseException

insert

public <S> void insert(S featureData,
                       JobConfiguration jobConfiguration)
                throws FeatureStoreException,
                       IOException,
                       ParseException

Throws:

FeatureStoreException

IOException

ParseException

insert

public <S> void insert(S featureData,
                       boolean overwrite,
                       SaveMode saveMode,
                       Map<String,String> writeOptions)
                throws FeatureStoreException,
                       IOException,
                       ParseException

Throws:

FeatureStoreException

IOException

ParseException

insert

public <S> void insert(S featureData,
                       boolean overwrite,
                       SaveMode saveMode,
                       JobConfiguration jobConfiguration)
                throws FeatureStoreException,
                       IOException,
                       ParseException

Throws:

FeatureStoreException

IOException

ParseException

insert

public <S> void insert(S featureData,
                       boolean overwrite,
                       SaveMode saveMode,
                       Map<String,String> writeOptions,
                       JobConfiguration jobConfiguration)
                throws FeatureStoreException,
                       IOException,
                       ParseException

Throws:

FeatureStoreException

IOException

ParseException

insertStream

public <S> Object insertStream(S featureData)
                        throws FeatureStoreException,
                               IOException,
                               ParseException

Throws:

FeatureStoreException

IOException

ParseException

insertStream

public <S> Object insertStream(S featureData,
                               String queryName)
                        throws FeatureStoreException,
                               IOException,
                               ParseException

Throws:

FeatureStoreException

IOException

ParseException

insertStream

public <S> Object insertStream(S featureData,
                               Map<String,String> writeOptions)
                        throws FeatureStoreException,
                               IOException,
                               ParseException

Throws:

FeatureStoreException

IOException

ParseException

insertStream

public <S> Object insertStream(S featureData,
                               String queryName,
                               Map<String,String> writeOptions)
                        throws FeatureStoreException,
                               IOException,
                               ParseException

Throws:

FeatureStoreException

IOException

ParseException

insertStream

public <S> Object insertStream(S featureData,
                               String queryName,
                               String outputMode)
                        throws FeatureStoreException,
                               IOException,
                               ParseException

Throws:

FeatureStoreException

IOException

ParseException

insertStream

public <S> Object insertStream(S featureData,
                               String queryName,
                               String outputMode,
                               String checkpointLocation)
                        throws FeatureStoreException,
                               IOException,
                               ParseException

Throws:

FeatureStoreException

IOException

ParseException

insertStream

public <S> Object insertStream(S featureData,
                               String queryName,
                               String outputMode,
                               boolean awaitTermination,
                               Long timeout)
                        throws FeatureStoreException,
                               IOException,
                               ParseException

Throws:

FeatureStoreException

IOException

ParseException

insertStream

public <S> Object insertStream(S featureData,
                               String queryName,
                               String outputMode,
                               boolean awaitTermination,
                               Long timeout,
                               String checkpointLocation)
                        throws FeatureStoreException,
                               IOException,
                               ParseException

Throws:

FeatureStoreException

IOException

ParseException

insertStream

public <S> Object insertStream(S featureData,
                               String queryName,
                               String outputMode,
                               boolean awaitTermination,
                               Long timeout,
                               String checkpointLocation,
                               Map<String,String> writeOptions)
                        throws FeatureStoreException,
                               IOException,
                               ParseException

Throws:

FeatureStoreException

IOException

ParseException

insertStream

public <S> Object insertStream(S featureData,
                               String queryName,
                               String outputMode,
                               boolean awaitTermination,
                               Long timeout,
                               String checkpointLocation,
                               Map<String,String> writeOptions,
                               JobConfiguration jobConfiguration)

commitDeleteRecord

public <S> void commitDeleteRecord(S featureData)
                            throws FeatureStoreException,
                                   IOException,
                                   ParseException

Throws:

FeatureStoreException

IOException

ParseException

commitDeleteRecord

public <S> void commitDeleteRecord(S featureData,
                                   Map<String,String> writeOptions)
                            throws FeatureStoreException,
                                   IOException,
                                   ParseException

Throws:

FeatureStoreException

IOException

ParseException

commitDetails

public Map<Long,Map<String,String>> commitDetails()
                                           throws IOException,
                                                  FeatureStoreException,
                                                  ParseException

Return commit details.

Returns:

commit details.

Throws:

FeatureStoreException - FeatureStoreException

IOException - FeatureStoreException

ParseException - ParseException

commitDetails

public Map<Long,Map<String,String>> commitDetails(Integer limit)
                                           throws IOException,
                                                  FeatureStoreException,
                                                  ParseException

Return commit details.

Parameters:

limit - number of commits to return.

Returns:

commit details.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

ParseException - ParseException

commitDetails

public Map<Long,Map<String,String>> commitDetails(String wallclockTime)
                                           throws IOException,
                                                  FeatureStoreException,
                                                  ParseException

Return commit details.

Parameters:

wallclockTime - point in time.

Returns:

commit details.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

ParseException - ParseException

commitDetails

public Map<Long,Map<String,String>> commitDetails(String wallclockTime,
                                                  Integer limit)
                                           throws IOException,
                                                  FeatureStoreException,
                                                  ParseException

Return commit details.

Parameters:

wallclockTime - point in time.

limit - number of commits to return.

Returns:

commit details.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

ParseException - ParseException

getAvroSchema

public String getAvroSchema()
                     throws FeatureStoreException,
                            IOException

Throws:

FeatureStoreException

IOException

getComplexFeatures

public List<String> getComplexFeatures()

Overrides:

getComplexFeatures in class FeatureGroupBase

getFeatureAvroSchema

public String getFeatureAvroSchema(String featureName)
                            throws FeatureStoreException,
                                   IOException

Overrides:

getFeatureAvroSchema in class FeatureGroupBase

Throws:

FeatureStoreException

IOException

getEncodedAvroSchema

public String getEncodedAvroSchema()
                            throws FeatureStoreException,
                                   IOException

Overrides:

getEncodedAvroSchema in class FeatureGroupBase

Throws:

FeatureStoreException

IOException

getDeserializedAvroSchema

public org.apache.avro.Schema getDeserializedAvroSchema()
                                                 throws FeatureStoreException,
                                                        IOException

Overrides:

getDeserializedAvroSchema in class FeatureGroupBase

Throws:

FeatureStoreException

IOException

builder

public static StreamFeatureGroup.StreamFeatureGroupBuilder builder()

getOnlineEnabled

public Boolean getOnlineEnabled()

setOnlineEnabled

public void setOnlineEnabled(Boolean onlineEnabled)

getOnlineStorageConnector

public StorageConnector getOnlineStorageConnector()

setOnlineStorageConnector

public void setOnlineStorageConnector(StorageConnector onlineStorageConnector)

getOfflineStorageConnector

public StorageConnector getOfflineStorageConnector()

setOfflineStorageConnector

public void setOfflineStorageConnector(StorageConnector offlineStorageConnector)

getStatisticColumns

public List<String> getStatisticColumns()

setStatisticColumns

public void setStatisticColumns(List<String> statisticColumns)

getOnlineTopicName

public String getOnlineTopicName()

Overrides:

getOnlineTopicName in class FeatureGroupBase

setOnlineTopicName

public void setOnlineTopicName(String onlineTopicName)

setDeltaStreamerJobConf

public void setDeltaStreamerJobConf(DeltaStreamerJobConf deltaStreamerJobConf)