com.logicalclocks.hsfs

Class TrainingDataset

java.lang.Object

com.logicalclocks.hsfs.TrainingDataset

public class TrainingDataset
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	TrainingDataset.TrainingDatasetBuilder

Constructor Summary

Constructors

Constructor and Description
TrainingDataset()
TrainingDataset(@NonNull String name, Integer version, String description, DataFormat dataFormat, Boolean coalesce, StorageConnector storageConnector, String location, List<Split> splits, String trainSplit, Long seed, FeatureStore featureStore, StatisticsConfig statisticsConfig, List<String> label, String eventStartTime, String eventEndTime, TrainingDatasetType trainingDatasetType, Float validationSize, Float testSize, String trainStart, String trainEnd, String validationStart, String validationEnd, String testStart, String testEnd, Integer timeSplitSize, FilterLogic extraFilterLogic, Filter extraFilter)

Method Summary

Modifier and Type	Method and Description
void	addTag(String name, Object value) Add name/value tag to the training dataset.
static TrainingDataset.TrainingDatasetBuilder	builder()
Statistics	computeStatistics() Recompute the statistics for the entire training dataset and save them to the feature store.
void	delete() Delete training dataset and all associated metadata.
void	deleteTag(String name) Delete a tag of the training dataset.
Boolean	getCoalesce()
DataFormat	getDataFormat()
String	getDescription()
Date	getEventEndTime()
Date	getEventStartTime()
FilterLogic	getExtraFilter()
List<TrainingDatasetFeature>	getFeatures()
FeatureStore	getFeatureStore()
Integer	getId()
List<String>	getLabel()
String	getLocation()
String	getName()
String	getQuery()
String	getQuery(boolean withLabel)
String	getQuery(Storage storage)
String	getQuery(Storage storage, boolean withLabel)
Query	getQueryInt()
Long	getSeed()
HashSet<String>	getServingKeys() Set of primary key names that is used as keys in input dict object for `get_serving_vector` method.
List<Object>	getServingVector(Map<String,Object> entry) Retrieve feature vector from online feature store.
List<Object>	getServingVector(Map<String,Object> entry, boolean external) Retrieve feature vector from online feature store.
List<List<Object>>	getServingVectors(Map<String,List<Object>> entry)
List<List<Object>>	getServingVectors(Map<String,List<Object>> entry, boolean external)
List<Split>	getSplits()
Statistics	getStatistics() Get the last statistics commit for the training dataset.
Statistics	getStatistics(String commitTime) Get the statistics of a specific commit time for the training dataset.
StatisticsConfig	getStatisticsConfig()
StorageConnector	getStorageConnector()
Object	getTag(String name) Get a single tag value of the training dataset.
Map<String,Object>	getTags() Get all tags of the training dataset.
TrainingDatasetType	getTrainingDatasetType()
String	getTrainSplit()
String	getType()
Integer	getVersion()
void	initPreparedStatement() Initialise and cache parametrised prepared statement to retrieve feature vector from online feature store.
void	initPreparedStatement(boolean external) Initialise and cache parametrised prepared statement to retrieve feature vector from online feature store.
void	initPreparedStatement(boolean external, boolean batch) Initialise and cache parametrised prepared statement to retrieve batch feature vectors from online feature store.
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>	read() Read the content of the training dataset.
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>	read(Map<String,String> readOptions) Read the content of the training dataset.
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>	read(String split) Read all a single split from the training dataset.
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row>	read(String split, Map<String,String> readOptions) Read a single split from the training dataset.
void	save(Query query) Create the training dataset based on the content of the feature store query.
void	save(Query query, Map<String,String> writeOptions) Create the training dataset based on the content of the feature store query.
void	setCoalesce(Boolean coalesce)
void	setDataFormat(DataFormat dataFormat)
void	setDescription(String description)
void	setEventEndTime(Date eventEndTime)
void	setEventStartTime(Date eventStartTime)
void	setExtraFilter(FilterLogic extraFilter)
void	setFeatures(List<TrainingDatasetFeature> features)
void	setFeatureStore(FeatureStore featureStore)
void	setId(Integer id)
void	setLabel(List<String> label)
void	setLocation(String location)
void	setName(String name)
void	setQueryInt(Query queryInt)
void	setSeed(Long seed)
void	setSplits(List<Split> splits)
void	setStatisticsConfig(StatisticsConfig statisticsConfig)
void	setStorageConnector(StorageConnector storageConnector)
void	setTrainingDatasetType(TrainingDatasetType trainingDatasetType)
void	setTrainSplit(String trainSplit)
void	setType(String type)
void	setVersion(Integer version)
void	show(int numRows) Show numRows from the training dataset (across all splits).
void	updateStatisticsConfig() Update the statistics configuration of the training dataset.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

TrainingDataset

public TrainingDataset(@NonNull
                       @NonNull String name,
                       Integer version,
                       String description,
                       DataFormat dataFormat,
                       Boolean coalesce,
                       StorageConnector storageConnector,
                       String location,
                       List<Split> splits,
                       String trainSplit,
                       Long seed,
                       FeatureStore featureStore,
                       StatisticsConfig statisticsConfig,
                       List<String> label,
                       String eventStartTime,
                       String eventEndTime,
                       TrainingDatasetType trainingDatasetType,
                       Float validationSize,
                       Float testSize,
                       String trainStart,
                       String trainEnd,
                       String validationStart,
                       String validationEnd,
                       String testStart,
                       String testEnd,
                       Integer timeSplitSize,
                       FilterLogic extraFilterLogic,
                       Filter extraFilter)
                throws FeatureStoreException,
                       ParseException

Throws:

FeatureStoreException

ParseException

TrainingDataset

public TrainingDataset()

Method Detail

save

public void save(Query query)
          throws FeatureStoreException,
                 IOException

Create the training dataset based on the content of the feature store query.

Parameters:

query - the query to save as training dataset

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

save

public void save(Query query,
                 Map<String,String> writeOptions)
          throws FeatureStoreException,
                 IOException

Create the training dataset based on the content of the feature store query.

Parameters:

query - the query to save as training dataset

writeOptions - options to pass to the Spark write operation

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

read

public org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> read()
                                                            throws FeatureStoreException,
                                                                   IOException

Read the content of the training dataset.

Returns:

Spark Dataset containing the training dataset data

Throws:

FeatureStoreException - if the training dataset has splits and the split was not specified

IOException - IOException

read

public org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> read(Map<String,String> readOptions)
                                                            throws FeatureStoreException,
                                                                   IOException

Read the content of the training dataset.

Parameters:

readOptions - options to pass to the Spark read operation

Returns:

Spark Dataset containing the training dataset data

Throws:

FeatureStoreException - if the training dataset has splits and the split was not specified

IOException - IOException

read

public org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> read(String split)
                                                            throws FeatureStoreException,
                                                                   IOException

Read all a single split from the training dataset.

Parameters:

split - the split name

Returns:

Spark Dataset containing the training dataset data

Throws:

FeatureStoreException - if the training dataset has splits and the split was not specified

IOException - IOException

read

public org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> read(String split,
                                                                   Map<String,String> readOptions)
                                                            throws FeatureStoreException,
                                                                   IOException

Read a single split from the training dataset.

Parameters:

split - the split name

readOptions - options to pass to the Spark read operation

Returns:

Spark Dataset containing the training dataset data

Throws:

FeatureStoreException - if the training dataset has splits and the split was not specified

IOException - IOException

show

public void show(int numRows)
          throws FeatureStoreException,
                 IOException

Show numRows from the training dataset (across all splits).

Parameters:

numRows - number of rows to display

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

computeStatistics

public Statistics computeStatistics()
                             throws FeatureStoreException,
                                    IOException

Recompute the statistics for the entire training dataset and save them to the feature store.

Returns:

statistics object of computed statistics

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

updateStatisticsConfig

public void updateStatisticsConfig()
                            throws FeatureStoreException,
                                   IOException

Update the statistics configuration of the training dataset. Change the `enabled`, `histograms`, `correlations` or `columns` attributes and persist the changes by calling this method.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

getStatistics

public Statistics getStatistics()
                         throws FeatureStoreException,
                                IOException

Get the last statistics commit for the training dataset.

Returns:

statistics object of latest commit

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

getStatistics

public Statistics getStatistics(String commitTime)
                         throws FeatureStoreException,
                                IOException

Get the statistics of a specific commit time for the training dataset.

Parameters:

commitTime - commit time in the format "YYYYMMDDhhmmss"

Returns:

statistics object for the commit time

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

addTag

public void addTag(String name,
                   Object value)
            throws FeatureStoreException,
                   IOException

Add name/value tag to the training dataset.

Parameters:

name - name of the tag

value - value of the tag. The value of a tag can be any valid json - primitives, arrays or json objects

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

getTags

public Map<String,Object> getTags()
                           throws FeatureStoreException,
                                  IOException

Get all tags of the training dataset.

Returns:

a map of tag name and values. The value of a tag can be any valid json - primitives, arrays or json objects

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

getTag

public Object getTag(String name)
              throws FeatureStoreException,
                     IOException

Get a single tag value of the training dataset.

Parameters:

name - name of the tag

Returns:

The value of a tag can be any valid json - primitives, arrays or json objects

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

deleteTag

public void deleteTag(String name)
               throws FeatureStoreException,
                      IOException

Delete a tag of the training dataset.

Parameters:

name - name of the tag to be deleted

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

getQuery

public String getQuery()
                throws FeatureStoreException,
                       IOException

Throws:

FeatureStoreException

IOException

getQuery

public String getQuery(boolean withLabel)
                throws FeatureStoreException,
                       IOException

Throws:

FeatureStoreException

IOException

getQuery

public String getQuery(Storage storage)
                throws FeatureStoreException,
                       IOException

Throws:

FeatureStoreException

IOException

getQuery

public String getQuery(Storage storage,
                       boolean withLabel)
                throws FeatureStoreException,
                       IOException

Throws:

FeatureStoreException

IOException

getLabel

public List<String> getLabel()

setLabel

public void setLabel(List<String> label)

initPreparedStatement

public void initPreparedStatement()
                           throws SQLException,
                                  IOException,
                                  FeatureStoreException,
                                  ClassNotFoundException

Initialise and cache parametrised prepared statement to retrieve feature vector from online feature store.

Throws:

SQLException - SQLException

IOException - IOException

FeatureStoreException - FeatureStoreException

ClassNotFoundException - ClassNotFoundException

initPreparedStatement

public void initPreparedStatement(boolean external)
                           throws SQLException,
                                  IOException,
                                  FeatureStoreException,
                                  ClassNotFoundException

Initialise and cache parametrised prepared statement to retrieve feature vector from online feature store.

Parameters:

external - whether is from external client or not

Throws:

SQLException - SQLException

IOException - IOException

FeatureStoreException - FeatureStoreException

ClassNotFoundException - ClassNotFoundException

initPreparedStatement

public void initPreparedStatement(boolean external,
                                  boolean batch)
                           throws SQLException,
                                  IOException,
                                  FeatureStoreException,
                                  ClassNotFoundException

Initialise and cache parametrised prepared statement to retrieve batch feature vectors from online feature store.

Parameters:

external - whether is from external client or not

batch - whether to initialise feature vector for batch retrieval

Throws:

SQLException - SQLException

IOException - IOException

FeatureStoreException - FeatureStoreException

ClassNotFoundException - ClassNotFoundException

getServingVector

public List<Object> getServingVector(Map<String,Object> entry)
                              throws SQLException,
                                     FeatureStoreException,
                                     IOException,
                                     ClassNotFoundException

Retrieve feature vector from online feature store.

Parameters:

entry - Map object with kes as primary key names of the training dataset features groups and values as corresponding ids to retrieve feature vector from online feature store.

Returns:

list of feature values sorted according to provided primary keys.

Throws:

SQLException - SQLException

IOException - IOException

FeatureStoreException - FeatureStoreException

ClassNotFoundException - ClassNotFoundException

getServingVector

public List<Object> getServingVector(Map<String,Object> entry,
                                     boolean external)
                              throws SQLException,
                                     FeatureStoreException,
                                     IOException,
                                     ClassNotFoundException

Retrieve feature vector from online feature store.

Parameters:

entry - Map object with kes as primary key names of the training dataset features groups and values as corresponding ids to retrieve feature vector from online feature store.

external - If true, the connection to the online feature store will be established using the hostname provided in the hsfs.connection() setup.

Returns:

list of feature values sorted according to provided primary keys.

Throws:

SQLException - SQLException

IOException - IOException

FeatureStoreException - FeatureStoreException

ClassNotFoundException - ClassNotFoundException

getServingVectors

public List<List<Object>> getServingVectors(Map<String,List<Object>> entry)
                                     throws SQLException,
                                            FeatureStoreException,
                                            IOException,
                                            ClassNotFoundException

Throws:

SQLException

FeatureStoreException

IOException

ClassNotFoundException

getServingVectors

public List<List<Object>> getServingVectors(Map<String,List<Object>> entry,
                                            boolean external)
                                     throws SQLException,
                                            FeatureStoreException,
                                            IOException,
                                            ClassNotFoundException

Throws:

SQLException

FeatureStoreException

IOException

ClassNotFoundException

delete

public void delete()
            throws FeatureStoreException,
                   IOException

Delete training dataset and all associated metadata. Note that this operation drops only files which were materialized in HopsFS. If you used a Storage Connector for a cloud storage such as S3, the data will not be deleted, but you will not be able to track it anymore from the Feature Store. This operation drops all metadata associated with this version of the training dataset and and the materialized data in HopsFS.

Throws:

FeatureStoreException - FeatureStoreException

IOException - IOException

getServingKeys

public HashSet<String> getServingKeys()
                               throws SQLException,
                                      IOException,
                                      FeatureStoreException,
                                      ClassNotFoundException

Set of primary key names that is used as keys in input dict object for `get_serving_vector` method.

Returns:

Set of serving keys

Throws:

SQLException - SQLException

IOException - IOException

FeatureStoreException - FeatureStoreException

ClassNotFoundException - ClassNotFoundException

builder

public static TrainingDataset.TrainingDatasetBuilder builder()

getId

public Integer getId()

setId

public void setId(Integer id)

getName

public String getName()

setName

public void setName(String name)

getVersion

public Integer getVersion()

setVersion

public void setVersion(Integer version)

getDescription

public String getDescription()

setDescription

public void setDescription(String description)

getDataFormat

public DataFormat getDataFormat()

setDataFormat

public void setDataFormat(DataFormat dataFormat)

getCoalesce

public Boolean getCoalesce()

setCoalesce

public void setCoalesce(Boolean coalesce)

getTrainingDatasetType

public TrainingDatasetType getTrainingDatasetType()

setTrainingDatasetType

public void setTrainingDatasetType(TrainingDatasetType trainingDatasetType)

getFeatures

public List<TrainingDatasetFeature> getFeatures()

setFeatures

public void setFeatures(List<TrainingDatasetFeature> features)

getFeatureStore

public FeatureStore getFeatureStore()

setFeatureStore

public void setFeatureStore(FeatureStore featureStore)

getStorageConnector

public StorageConnector getStorageConnector()

setStorageConnector

public void setStorageConnector(StorageConnector storageConnector)

getLocation

public String getLocation()

setLocation

public void setLocation(String location)

getSeed

public Long getSeed()

setSeed

public void setSeed(Long seed)

getSplits

public List<Split> getSplits()

setSplits

public void setSplits(List<Split> splits)

getTrainSplit

public String getTrainSplit()

setTrainSplit

public void setTrainSplit(String trainSplit)

getStatisticsConfig

public StatisticsConfig getStatisticsConfig()

setStatisticsConfig

public void setStatisticsConfig(StatisticsConfig statisticsConfig)

getQueryInt

public Query getQueryInt()

setQueryInt

public void setQueryInt(Query queryInt)

getEventStartTime

public Date getEventStartTime()

setEventStartTime

public void setEventStartTime(Date eventStartTime)

getEventEndTime

public Date getEventEndTime()

setEventEndTime

public void setEventEndTime(Date eventEndTime)

getExtraFilter

public FilterLogic getExtraFilter()

setExtraFilter

public void setExtraFilter(FilterLogic extraFilter)

getType

public String getType()

setType

public void setType(String type)