From 1ba2ff30862a7161549ab9fb49e0d9e3d550de3a Mon Sep 17 00:00:00 2001 From: shacharPash Date: Tue, 6 Sep 2022 18:15:44 +0300 Subject: [PATCH 01/28] Add Alias command --- src/NRedisStack/Search/SearchCommands.cs | 102 ++++++++++++++++++++++- 1 file changed, 100 insertions(+), 2 deletions(-) diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index 875bb43b..e7b26da9 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -9,9 +9,107 @@ public SearchCommands(IDatabase db) { _db = db; } - public RedisResult Info(RedisValue index) + + /// + /// Returns a list of all existing indexes. + /// + /// Array with index names. + /// + public RedisResult[] _List() { - return _db.Execute(FT.INFO, index); + return _db.Execute(FT._LIST).ToArray(); } + + // TODO: Aggregate + + /// + /// Add an alias to an index. + /// + /// Alias to be added to an index. + /// The index name. + /// if executed correctly, error otherwise + /// + public bool AliasAdd(string alias, string index) + { + return _db.Execute(FT.ALIASADD, alias, index).OKtoBoolean(); + } + + /// + /// Add an alias to an index. + /// + /// Alias to be added to an index. + /// The index name. + /// if executed correctly, error otherwise + /// + public async Task AliasAddAsync(string alias, string index) + { + return (await _db.ExecuteAsync(FT.ALIASADD, alias, index)).OKtoBoolean(); + } + + /// + /// Remove an alias to an index. + /// + /// Alias to be removed. + /// if executed correctly, error otherwise + /// + public bool AliasDel(string alias) + { + return _db.Execute(FT.ALIASDEL, alias).OKtoBoolean(); + } + + /// + /// Remove an alias to an index. + /// + /// Alias to be removed. + /// if executed correctly, error otherwise + /// + public async Task AliasDelAsync(string alias) + { + return (await _db.ExecuteAsync(FT.ALIASDEL, alias)).OKtoBoolean(); + } + + /// + /// Add an alias to an index. If the alias is already associated with another index, + /// FT.ALIASUPDATE removes the alias association with the previous index. + /// + /// Alias to be removed. + /// The index name. + /// if executed correctly, error otherwise + /// + public bool AliasUpdate(string alias, string index) + { + return _db.Execute(FT.ALIASUPDATE, alias, index).OKtoBoolean(); + } + + /// + /// Add an alias to an index. If the alias is already associated with another index, + /// FT.ALIASUPDATE removes the alias association with the previous index. + /// + /// Alias to be removed. + /// The index name. + /// if executed correctly, error otherwise + /// + public async Task AliasUpdateAsync(string alias, string index) + { + return (await _db.ExecuteAsync(FT.ALIASUPDATE, alias, index)).OKtoBoolean(); + } + + // TODO: finish this: + // /// + // /// Add a new attribute to the index + // /// + // /// Alias to be removed. + // /// The index name. + // /// if executed correctly, error otherwise + // /// + // public bool Alter(string alias, string index) + // { + // return _db.Execute(FT.ALIASUPDATE, alias, index).OKtoBoolean(); + // } + + // public RedisResult Info(RedisValue index) + // { + // return _db.Execute(FT.INFO, index); + // } } } \ No newline at end of file From 6edac52712b70c6cdc40567495e1b00ef134e008 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Wed, 7 Sep 2022 19:22:32 +0300 Subject: [PATCH 02/28] Start FT.CERATE Command --- .../Search/DataTypes/SearchInformation.cs | 34 +++++++++++++++++++ .../Extensions/IndexDataTypeExtensions.cs | 22 ++++++++++++ .../Search/FT.CREATE/CreateCommand.cs | 23 +++++++++++++ .../Search/FT.CREATE/FTCreateParams.cs | 24 +++++++++++++ .../Search/Literals/AttributeOptions.cs | 12 +++++++ .../Search/Literals/CommandArgs.cs | 18 +++++++++- .../Search/Literals/Enums/IndexDataType.cs | 8 +++++ src/NRedisStack/Search/SearchCommands.cs | 33 +++++++++--------- 8 files changed, 157 insertions(+), 17 deletions(-) create mode 100644 src/NRedisStack/Search/DataTypes/SearchInformation.cs create mode 100644 src/NRedisStack/Search/Extensions/IndexDataTypeExtensions.cs create mode 100644 src/NRedisStack/Search/FT.CREATE/CreateCommand.cs create mode 100644 src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs create mode 100644 src/NRedisStack/Search/Literals/AttributeOptions.cs create mode 100644 src/NRedisStack/Search/Literals/Enums/IndexDataType.cs diff --git a/src/NRedisStack/Search/DataTypes/SearchInformation.cs b/src/NRedisStack/Search/DataTypes/SearchInformation.cs new file mode 100644 index 00000000..3f4f34b2 --- /dev/null +++ b/src/NRedisStack/Search/DataTypes/SearchInformation.cs @@ -0,0 +1,34 @@ +namespace NRedisStack.Search.DataTypes +{ + /// + /// This class represents the response for SEARCH.INFO command. + /// This object has Read-only properties and cannot be generated outside a SEARCH.INFO response. + /// + public class SearchInformation + { + // TODO: work on it with someone from Search team + // public string IndexName { get; private set; } + // public string[] IndexOptions { get; private set; } + // public long IndexDefinition { get; private set; } + // public long UnmergedNodes { get; private set; } + // public double MergedWeight { get; private set; } + // public double UnmergedWeight { get; private set; } + + // public long TotalCompressions { get; private set; } + + + // internal SearchInformation(long compression, long capacity, long mergedNodes, + // long unmergedNodes, double mergedWeight, + // double unmergedWeight, long totalCompressions) + + // { + // Compression = compression; + // Capacity = capacity; + // MergedNodes = mergedNodes; + // UnmergedNodes = unmergedNodes; + // MergedWeight = mergedWeight; + // UnmergedWeight = unmergedWeight; + // TotalCompressions = totalCompressions; + // } + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/Extensions/IndexDataTypeExtensions.cs b/src/NRedisStack/Search/Extensions/IndexDataTypeExtensions.cs new file mode 100644 index 00000000..13d59c3c --- /dev/null +++ b/src/NRedisStack/Search/Extensions/IndexDataTypeExtensions.cs @@ -0,0 +1,22 @@ +using System; +using NRedisStack.Literals.Enums; + +namespace NRedisStack.Extensions +{ + internal static class IndexIndexDataType + { + public static string AsArg(this IndexDataType dataType) => dataType switch + { + IndexDataType.Hash => "HASH", + IndexDataType.Json => "JSON", + _ => throw new ArgumentOutOfRangeException(nameof(dataType), "Invalid Index DataType"), + }; + + public static IndexDataType AsDataType(string dataType) => dataType switch + { + "HASH" => IndexDataType.Hash, + "JSON" => IndexDataType.Json, + _ => throw new ArgumentOutOfRangeException(nameof(dataType), $"Invalid Index DataType '{dataType}'"), + }; + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/FT.CREATE/CreateCommand.cs b/src/NRedisStack/Search/FT.CREATE/CreateCommand.cs new file mode 100644 index 00000000..8fcff2c1 --- /dev/null +++ b/src/NRedisStack/Search/FT.CREATE/CreateCommand.cs @@ -0,0 +1,23 @@ +using NRedisStack.Literals; +using StackExchange.Redis; +namespace NRedisStack +{ + public class CreateCommand + { + IDatabase _db; + public CreateCommand(IDatabase db) + { + _db = db; + } + + /// + /// Create an index with the given specification. + /// + /// Array with index names. + /// + // public RedisResult[] Create() + // { + // return _db.Execute(FT._LIST).ToArray(); + // } + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs b/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs new file mode 100644 index 00000000..366bca3a --- /dev/null +++ b/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs @@ -0,0 +1,24 @@ +using NRedisStack.Literals.Enums; + +namespace NRedisStack.Search.FT.CREATE +{ + public class FTCreateParams + { + private IndexDataType dataType; + private List prefix; + private string filter; + private string language; + private string languageField; + private double score; + private string scoreField; + private byte[] payloadField; + private bool maxTextFields; + private bool noOffsets; + private long temporary; + private bool noHL; + private bool noFields; + private bool noFreqs; + private List stopwords; + private bool skipInitialScan; + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/Literals/AttributeOptions.cs b/src/NRedisStack/Search/Literals/AttributeOptions.cs new file mode 100644 index 00000000..60a161f9 --- /dev/null +++ b/src/NRedisStack/Search/Literals/AttributeOptions.cs @@ -0,0 +1,12 @@ +namespace NRedisStack.Literals +{ + internal class AttributeOptions + { + public const string SORTABLE = "SORTABLE"; + public const string UNF = "UNF"; + public const string NOSTEM = "NOSTEM"; + public const string NOINDEX = "NOINDEX"; + + //TODO: add all options + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/Literals/CommandArgs.cs b/src/NRedisStack/Search/Literals/CommandArgs.cs index 48e36a47..eddb7134 100644 --- a/src/NRedisStack/Search/Literals/CommandArgs.cs +++ b/src/NRedisStack/Search/Literals/CommandArgs.cs @@ -2,6 +2,22 @@ namespace NRedisStack.Literals { internal class SearchArgs { - + public const string ON_HASH = "ON HASH"; + public const string JSON = "JSON"; + public const string PREFIX = "PREFIX"; + public const string FILTER = "FILTER"; + public const string LANGUAGE = "LANGUAGE"; + public const string LANGUAGE_FIELD = "LANGUAGE_FIELD"; + public const string SCORE = "SCORE"; + public const string SCORE_FIELD = "SCORE_FIELD"; + public const string PAYLOAD_FIELD = "PAYLOAD_FIELD"; + public const string MAXTEXTFIELDS = "MAXTEXTFIELDS"; + public const string TEMPORARY = "TEMPORARY"; + public const string NOOFFSETS = "NOOFFSETS"; + public const string NOHL = "NOHL"; + public const string NOFIELDS = "NOFIELDS"; + public const string NOFREQS = "NOFREQS"; + public const string STOPWORDS = "STOPWORDS"; + public const string SKIPINITIALSCAN = "SKIPINITIALSCAN"; } } \ No newline at end of file diff --git a/src/NRedisStack/Search/Literals/Enums/IndexDataType.cs b/src/NRedisStack/Search/Literals/Enums/IndexDataType.cs new file mode 100644 index 00000000..e4b9ae34 --- /dev/null +++ b/src/NRedisStack/Search/Literals/Enums/IndexDataType.cs @@ -0,0 +1,8 @@ +namespace NRedisStack.Literals.Enums +{ + public enum IndexDataType + { + Json, + Hash, + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index e7b26da9..274373da 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -94,22 +94,23 @@ public async Task AliasUpdateAsync(string alias, string index) return (await _db.ExecuteAsync(FT.ALIASUPDATE, alias, index)).OKtoBoolean(); } - // TODO: finish this: - // /// - // /// Add a new attribute to the index - // /// - // /// Alias to be removed. - // /// The index name. - // /// if executed correctly, error otherwise - // /// - // public bool Alter(string alias, string index) - // { - // return _db.Execute(FT.ALIASUPDATE, alias, index).OKtoBoolean(); - // } + /// + /// Add a new attribute to the index + /// + /// The index name to create. + /// If set, does not scan and index. + /// attribute to add. + /// attribute options. + /// if executed correctly, error otherwise + /// + public bool Alter(string alias, string index) + { + return _db.Execute(FT.ALIASUPDATE, alias, index).OKtoBoolean(); + } - // public RedisResult Info(RedisValue index) - // { - // return _db.Execute(FT.INFO, index); - // } + public RedisResult Info(RedisValue index) + { + return _db.Execute(FT.INFO, index); + } } } \ No newline at end of file From 8d017816656245a11e6f85f51d19af4a68046cf3 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Thu, 8 Sep 2022 18:34:08 +0300 Subject: [PATCH 03/28] Start FTCreateParams class for FT.CERATE --- .../Search/FT.CREATE/FTCreateParams.cs | 301 +++++++++++++++++- 1 file changed, 299 insertions(+), 2 deletions(-) diff --git a/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs b/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs index 366bca3a..af27fc1a 100644 --- a/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs +++ b/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs @@ -1,11 +1,12 @@ +using NRedisStack.Literals; using NRedisStack.Literals.Enums; - +// TODO: look at NRediSearch namespace NRedisStack.Search.FT.CREATE { public class FTCreateParams { private IndexDataType dataType; - private List prefix; + private List prefixes; private string filter; private string language; private string languageField; @@ -20,5 +21,301 @@ public class FTCreateParams private bool noFreqs; private List stopwords; private bool skipInitialScan; + + public FTCreateParams() + { + } + + public static FTCreateParams createParams() + { + return new FTCreateParams(); + } + + /* Currently supports HASH (default) and JSON. To index JSON, you must have the RedisJSON module + installed. + */ + public FTCreateParams on(IndexDataType dataType) + { + this.dataType = dataType; + return this; + } + + /** + * Tells the index which keys it should index. You can add several prefixes to index. + */ + public FTCreateParams Prefix(params string[] prefixes) + { + if (this.prefixes == null) + { + this.prefixes = new List(prefixes.Length); + } + this.prefixes.AddRange(prefixes); + return this; + } + + /** + * This method can be chained to add multiple prefixes. + * + * @see FTCreateParams#prefix(java.lang.params string[]) + */ + public FTCreateParams AddPrefix(string prefix) + { + if (this.prefixes == null) + { + this.prefixes = new List(); + } + this.prefixes.Add(prefix); + return this; + } + + /** + * A filter expression with the full RediSearch aggregation expression language. + */ + public FTCreateParams Filter(string filter) + { + this.filter = filter; + return this; + } + + /** + * Indicates the default language for documents in the index. + */ + public FTCreateParams Language(string defaultLanguage) + { + this.language = defaultLanguage; + return this; + } + + /** + * Document attribute set as the document language. + */ + public FTCreateParams LanguageField(string languageAttribute) + { + this.languageField = languageAttribute; + return this; + } + + /** + * Default score for documents in the index. + */ + public FTCreateParams Score(double defaultScore) + { + this.score = defaultScore; + return this; + } + + /** + * Document attribute that you use as the document rank based on the user ranking. + * Ranking must be between 0.0 and 1.0. + */ + public FTCreateParams ScoreField(string scoreField) + { + this.scoreField = scoreField; + return this; + } + + /** + * Document attribute that you use as a binary safe payload string to the document that can be + * evaluated at query time by a custom scoring function or retrieved to the client. + */ + public FTCreateParams PayloadField(byte[] payloadAttribute) + { + Array.Copy(this.payloadField, payloadAttribute, payloadAttribute.Length); + return this; + } + + /** + * Forces RediSearch to encode indexes as if there were more than 32 text attributes. + */ + public FTCreateParams MaxTextFields() + { + this.maxTextFields = true; + return this; + } + + /** + * Does not store term offsets for documents. It saves memory, but does not allow exact searches + * or highlighting. + */ + public FTCreateParams NoOffsets() + { + this.noOffsets = true; + return this; + } + + /** + * Creates a lightweight temporary index that expires after a specified period of inactivity. + */ + public FTCreateParams Temporary(long seconds) + { + this.temporary = seconds; + return this; + } + + /** + * Conserves storage space and memory by disabling highlighting support. + */ + public FTCreateParams NoHL() + { + this.noHL = true; + return this; + } + + /** + * @see FTCreateParams#noHL() + */ + public FTCreateParams NoHighlights() + { + return NoHL(); + } + + /** + * Does not store attribute bits for each term. It saves memory, but it does not allow filtering + * by specific attributes. + */ + public FTCreateParams NoFields() + { + this.noFields = true; + return this; + } + + /** + * Avoids saving the term frequencies in the index. It saves memory, but does not allow sorting + * based on the frequencies of a given term within the document. + */ + public FTCreateParams NoFreqs() + { + this.noFreqs = true; + return this; + } + + /** + * Sets the index with a custom stopword list, to be ignored during indexing and search time. + */ + public FTCreateParams topwords(params string[] stopwords) + { + this.stopwords = stopwords.ToList(); + return this; + } + + /** + * The index does not have stopwords, not even the default ones. + */ + public FTCreateParams NoStopwords() + { + this.stopwords = new List { }; + return this; + } + + /** + * Does not scan and index. + */ + public FTCreateParams SkipInitialScan() + { + this.skipInitialScan = true; + return this; + } + + public void AddParams(List args) + { + + if (dataType != null) + { + args.Add("ON"); + args.Add(dataType); + } + + if (prefixes != null) + { + args.Add(SearchArgs.PREFIX); + args.Add(prefixes.Count); + foreach(var prefix in prefixes) + if(prefix != null) + args.Add(prefix); + } + + if (filter != null) + { + args.Add(SearchArgs.FILTER); + args.Add(filter); + } + + if (language != null) + { + args.Add(SearchArgs.LANGUAGE); + args.Add(language); + } + if (languageField != null) + { + args.Add(SearchArgs.LANGUAGE_FIELD); + args.Add(languageField); + } + + if (score != null) + { + args.Add(SearchArgs.SCORE); + args.Add(score); + } + if (scoreField != null) + { + args.Add(SearchArgs.SCORE_FIELD); + args.Add(scoreField); + } + + if (payloadField != null) + { + args.Add(SearchArgs.PAYLOAD_FIELD); + args.Add(payloadField); + } + + if (maxTextFields) + { + args.Add(SearchArgs.MAXTEXTFIELDS); + } + //[TEMPORARY seconds] seposed to be here + if (noOffsets) + { + args.Add(SearchArgs.NOOFFSETS); + } + + if (temporary != null) + { + args.Add(SearchArgs.TEMPORARY); + args.Add(temporary); + } + + if (noHL) + { + args.Add(SearchArgs.NOHL); + } + + if (noFields) + { + args.Add(SearchArgs.NOFIELDS); + } + + if (noFreqs) + { + args.Add(SearchArgs.NOFREQS); + } + + if (stopwords != null) + { + args.Add(SearchArgs.STOPWORDS); + args.Add(stopwords.Count); + stopwords.ForEach(w => args.Add(w)); + } + + if (skipInitialScan) + { + args.Add(SearchArgs.SKIPINITIALSCAN); + } + /* + here sepose to be: + SCHEMA field_name [AS alias] TEXT | TAG | NUMERIC | GEO | VECTOR [ SORTABLE [UNF]] + [NOINDEX] [ field_name [AS alias] TEXT | TAG | NUMERIC | GEO | VECTOR [ SORTABLE [UNF]] [NOINDEX] ...] + */ + + } } } \ No newline at end of file From 45975c2717e3bddd0fb1da47bfc891c1335364c0 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Mon, 12 Sep 2022 15:03:44 +0300 Subject: [PATCH 04/28] Add FT.CERATE command --- .../Search/FT.CREATE/CreateCommand.cs | 23 -- .../Search/FT.CREATE/FTCreateParams.cs | 144 ++++---- src/NRedisStack/Search/FieldName.cs | 47 +++ src/NRedisStack/Search/Schema.cs | 336 ++++++++++++++++++ src/NRedisStack/Search/SearchCommands.cs | 25 ++ tests/NRedisStack.Tests/Search/SearchTests.cs | 36 +- 6 files changed, 515 insertions(+), 96 deletions(-) delete mode 100644 src/NRedisStack/Search/FT.CREATE/CreateCommand.cs create mode 100644 src/NRedisStack/Search/FieldName.cs create mode 100644 src/NRedisStack/Search/Schema.cs diff --git a/src/NRedisStack/Search/FT.CREATE/CreateCommand.cs b/src/NRedisStack/Search/FT.CREATE/CreateCommand.cs deleted file mode 100644 index 8fcff2c1..00000000 --- a/src/NRedisStack/Search/FT.CREATE/CreateCommand.cs +++ /dev/null @@ -1,23 +0,0 @@ -using NRedisStack.Literals; -using StackExchange.Redis; -namespace NRedisStack -{ - public class CreateCommand - { - IDatabase _db; - public CreateCommand(IDatabase db) - { - _db = db; - } - - /// - /// Create an index with the given specification. - /// - /// Array with index names. - /// - // public RedisResult[] Create() - // { - // return _db.Execute(FT._LIST).ToArray(); - // } - } -} \ No newline at end of file diff --git a/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs b/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs index af27fc1a..32f1ca30 100644 --- a/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs +++ b/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs @@ -1,3 +1,4 @@ +using NRedisStack.Extensions; using NRedisStack.Literals; using NRedisStack.Literals.Enums; // TODO: look at NRediSearch @@ -31,18 +32,19 @@ public static FTCreateParams createParams() return new FTCreateParams(); } - /* Currently supports HASH (default) and JSON. To index JSON, you must have the RedisJSON module - installed. - */ + /// + /// Currently supports HASH (default) and JSON. To index JSON, you must have the RedisJSON module + /// installed. + /// public FTCreateParams on(IndexDataType dataType) { this.dataType = dataType; return this; } - /** - * Tells the index which keys it should index. You can add several prefixes to index. - */ + /// + /// Tells the index which keys it should index. You can add several prefixes to index. + /// public FTCreateParams Prefix(params string[] prefixes) { if (this.prefixes == null) @@ -53,11 +55,10 @@ public FTCreateParams Prefix(params string[] prefixes) return this; } - /** - * This method can be chained to add multiple prefixes. - * - * @see FTCreateParams#prefix(java.lang.params string[]) - */ + /// + /// This method can be chained to add multiple prefixes. + /// @see FTCreateParams#prefix(java.lang.params string[]) + /// public FTCreateParams AddPrefix(string prefix) { if (this.prefixes == null) @@ -68,148 +69,148 @@ public FTCreateParams AddPrefix(string prefix) return this; } - /** - * A filter expression with the full RediSearch aggregation expression language. - */ + /// + /// A filter expression with the full RediSearch aggregation expression language. + /// public FTCreateParams Filter(string filter) { this.filter = filter; return this; } - /** - * Indicates the default language for documents in the index. - */ + /// + /// default language for documents in the index. + /// public FTCreateParams Language(string defaultLanguage) { this.language = defaultLanguage; return this; } - /** - * Document attribute set as the document language. - */ + /// + /// Document attribute set as the document language. + /// public FTCreateParams LanguageField(string languageAttribute) { this.languageField = languageAttribute; return this; } - /** - * Default score for documents in the index. - */ + /// + /// Default score for documents in the index. + /// public FTCreateParams Score(double defaultScore) { this.score = defaultScore; return this; } - /** - * Document attribute that you use as the document rank based on the user ranking. - * Ranking must be between 0.0 and 1.0. - */ + /// + /// Document attribute that you use as the document rank based on the user ranking. + /// Ranking must be between 0.0 and 1.0. + /// public FTCreateParams ScoreField(string scoreField) { this.scoreField = scoreField; return this; } - /** - * Document attribute that you use as a binary safe payload string to the document that can be - * evaluated at query time by a custom scoring function or retrieved to the client. - */ + /// + /// Document attribute that you use as a binary safe payload string to the document that can be + /// evaluated at query time by a custom scoring function or retrieved to the client. + /// public FTCreateParams PayloadField(byte[] payloadAttribute) { Array.Copy(this.payloadField, payloadAttribute, payloadAttribute.Length); return this; } - /** - * Forces RediSearch to encode indexes as if there were more than 32 text attributes. - */ + /// + /// Forces RediSearch to encode indexes as if there were more than 32 text attributes. + /// public FTCreateParams MaxTextFields() { this.maxTextFields = true; return this; } - /** - * Does not store term offsets for documents. It saves memory, but does not allow exact searches - * or highlighting. - */ + /// + /// Does not store term offsets for documents. It saves memory, but does not allow exact searches + /// or highlighting. + /// public FTCreateParams NoOffsets() { this.noOffsets = true; return this; } - /** - * Creates a lightweight temporary index that expires after a specified period of inactivity. - */ + /// + /// Creates a lightweight temporary index that expires after a specified period of inactivity. + /// public FTCreateParams Temporary(long seconds) { this.temporary = seconds; return this; } - /** - * Conserves storage space and memory by disabling highlighting support. - */ + /// + /// Conserves storage space and memory by disabling highlighting support. + /// public FTCreateParams NoHL() { this.noHL = true; return this; } - /** - * @see FTCreateParams#noHL() - */ + /// + /// @see FTCreateParams#noHL() + /// public FTCreateParams NoHighlights() { return NoHL(); } - /** - * Does not store attribute bits for each term. It saves memory, but it does not allow filtering - * by specific attributes. - */ + /// + /// Does not store attribute bits for each term. It saves memory, but it does not allow filtering + /// by specific attributes. + /// public FTCreateParams NoFields() { this.noFields = true; return this; } - /** - * Avoids saving the term frequencies in the index. It saves memory, but does not allow sorting - * based on the frequencies of a given term within the document. - */ + /// + /// Avoids saving the term frequencies in the index. It saves memory, but does not allow sorting + /// based on the frequencies of a given term within the document. + /// public FTCreateParams NoFreqs() { this.noFreqs = true; return this; } - /** - * Sets the index with a custom stopword list, to be ignored during indexing and search time. - */ + /// + /// Sets the index with a custom stopword list, to be ignored during indexing and search time. + /// public FTCreateParams topwords(params string[] stopwords) { this.stopwords = stopwords.ToList(); return this; } - /** - * The index does not have stopwords, not even the default ones. - */ + /// + /// The index does not have stopwords, not even the default ones. + /// public FTCreateParams NoStopwords() { this.stopwords = new List { }; return this; } - /** - * Does not scan and index. - */ + /// + /// Does not scan and index. + /// public FTCreateParams SkipInitialScan() { this.skipInitialScan = true; @@ -222,15 +223,15 @@ public void AddParams(List args) if (dataType != null) { args.Add("ON"); - args.Add(dataType); + args.Add(dataType.AsArg()); } if (prefixes != null) { args.Add(SearchArgs.PREFIX); args.Add(prefixes.Count); - foreach(var prefix in prefixes) - if(prefix != null) + foreach (var prefix in prefixes) + if (prefix != null) args.Add(prefix); } @@ -310,11 +311,10 @@ public void AddParams(List args) { args.Add(SearchArgs.SKIPINITIALSCAN); } - /* - here sepose to be: - SCHEMA field_name [AS alias] TEXT | TAG | NUMERIC | GEO | VECTOR [ SORTABLE [UNF]] - [NOINDEX] [ field_name [AS alias] TEXT | TAG | NUMERIC | GEO | VECTOR [ SORTABLE [UNF]] [NOINDEX] ...] - */ + // here sepose to be: + // SCHEMA field_name[AS alias] TEXT | TAG | NUMERIC | GEO | VECTOR[SORTABLE[UNF]] + // [NOINDEX][field_name[AS alias] TEXT | TAG | NUMERIC | GEO | VECTOR[SORTABLE[UNF]][NOINDEX]...] + } } diff --git a/src/NRedisStack/Search/FieldName.cs b/src/NRedisStack/Search/FieldName.cs new file mode 100644 index 00000000..e7f63b9c --- /dev/null +++ b/src/NRedisStack/Search/FieldName.cs @@ -0,0 +1,47 @@ +using System.Collections.Generic; + +namespace NRedisStack.Search +{ + public class FieldName { + private readonly string name; + private string attribute; + + public FieldName(string name) : this(name, null) { + + } + + public FieldName(string name, string attribute) { + this.name = name; + this.attribute = attribute; + } + + public int AddCommandArguments(List args) { + args.Add(name); + if (attribute == null) { + return 1; + } + + args.Add("AS"); + args.Add(attribute); + return 3; + } + + public static FieldName Of(string name) { + return new FieldName(name); + } + + public FieldName As(string attribute) { + this.attribute = attribute; + return this; + } + + public static FieldName[] convert(params string[] names) { + if (names == null) return null; + FieldName[] fields = new FieldName[names.Length]; + for (int i = 0; i < names.Length; i++) + fields[i] = FieldName.Of(names[i]); + + return fields; + } + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/Schema.cs b/src/NRedisStack/Search/Schema.cs new file mode 100644 index 00000000..b3302ecf --- /dev/null +++ b/src/NRedisStack/Search/Schema.cs @@ -0,0 +1,336 @@ +// .NET port of https://github.com/RedisLabs/JRediSearch/ + +using System; +using System.Collections.Generic; + +namespace NRedisStack.Search +{ + /// + /// Schema abstracts the schema definition when creating an index. + /// Documents can contain fields not mentioned in the schema, but the index will only index pre-defined fields + /// + public sealed class Schema + { + public enum FieldType + { + Text, + Geo, + Numeric, + Tag, + Vector + } + + public class Field + { + public FieldName FieldName { get; } + public string Name { get; } + public FieldType Type { get; } + public bool Sortable { get; } + public bool NoIndex { get; } + public bool Unf { get; } + + internal Field(string name, FieldType type, bool sortable, bool noIndex = false, bool unf = false) + : this(FieldName.Of(name), type, sortable, noIndex, unf) + { + Name = name; + } + + internal Field(FieldName name, FieldType type, bool sortable, bool noIndex = false, bool unf = false) + { + FieldName = name; + Type = type; + Sortable = sortable; + NoIndex = noIndex; + if (unf && !sortable){ + throw new ArgumentException("UNF can't be applied on a non-sortable field."); + } + Unf = unf; + } + + internal virtual void SerializeRedisArgs(List args) + { + static object GetForRedis(FieldType type) => type switch + { + FieldType.Text => "TEXT", + FieldType.Geo => "GEO", + FieldType.Numeric => "NUMERIC", + FieldType.Tag => "TAG", + FieldType.Vector => "VECTOR", + _ => throw new ArgumentOutOfRangeException(nameof(type)), + }; + FieldName.AddCommandArguments(args); + args.Add(GetForRedis(Type)); + if (Sortable) { args.Add("SORTABLE"); } + if (Unf) args.Add("UNF"); + if (NoIndex) { args.Add("NOINDEX"); } + } + } + + public class TextField : Field + { + public double Weight { get; } + public bool NoStem { get; } + + public TextField(string name, double weight, bool sortable, bool noStem, bool noIndex) + : this(name, weight, sortable, noStem, noIndex, false) { } + + public TextField(string name, double weight = 1.0, bool sortable = false, bool noStem = false, bool noIndex = false, bool unNormalizedForm = false) + : base(name, FieldType.Text, sortable, noIndex, unNormalizedForm) + { + Weight = weight; + NoStem = noStem; + } + + public TextField(FieldName name, double weight, bool sortable, bool noStem, bool noIndex) + : this(name, weight, sortable, noStem, noIndex, false) { } + + public TextField(FieldName name, double weight = 1.0, bool sortable = false, bool noStem = false, bool noIndex = false, bool unNormalizedForm = false) + : base(name, FieldType.Text, sortable, noIndex, unNormalizedForm) + { + Weight = weight; + NoStem = noStem; + } + + internal override void SerializeRedisArgs(List args) + { + base.SerializeRedisArgs(args); + if (Weight != 1.0) + { + args.Add("WEIGHT"); + args.Add(Weight); + } + if (NoStem) args.Add("NOSTEM"); + } + } + + public List Fields { get; } = new List(); + + /// + /// Add a field to the schema. + /// + /// The to add. + /// The object. + public Schema AddField(Field field) + { + Fields.Add(field ?? throw new ArgumentNullException(nameof(field))); + return this; + } + + /// + /// Add a text field to the schema with a given weight. + /// + /// The field's name. + /// Its weight, a positive floating point number. + /// The object. + public Schema AddTextField(string name, double weight = 1.0) + { + Fields.Add(new TextField(name, weight)); + return this; + } + + /// + /// Add a text field to the schema with a given weight. + /// + /// The field's name. + /// Its weight, a positive floating point number. + /// The object. + public Schema AddTextField(FieldName name, double weight = 1.0) + { + Fields.Add(new TextField(name, weight)); + return this; + } + + /// + /// Add a text field that can be sorted on. + /// + /// The field's name. + /// Its weight, a positive floating point number. + /// Set this to true to prevent the indexer from sorting on the normalized form. + /// Normalied form is the field sent to lower case with all diaretics removed + /// The object. + public Schema AddSortableTextField(string name, double weight = 1.0, bool unf = false) + { + Fields.Add(new TextField(name, weight, true, unNormalizedForm: unf)); + return this; + } + + /// + /// Add a text field that can be sorted on. + /// + /// The field's name. + /// Its weight, a positive floating point number. + /// The object. + public Schema AddSortableTextField(string name, double weight) => AddSortableTextField(name, weight, false); + + /// + /// Add a text field that can be sorted on. + /// + /// The field's name. + /// Its weight, a positive floating point number. + /// Set this to true to prevent the indexer from sorting on the normalized form. + /// Normalied form is the field sent to lower case with all diaretics removed + /// The object. + public Schema AddSortableTextField(FieldName name, double weight = 1.0, bool unNormalizedForm = false) + { + Fields.Add(new TextField(name, weight, true, unNormalizedForm: unNormalizedForm)); + return this; + } + + /// + /// Add a text field that can be sorted on. + /// + /// The field's name. + /// Its weight, a positive floating point number. + /// The object. + public Schema AddSortableTextField(FieldName name, double weight) => AddSortableTextField(name, weight, false); + + /// + /// Add a numeric field to the schema. + /// + /// The field's name. + /// The object. + public Schema AddGeoField(string name) + { + Fields.Add(new Field(name, FieldType.Geo, false)); + return this; + } + + /// + /// Add a numeric field to the schema. + /// + /// The field's name. + /// The object. + public Schema AddGeoField(FieldName name) + { + Fields.Add(new Field(name, FieldType.Geo, false)); + return this; + } + + /// + /// Add a numeric field to the schema. + /// + /// The field's name. + /// The object. + public Schema AddNumericField(string name) + { + Fields.Add(new Field(name, FieldType.Numeric, false)); + return this; + } + + /// + /// Add a numeric field to the schema. + /// + /// The field's name. + /// The object. + public Schema AddNumericField(FieldName name) + { + Fields.Add(new Field(name, FieldType.Numeric, false)); + return this; + } + + /// + /// Add a numeric field that can be sorted on. + /// + /// The field's name. + /// The object. + public Schema AddSortableNumericField(string name) + { + Fields.Add(new Field(name, FieldType.Numeric, true)); + return this; + } + + /// + /// Add a numeric field that can be sorted on. + /// + /// The field's name. + /// The object. + public Schema AddSortableNumericField(FieldName name) + { + Fields.Add(new Field(name, FieldType.Numeric, true)); + return this; + } + + public class TagField : Field + { + public string Separator { get; } + + internal TagField(string name, string separator = ",", bool sortable = false, bool unNormalizedForm = false) + : base(name, FieldType.Tag, sortable, unf: unNormalizedForm) + { + Separator = separator; + } + + internal TagField(FieldName name, string separator = ",", bool sortable = false, bool unNormalizedForm = false) + : base(name, FieldType.Tag, sortable, unf: unNormalizedForm) + { + Separator = separator; + } + + internal override void SerializeRedisArgs(List args) + { + base.SerializeRedisArgs(args); + if (Separator != ",") + { + if (Sortable) args.Remove("SORTABLE"); + if (Unf) args.Remove("UNF"); + args.Add("SEPARATOR"); + args.Add(Separator); + if (Sortable) args.Add("SORTABLE"); + if (Unf) args.Add("UNF"); + } + } + } + + /// + /// Add a TAG field. + /// + /// The field's name. + /// The tag separator. + /// The object. + public Schema AddTagField(string name, string separator = ",") + { + Fields.Add(new TagField(name, separator)); + return this; + } + + /// + /// Add a TAG field. + /// + /// The field's name. + /// The tag separator. + /// The object. + public Schema AddTagField(FieldName name, string separator = ",") + { + Fields.Add(new TagField(name, separator)); + return this; + } + + /// + /// Add a sortable TAG field. + /// + /// The field's name. + /// The tag separator. + /// Set this to true to prevent the indexer from sorting on the normalized form. + /// Normalied form is the field sent to lower case with all diaretics removed + /// The object. + public Schema AddSortableTagField(string name, string separator = ",", bool unNormalizedForm = false) + { + Fields.Add(new TagField(name, separator, sortable: true, unNormalizedForm: unNormalizedForm)); + return this; + } + + /// + /// Add a sortable TAG field. + /// + /// The field's name. + /// The tag separator. + /// Set this to true to prevent the indexer from sorting on the normalized form. + /// Normalied form is the field sent to lower case with all diaretics removed + /// The object. + public Schema AddSortableTagField(FieldName name, string separator = ",", bool unNormalizedForm = false) + { + Fields.Add(new TagField(name, separator, sortable: true, unNormalizedForm: unNormalizedForm)); + return this; + } + } +} diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index 274373da..cbca7286 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -1,4 +1,6 @@ using NRedisStack.Literals; +using NRedisStack.Search; +using NRedisStack.Search.FT.CREATE; using StackExchange.Redis; namespace NRedisStack { @@ -112,5 +114,28 @@ public RedisResult Info(RedisValue index) { return _db.Execute(FT.INFO, index); } + + /// + /// Create an index with the given specification. + /// + /// The index name. + /// Command's parameters. + /// The index schema. + /// if executed correctly, error otherwise + /// + public bool Create(string indexName, FTCreateParams parameters, Schema schema) + { + var args = new List() { indexName }; + parameters.AddParams(args); + + args.Add("SCHEMA"); + + foreach (var f in schema.Fields) + { + f.SerializeRedisArgs(args); + } + + return _db.Execute(FT.CREATE, args).OKtoBoolean(); + } } } \ No newline at end of file diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index ca963510..ad62fa21 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -2,7 +2,8 @@ using StackExchange.Redis; using NRedisStack.RedisStackCommands; using Moq; - +using NRedisStack.Search.FT.CREATE; +using NRedisStack.Search; namespace NRedisStack.Tests.Search; @@ -10,6 +11,7 @@ public class SearchTests : AbstractNRedisStackTest, IDisposable { Mock _mock = new Mock(); private readonly string key = "SEARCH_TESTS"; + private readonly string index = "TEST_INDEX"; public SearchTests(RedisFixture redisFixture) : base(redisFixture) { } public void Dispose() @@ -17,6 +19,38 @@ public void Dispose() redisFixture.Redis.GetDatabase().KeyDelete(key); } + [Fact] + public void TestCreate() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + var schema = new Schema().AddTextField("first").AddTextField("last").AddNumericField("age"); + var parameters = FTCreateParams.createParams().Filter("@age>16").Prefix("student:", "pupil:"); + + Assert.True(ft.Create(index, parameters, schema)); + + db.HashSet("profesor:5555", new HashEntry[] { new("first", "Albert"), new("last", "Blue"), new("age", "55") }); + db.HashSet("student:1111", new HashEntry[] { new("first", "Joe"), new("last", "Dod"), new("age", "18") }); + db.HashSet("pupil:2222", new HashEntry[] { new("first", "Jen"), new("last", "Rod"), new("age", "14") }); + db.HashSet("student:3333", new HashEntry[] { new("first", "El"), new("last", "Mark"), new("age", "17") }); + db.HashSet("pupil:4444", new HashEntry[] { new("first", "Pat"), new("last", "Shu"), new("age", "21") }); + db.HashSet("student:5555", new HashEntry[] { new("first", "Joen"), new("last", "Ko"), new("age", "20") }); + db.HashSet("teacher:6666", new HashEntry[] { new("first", "Pat"), new("last", "Rod"), new("age", "20") }); + + // var noFilters = ft.Search(index, new Query()); + // Assert.Equal(4, noFilters.getTotalResults()); + + // var res1 = ft.Search(index, new Query("@first:Jo*")); + // Assert.Equal(2, res1.getTotalResults()); + + // var res2 = ft.Search(index, new Query("@first:Pat")); + // Assert.Equal(1, res2.getTotalResults()); + + // var res3 = ft.Search(index, new Query("@last:Rod")); + // Assert.Equal(0, res3.getTotalResults()); + } + [Fact] public void TestModulePrefixs() From 03d6f1f884dc2015941ae7c7b6ebe66a55714415 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Tue, 13 Sep 2022 17:52:42 +0300 Subject: [PATCH 05/28] Fixing Schema and FieldName Classes --- src/NRedisStack/Search/FieldName.cs | 39 +++-- src/NRedisStack/Search/Schema.cs | 216 +++++++++++++++++++++------- 2 files changed, 187 insertions(+), 68 deletions(-) diff --git a/src/NRedisStack/Search/FieldName.cs b/src/NRedisStack/Search/FieldName.cs index e7f63b9c..8044bea3 100644 --- a/src/NRedisStack/Search/FieldName.cs +++ b/src/NRedisStack/Search/FieldName.cs @@ -1,41 +1,50 @@ using System.Collections.Generic; +using System.Text; namespace NRedisStack.Search { - public class FieldName { - private readonly string name; - private string attribute; + public class FieldName + { + private readonly string fieldName; + private string alias; - public FieldName(string name) : this(name, null) { + public FieldName(string name) : this(name, null) + { } - public FieldName(string name, string attribute) { - this.name = name; - this.attribute = attribute; + public FieldName(string name, string attribute) + { + this.fieldName = name; + this.alias = attribute; } - public int AddCommandArguments(List args) { - args.Add(name); - if (attribute == null) { + public int AddCommandArguments(List args) + { + args.Add(fieldName); + if (alias == null) + { return 1; } args.Add("AS"); - args.Add(attribute); + args.Add(alias); return 3; } - public static FieldName Of(string name) { + public static FieldName Of(string name) + { return new FieldName(name); } - public FieldName As(string attribute) { - this.attribute = attribute; + public FieldName As(string attribute) + { + this.alias = attribute; return this; } - public static FieldName[] convert(params string[] names) { + public static FieldName[] convert(params string[] names) + { if (names == null) return null; FieldName[] fields = new FieldName[names.Length]; for (int i = 0; i < names.Length; i++) diff --git a/src/NRedisStack/Search/Schema.cs b/src/NRedisStack/Search/Schema.cs index b3302ecf..d7781920 100644 --- a/src/NRedisStack/Search/Schema.cs +++ b/src/NRedisStack/Search/Schema.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using static NRedisStack.Search.Schema.VectorField; namespace NRedisStack.Search { @@ -12,7 +13,7 @@ namespace NRedisStack.Search public sealed class Schema { public enum FieldType - { + { // TODO: check if this is the correct order Text, Geo, Numeric, @@ -23,11 +24,11 @@ public enum FieldType public class Field { public FieldName FieldName { get; } - public string Name { get; } + public string Name { get; } //TODO: check if this is needed public FieldType Type { get; } public bool Sortable { get; } - public bool NoIndex { get; } public bool Unf { get; } + public bool NoIndex { get; } internal Field(string name, FieldType type, bool sortable, bool noIndex = false, bool unf = false) : this(FieldName.Of(name), type, sortable, noIndex, unf) @@ -47,7 +48,7 @@ internal Field(FieldName name, FieldType type, bool sortable, bool noIndex = fal Unf = unf; } - internal virtual void SerializeRedisArgs(List args) + internal /*virtual*/ void SerializeRedisArgs(List args) { static object GetForRedis(FieldType type) => type switch { @@ -60,49 +61,132 @@ internal virtual void SerializeRedisArgs(List args) }; FieldName.AddCommandArguments(args); args.Add(GetForRedis(Type)); + SerializeTypeArgs(args); if (Sortable) { args.Add("SORTABLE"); } if (Unf) args.Add("UNF"); if (NoIndex) { args.Add("NOINDEX"); } } + internal virtual void SerializeTypeArgs(List args) { } } public class TextField : Field { public double Weight { get; } public bool NoStem { get; } + public string Phonetic { get; } - public TextField(string name, double weight, bool sortable, bool noStem, bool noIndex) - : this(name, weight, sortable, noStem, noIndex, false) { } + public TextField(string name, double weight, bool sortable, bool noStem, string? phonetic, bool noIndex) + : this(name, weight, sortable, noStem, phonetic, noIndex, false) { } - public TextField(string name, double weight = 1.0, bool sortable = false, bool noStem = false, bool noIndex = false, bool unNormalizedForm = false) + public TextField(string name, double weight = 1.0, bool sortable = false, bool noStem = false, string? phonetic = null, bool noIndex = false, bool unNormalizedForm = false) : base(name, FieldType.Text, sortable, noIndex, unNormalizedForm) { Weight = weight; NoStem = noStem; + Phonetic = phonetic; } - public TextField(FieldName name, double weight, bool sortable, bool noStem, bool noIndex) - : this(name, weight, sortable, noStem, noIndex, false) { } + public TextField(FieldName name, double weight, bool sortable, bool noStem, string? phonetic, bool noIndex) + : this(name, weight, sortable, noStem, phonetic, noIndex, false) { } - public TextField(FieldName name, double weight = 1.0, bool sortable = false, bool noStem = false, bool noIndex = false, bool unNormalizedForm = false) + public TextField(FieldName name, double weight = 1.0, bool sortable = false, bool noStem = false, string? phonetic = null, bool noIndex = false, bool unNormalizedForm = false) : base(name, FieldType.Text, sortable, noIndex, unNormalizedForm) { Weight = weight; NoStem = noStem; + Phonetic = phonetic; } - internal override void SerializeRedisArgs(List args) + internal override void SerializeTypeArgs(List args) { - base.SerializeRedisArgs(args); + // base.SerializeRedisArgs(args); if (Weight != 1.0) { args.Add("WEIGHT"); args.Add(Weight); } if (NoStem) args.Add("NOSTEM"); + if (Phonetic != null) + { + args.Add("PHONETIC"); + args.Add(this.Phonetic); + } + + } + } + + public class TagField : Field + { + public string Separator { get; } + public bool CaseSensitive { get; } + + internal TagField(string name, string separator = ",", bool caseSensitive = false, bool sortable = false, bool unNormalizedForm = false) + : base(name, FieldType.Tag, sortable, unf: unNormalizedForm) + { + Separator = separator; + CaseSensitive = caseSensitive; + } + + internal TagField(FieldName name, string separator = ",", bool caseSensitive = false, bool sortable = false, bool unNormalizedForm = false) + : base(name, FieldType.Tag, sortable, unf: unNormalizedForm) + { + Separator = separator; + CaseSensitive = caseSensitive; + } + + internal override void SerializeTypeArgs(List args) + { + // base.SerializeRedisArgs(args); + if (Separator != ",") + { + // if (Sortable) args.Remove("SORTABLE"); + // if (Unf) args.Remove("UNF"); + args.Add("SEPARATOR"); + args.Add(Separator); + // if (Sortable) args.Add("SORTABLE"); + // if (Unf) args.Add("UNF"); + } + if (CaseSensitive) args.Add("CASESENSITIVE"); } } + public class VectorField : Field + { + public enum VectorAlgo + { + FLAT, + HNSW + } + + public VectorAlgo Algorithm { get; } + public Dictionary attributes { get; } + public VectorField(string name, VectorAlgo algorithm, Dictionary attributes, + bool sortable = false, bool noIndex = false, bool unNormalizedForm = false) + : base(name, FieldType.Vector, sortable, noIndex, unNormalizedForm) + { + Algorithm = algorithm; + this.attributes = attributes; + } + + public VectorField(FieldName name, VectorAlgo algorithm, Dictionary attributes, + bool sortable = false, bool noIndex = false, bool unNormalizedForm = false) + : base(name, FieldType.Vector, sortable, noIndex, unNormalizedForm) + { + Algorithm = algorithm; + this.attributes = attributes; + } + + internal override void SerializeTypeArgs(List args) + { + args.Add("ALGORITHM"); + args.Add(Algorithm.ToString()); + foreach (var attribute in attributes) + { + args.Add(attribute.Key); + args.Add(attribute.Value); + } + } + } public List Fields { get; } = new List(); /// @@ -117,26 +201,39 @@ public Schema AddField(Field field) } /// - /// Add a text field to the schema with a given weight. + /// Add a text field to the schema. /// /// The field's name. /// Its weight, a positive floating point number. + /// If true, the text field can be sorted. + /// Disable stemming when indexing its values. + /// Declaring a text attribute as PHONETIC will perform phonetic matching on it in searches by default. + /// Attributes can have the NOINDEX option, which means they will not be indexed. + /// Set this to true to prevent the indexer from sorting on the normalized form. + /// Normalied form is the field sent to lower case with all diaretics removed /// The object. - public Schema AddTextField(string name, double weight = 1.0) + public Schema AddTextField(string name, double weight = 1.0, bool sortable = false, bool noStem = false, + string? phonetic = null, bool noIndex = false, bool unNormalizedForm = false) { - Fields.Add(new TextField(name, weight)); + Fields.Add(new TextField(name, weight, sortable, noStem, phonetic, noIndex, unNormalizedForm)); return this; } /// - /// Add a text field to the schema with a given weight. + /// Add a text field to the schema. /// /// The field's name. /// Its weight, a positive floating point number. + /// If true, the text field can be sorted. + /// Disable stemming when indexing its values. + /// Declaring a text attribute as PHONETIC will perform phonetic matching on it in searches by default. + /// Attributes can have the NOINDEX option, which means they will not be indexed. + /// Set this to true to prevent the indexer from sorting on the normalized form. /// The object. - public Schema AddTextField(FieldName name, double weight = 1.0) + public Schema AddTextField(FieldName name, double weight = 1.0, bool sortable = false, bool noStem = false, + string? phonetic = null, bool noIndex = false, bool unNormalizedForm = false) { - Fields.Add(new TextField(name, weight)); + Fields.Add(new TextField(name, weight, sortable, noStem, phonetic, noIndex, unNormalizedForm)); return this; } @@ -250,46 +347,19 @@ public Schema AddSortableNumericField(FieldName name) return this; } - public class TagField : Field - { - public string Separator { get; } - - internal TagField(string name, string separator = ",", bool sortable = false, bool unNormalizedForm = false) - : base(name, FieldType.Tag, sortable, unf: unNormalizedForm) - { - Separator = separator; - } - - internal TagField(FieldName name, string separator = ",", bool sortable = false, bool unNormalizedForm = false) - : base(name, FieldType.Tag, sortable, unf: unNormalizedForm) - { - Separator = separator; - } - - internal override void SerializeRedisArgs(List args) - { - base.SerializeRedisArgs(args); - if (Separator != ",") - { - if (Sortable) args.Remove("SORTABLE"); - if (Unf) args.Remove("UNF"); - args.Add("SEPARATOR"); - args.Add(Separator); - if (Sortable) args.Add("SORTABLE"); - if (Unf) args.Add("UNF"); - } - } - } - /// /// Add a TAG field. /// /// The field's name. /// The tag separator. + /// If true, Keeps the original letter cases of the tags. + /// If true, the field can be sorted. + /// Set this to true to prevent the indexer from sorting on the normalized form. + /// Normalied form is the field sent to lower case with all diaretics removed /// The object. - public Schema AddTagField(string name, string separator = ",") + public Schema AddTagField(string name, string separator = ",", bool caseSensitive = false, bool sortable = false, bool unNormalizedForm = false) { - Fields.Add(new TagField(name, separator)); + Fields.Add(new TagField(name, separator, caseSensitive, sortable, unNormalizedForm)); return this; } @@ -298,10 +368,14 @@ public Schema AddTagField(string name, string separator = ",") /// /// The field's name. /// The tag separator. + /// If true, Keeps the original letter cases of the tags. + /// If true, the field can be sorted. + /// Set this to true to prevent the indexer from sorting on the normalized form. + /// Normalied form is the field sent to lower case with all diaretics removed /// The object. - public Schema AddTagField(FieldName name, string separator = ",") + public Schema AddTagField(FieldName name, string separator = ",", bool caseSensitive = false, bool sortable = false, bool unNormalizedForm = false) { - Fields.Add(new TagField(name, separator)); + Fields.Add(new TagField(name, separator, caseSensitive, sortable, unNormalizedForm)); return this; } @@ -332,5 +406,41 @@ public Schema AddSortableTagField(FieldName name, string separator = ",", bool u Fields.Add(new TagField(name, separator, sortable: true, unNormalizedForm: unNormalizedForm)); return this; } + + /// + /// Add a TAG field. + /// + /// The field's name. + /// The vector similarity algorithm to use. + /// The algorithm attributes for the creation of the vector index. + /// If true, the field can be sorted. + /// Attributes can have the NOINDEX option, which means they will not be indexed. + /// Set this to true to prevent the indexer from sorting on the normalized form. + /// Normalied form is the field sent to lower case with all diaretics removed + /// The object. + public Schema AddVectorField(string name, VectorAlgo algorithm, Dictionary attributes, + bool sortable = false, bool noIndex = false, bool unNormalizedForm = false) + { + Fields.Add(new VectorField(name, algorithm, attributes, sortable, noIndex, unNormalizedForm)); + return this; + } + + /// + /// Add a TAG field. + /// + /// The field's name. + /// The vector similarity algorithm to use. + /// The algorithm attributes for the creation of the vector index. + /// If true, the field can be sorted. + /// Attributes can have the NOINDEX option, which means they will not be indexed. + /// Set this to true to prevent the indexer from sorting on the normalized form. + /// Normalied form is the field sent to lower case with all diaretics removed + /// The object. + public Schema AddVectorField(FieldName name, VectorAlgo algorithm, Dictionary attributes, + bool sortable = false, bool noIndex = false, bool unNormalizedForm = false) + { + Fields.Add(new VectorField(name, algorithm, attributes, sortable, noIndex, unNormalizedForm)); + return this; + } } } From 6bf4e427e5bc68a6372fec6a16ce31280b872ff5 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Sun, 18 Sep 2022 18:20:39 +0300 Subject: [PATCH 06/28] Work on Schema class --- src/NRedisStack/Search/FieldName.cs | 5 +- src/NRedisStack/Search/Schema.cs | 398 ++++++++++------------- src/NRedisStack/Search/SearchCommands.cs | 4 +- 3 files changed, 176 insertions(+), 231 deletions(-) diff --git a/src/NRedisStack/Search/FieldName.cs b/src/NRedisStack/Search/FieldName.cs index 8044bea3..e277a30f 100644 --- a/src/NRedisStack/Search/FieldName.cs +++ b/src/NRedisStack/Search/FieldName.cs @@ -8,10 +8,7 @@ public class FieldName private readonly string fieldName; private string alias; - public FieldName(string name) : this(name, null) - { - - } + public FieldName(string name) : this(name, null) { } public FieldName(string name, string attribute) { diff --git a/src/NRedisStack/Search/Schema.cs b/src/NRedisStack/Search/Schema.cs index d7781920..abecb187 100644 --- a/src/NRedisStack/Search/Schema.cs +++ b/src/NRedisStack/Search/Schema.cs @@ -1,6 +1,4 @@ -// .NET port of https://github.com/RedisLabs/JRediSearch/ - -using System; +using System; using System.Collections.Generic; using static NRedisStack.Search.Schema.VectorField; @@ -13,7 +11,7 @@ namespace NRedisStack.Search public sealed class Schema { public enum FieldType - { // TODO: check if this is the correct order + { Text, Geo, Numeric, @@ -24,31 +22,18 @@ public enum FieldType public class Field { public FieldName FieldName { get; } - public string Name { get; } //TODO: check if this is needed public FieldType Type { get; } - public bool Sortable { get; } - public bool Unf { get; } - public bool NoIndex { get; } - internal Field(string name, FieldType type, bool sortable, bool noIndex = false, bool unf = false) - : this(FieldName.Of(name), type, sortable, noIndex, unf) - { - Name = name; - } + internal Field(string name, FieldType type) + : this(FieldName.Of(name), type) { } - internal Field(FieldName name, FieldType type, bool sortable, bool noIndex = false, bool unf = false) + internal Field(FieldName name, FieldType type) { FieldName = name; Type = type; - Sortable = sortable; - NoIndex = noIndex; - if (unf && !sortable){ - throw new ArgumentException("UNF can't be applied on a non-sortable field."); - } - Unf = unf; } - internal /*virtual*/ void SerializeRedisArgs(List args) + internal void AddSchemaArgs(List args) { static object GetForRedis(FieldType type) => type switch { @@ -61,95 +46,161 @@ internal Field(FieldName name, FieldType type, bool sortable, bool noIndex = fal }; FieldName.AddCommandArguments(args); args.Add(GetForRedis(Type)); - SerializeTypeArgs(args); - if (Sortable) { args.Add("SORTABLE"); } - if (Unf) args.Add("UNF"); - if (NoIndex) { args.Add("NOINDEX"); } + AddFieldTypeArgs(args); } - internal virtual void SerializeTypeArgs(List args) { } + internal virtual void AddFieldTypeArgs(List args) { } } public class TextField : Field { public double Weight { get; } public bool NoStem { get; } - public string Phonetic { get; } - - public TextField(string name, double weight, bool sortable, bool noStem, string? phonetic, bool noIndex) - : this(name, weight, sortable, noStem, phonetic, noIndex, false) { } + public string? Phonetic { get; } + public bool Sortable { get; } + public bool Unf { get; } + public bool NoIndex { get; } + public bool WithSuffixTrie { get; } - public TextField(string name, double weight = 1.0, bool sortable = false, bool noStem = false, string? phonetic = null, bool noIndex = false, bool unNormalizedForm = false) - : base(name, FieldType.Text, sortable, noIndex, unNormalizedForm) + public TextField(FieldName name, double weight = 1.0, bool noStem = false, + string? phonetic = null, bool sortable = false, bool unf = false, + bool noIndex = false, bool withSuffixTrie = false) + : base(name, FieldType.Text) { Weight = weight; NoStem = noStem; Phonetic = phonetic; + Sortable = sortable; + if (unf && !sortable) + { + throw new ArgumentException("UNF can't be applied on a non-sortable field."); + } + Unf = unf; + NoIndex = noIndex; + WithSuffixTrie = withSuffixTrie; } - public TextField(FieldName name, double weight, bool sortable, bool noStem, string? phonetic, bool noIndex) - : this(name, weight, sortable, noStem, phonetic, noIndex, false) { } + public TextField(string name, double weight = 1.0, bool noStem = false, + string? phonetic = null, bool sortable = false, bool unf = false, + bool noIndex = false, bool withSuffixTrie = false) + : this(FieldName.Of(name), weight, noStem, phonetic, sortable, unf, noIndex, withSuffixTrie) { } - public TextField(FieldName name, double weight = 1.0, bool sortable = false, bool noStem = false, string? phonetic = null, bool noIndex = false, bool unNormalizedForm = false) - : base(name, FieldType.Text, sortable, noIndex, unNormalizedForm) + internal override void AddFieldTypeArgs(List args) { - Weight = weight; - NoStem = noStem; - Phonetic = phonetic; + if (Sortable) args.Add("SORTABLE"); + if (Unf) args.Add("UNF"); + if (NoStem) args.Add("NOSTEM"); + if (NoIndex) args.Add("NOINDEX"); + AddPhonetic(args); + AddWeight(args); + if (WithSuffixTrie) args.Add("WITHSUFFIXTRIE"); } - internal override void SerializeTypeArgs(List args) + private void AddWeight(List args) { - // base.SerializeRedisArgs(args); if (Weight != 1.0) { args.Add("WEIGHT"); args.Add(Weight); } - if (NoStem) args.Add("NOSTEM"); + } + + private void AddPhonetic(List args) + { if (Phonetic != null) { args.Add("PHONETIC"); args.Add(this.Phonetic); } - } } public class TagField : Field { + public bool Sortable { get; } + public bool Unf { get; } + public bool NoIndex { get; } public string Separator { get; } public bool CaseSensitive { get; } + public bool WithSuffixTrie { get; } - internal TagField(string name, string separator = ",", bool caseSensitive = false, bool sortable = false, bool unNormalizedForm = false) - : base(name, FieldType.Tag, sortable, unf: unNormalizedForm) + internal TagField(FieldName name, bool sortable = false, bool unf = false, + bool noIndex = false, string separator = ",", + bool caseSensitive = false, bool withSuffixTrie = false) + : base(name, FieldType.Tag) { + Sortable = sortable; + Unf = unf; + NoIndex = noIndex; Separator = separator; CaseSensitive = caseSensitive; + WithSuffixTrie = withSuffixTrie; } - internal TagField(FieldName name, string separator = ",", bool caseSensitive = false, bool sortable = false, bool unNormalizedForm = false) - : base(name, FieldType.Tag, sortable, unf: unNormalizedForm) - { - Separator = separator; - CaseSensitive = caseSensitive; - } + internal TagField(string name, bool sortable = false, bool unf = false, + bool noIndex = false, string separator = ",", + bool caseSensitive = false, bool withSuffixTrie = false) + : this(FieldName.Of(name), sortable, unf, noIndex, separator, caseSensitive, withSuffixTrie) { } - internal override void SerializeTypeArgs(List args) + internal override void AddFieldTypeArgs(List args) { - // base.SerializeRedisArgs(args); + if (Sortable) args.Add("SORTABLE"); + if (Unf) args.Add("UNF"); + if (NoIndex) args.Add("NOINDEX"); + if (WithSuffixTrie) args.Add("WITHSUFFIXTRIE"); if (Separator != ",") { - // if (Sortable) args.Remove("SORTABLE"); - // if (Unf) args.Remove("UNF"); + args.Add("SEPARATOR"); args.Add(Separator); - // if (Sortable) args.Add("SORTABLE"); - // if (Unf) args.Add("UNF"); } if (CaseSensitive) args.Add("CASESENSITIVE"); } } + public class GeoField : Field + { + public bool Sortable { get; } + public bool NoIndex { get; } + internal GeoField(FieldName name, bool sortable = false, bool noIndex = false) + : base(name, FieldType.Geo) + { + Sortable = sortable; + NoIndex = noIndex; + } + + internal GeoField(string name, bool sortable = false, bool noIndex = false) + : this(FieldName.Of(name), sortable, noIndex) { } + + internal override void AddFieldTypeArgs(List args) + { + if (Sortable) args.Add("SORTABLE"); + if (NoIndex) args.Add("NOINDEX"); + } + + } + + public class NumericField : Field + { + public bool Sortable { get; } + public bool NoIndex { get; } + internal NumericField(FieldName name, bool sortable = false, bool noIndex = false) + : base(name, FieldType.Numeric) + { + Sortable = sortable; + NoIndex = noIndex; + } + + internal NumericField(string name, bool sortable = false, bool noIndex = false) + : this(FieldName.Of(name), sortable, noIndex) { } + + internal override void AddFieldTypeArgs(List args) + { + if (Sortable) args.Add("SORTABLE"); + if (NoIndex) args.Add("NOINDEX"); + } + + } + public class VectorField : Field { public enum VectorAlgo @@ -159,31 +210,26 @@ public enum VectorAlgo } public VectorAlgo Algorithm { get; } - public Dictionary attributes { get; } - public VectorField(string name, VectorAlgo algorithm, Dictionary attributes, - bool sortable = false, bool noIndex = false, bool unNormalizedForm = false) - : base(name, FieldType.Vector, sortable, noIndex, unNormalizedForm) - { - Algorithm = algorithm; - this.attributes = attributes; - } - - public VectorField(FieldName name, VectorAlgo algorithm, Dictionary attributes, - bool sortable = false, bool noIndex = false, bool unNormalizedForm = false) - : base(name, FieldType.Vector, sortable, noIndex, unNormalizedForm) + public Dictionary? Attributes { get; } + public VectorField(string name, VectorAlgo algorithm, Dictionary? attributes = null) + : base(name, FieldType.Vector) { Algorithm = algorithm; - this.attributes = attributes; + Attributes = attributes; } - internal override void SerializeTypeArgs(List args) + internal override void AddFieldTypeArgs(List args) { - args.Add("ALGORITHM"); args.Add(Algorithm.ToString()); - foreach (var attribute in attributes) + if (Attributes != null) { - args.Add(attribute.Key); - args.Add(attribute.Value); + args.Add(Attributes.Count()); + + foreach (var attribute in Attributes) + { + args.Add(attribute.Key); + args.Add(attribute.Value); + } } } } @@ -201,7 +247,7 @@ public Schema AddField(Field field) } /// - /// Add a text field to the schema. + /// Add a Text field to the schema. /// /// The field's name. /// Its weight, a positive floating point number. @@ -209,18 +255,19 @@ public Schema AddField(Field field) /// Disable stemming when indexing its values. /// Declaring a text attribute as PHONETIC will perform phonetic matching on it in searches by default. /// Attributes can have the NOINDEX option, which means they will not be indexed. - /// Set this to true to prevent the indexer from sorting on the normalized form. + /// Set this to true to prevent the indexer from sorting on the normalized form. /// Normalied form is the field sent to lower case with all diaretics removed + /// Keeps a suffix trie with all terms which match the suffix. /// The object. - public Schema AddTextField(string name, double weight = 1.0, bool sortable = false, bool noStem = false, - string? phonetic = null, bool noIndex = false, bool unNormalizedForm = false) + public Schema AddTextField(string name, double weight = 1.0, bool sortable = false, bool unf = false, bool noStem = false, + string? phonetic = null, bool noIndex = false, bool withSuffixTrie = false) { - Fields.Add(new TextField(name, weight, sortable, noStem, phonetic, noIndex, unNormalizedForm)); + Fields.Add(new TextField(name, weight, noStem, phonetic, sortable, unf, noIndex, withSuffixTrie)); return this; } /// - /// Add a text field to the schema. + /// Add a Text field to the schema. /// /// The field's name. /// Its weight, a positive floating point number. @@ -228,218 +275,119 @@ public Schema AddTextField(string name, double weight = 1.0, bool sortable = fal /// Disable stemming when indexing its values. /// Declaring a text attribute as PHONETIC will perform phonetic matching on it in searches by default. /// Attributes can have the NOINDEX option, which means they will not be indexed. - /// Set this to true to prevent the indexer from sorting on the normalized form. - /// The object. - public Schema AddTextField(FieldName name, double weight = 1.0, bool sortable = false, bool noStem = false, - string? phonetic = null, bool noIndex = false, bool unNormalizedForm = false) - { - Fields.Add(new TextField(name, weight, sortable, noStem, phonetic, noIndex, unNormalizedForm)); - return this; - } - - /// - /// Add a text field that can be sorted on. - /// - /// The field's name. - /// Its weight, a positive floating point number. - /// Set this to true to prevent the indexer from sorting on the normalized form. + /// Set this to true to prevent the indexer from sorting on the normalized form. /// Normalied form is the field sent to lower case with all diaretics removed + /// Keeps a suffix trie with all terms which match the suffix. /// The object. - public Schema AddSortableTextField(string name, double weight = 1.0, bool unf = false) + public Schema AddTextField(FieldName name, double weight = 1.0, bool sortable = false, bool unf = false, bool noStem = false, + string? phonetic = null, bool noIndex = false, bool withSuffixTrie = false) { - Fields.Add(new TextField(name, weight, true, unNormalizedForm: unf)); + Fields.Add(new TextField(name, weight, noStem, phonetic, sortable, unf, noIndex, withSuffixTrie)); return this; } /// - /// Add a text field that can be sorted on. - /// - /// The field's name. - /// Its weight, a positive floating point number. - /// The object. - public Schema AddSortableTextField(string name, double weight) => AddSortableTextField(name, weight, false); - - /// - /// Add a text field that can be sorted on. - /// - /// The field's name. - /// Its weight, a positive floating point number. - /// Set this to true to prevent the indexer from sorting on the normalized form. - /// Normalied form is the field sent to lower case with all diaretics removed - /// The object. - public Schema AddSortableTextField(FieldName name, double weight = 1.0, bool unNormalizedForm = false) - { - Fields.Add(new TextField(name, weight, true, unNormalizedForm: unNormalizedForm)); - return this; - } - - /// - /// Add a text field that can be sorted on. - /// - /// The field's name. - /// Its weight, a positive floating point number. - /// The object. - public Schema AddSortableTextField(FieldName name, double weight) => AddSortableTextField(name, weight, false); - - /// - /// Add a numeric field to the schema. - /// - /// The field's name. - /// The object. - public Schema AddGeoField(string name) - { - Fields.Add(new Field(name, FieldType.Geo, false)); - return this; - } - - /// - /// Add a numeric field to the schema. - /// - /// The field's name. - /// The object. - public Schema AddGeoField(FieldName name) - { - Fields.Add(new Field(name, FieldType.Geo, false)); - return this; - } - - /// - /// Add a numeric field to the schema. + /// Add a Geo field to the schema. /// /// The field's name. + /// If true, the text field can be sorted. + /// Attributes can have the NOINDEX option, which means they will not be indexed. /// The object. - public Schema AddNumericField(string name) + public Schema AddGeoField(FieldName name, bool sortable = false, bool noIndex = false) { - Fields.Add(new Field(name, FieldType.Numeric, false)); + Fields.Add(new GeoField(name, sortable, noIndex)); return this; } /// - /// Add a numeric field to the schema. + /// Add a Geo field to the schema. /// /// The field's name. + /// If true, the text field can be sorted. + /// Attributes can have the NOINDEX option, which means they will not be indexed. /// The object. - public Schema AddNumericField(FieldName name) + public Schema AddGeoField(string name, bool sortable = false, bool noIndex = false) { - Fields.Add(new Field(name, FieldType.Numeric, false)); + Fields.Add(new GeoField(name, sortable, noIndex)); return this; } /// - /// Add a numeric field that can be sorted on. + /// Add a Numeric field to the schema. /// /// The field's name. + /// If true, the text field can be sorted. + /// Attributes can have the NOINDEX option, which means they will not be indexed. /// The object. - public Schema AddSortableNumericField(string name) + public Schema AddNumericField(FieldName name, bool sortable = false, bool noIndex = false) { - Fields.Add(new Field(name, FieldType.Numeric, true)); + Fields.Add(new NumericField(name, sortable, noIndex)); return this; } /// - /// Add a numeric field that can be sorted on. + /// Add a Numeric field to the schema. /// /// The field's name. + /// If true, the text field can be sorted. + /// Attributes can have the NOINDEX option, which means they will not be indexed. /// The object. - public Schema AddSortableNumericField(FieldName name) + public Schema AddNumericField(string name, bool sortable = false, bool noIndex = false) { - Fields.Add(new Field(name, FieldType.Numeric, true)); + Fields.Add(new NumericField(name, sortable, noIndex)); return this; } /// - /// Add a TAG field. + /// Add a Tag field to the schema. /// /// The field's name. - /// The tag separator. - /// If true, Keeps the original letter cases of the tags. /// If true, the field can be sorted. - /// Set this to true to prevent the indexer from sorting on the normalized form. - /// Normalied form is the field sent to lower case with all diaretics removed - /// The object. - public Schema AddTagField(string name, string separator = ",", bool caseSensitive = false, bool sortable = false, bool unNormalizedForm = false) - { - Fields.Add(new TagField(name, separator, caseSensitive, sortable, unNormalizedForm)); - return this; - } - - /// - /// Add a TAG field. - /// - /// The field's name. + /// Set this to true to prevent the indexer from sorting on the normalized form. + /// Attributes can have the NOINDEX option, which means they will not be indexed. /// The tag separator. /// If true, Keeps the original letter cases of the tags. - /// If true, the field can be sorted. - /// Set this to true to prevent the indexer from sorting on the normalized form. /// Normalied form is the field sent to lower case with all diaretics removed + /// Keeps a suffix trie with all terms which match the suffix. /// The object. - public Schema AddTagField(FieldName name, string separator = ",", bool caseSensitive = false, bool sortable = false, bool unNormalizedForm = false) + public Schema AddTagField(FieldName name, bool sortable = false, bool unf = false, + bool noIndex = false, string separator = ",", + bool caseSensitive = false, bool withSuffixTrie = false) { - Fields.Add(new TagField(name, separator, caseSensitive, sortable, unNormalizedForm)); + Fields.Add(new TagField(name, sortable, unf, noIndex, separator, caseSensitive, withSuffixTrie)); return this; } /// - /// Add a sortable TAG field. + /// Add a Tag field to the schema. /// /// The field's name. - /// The tag separator. - /// Set this to true to prevent the indexer from sorting on the normalized form. - /// Normalied form is the field sent to lower case with all diaretics removed - /// The object. - public Schema AddSortableTagField(string name, string separator = ",", bool unNormalizedForm = false) - { - Fields.Add(new TagField(name, separator, sortable: true, unNormalizedForm: unNormalizedForm)); - return this; - } - - /// - /// Add a sortable TAG field. - /// - /// The field's name. - /// The tag separator. - /// Set this to true to prevent the indexer from sorting on the normalized form. - /// Normalied form is the field sent to lower case with all diaretics removed - /// The object. - public Schema AddSortableTagField(FieldName name, string separator = ",", bool unNormalizedForm = false) - { - Fields.Add(new TagField(name, separator, sortable: true, unNormalizedForm: unNormalizedForm)); - return this; - } - - /// - /// Add a TAG field. - /// - /// The field's name. - /// The vector similarity algorithm to use. - /// The algorithm attributes for the creation of the vector index. /// If true, the field can be sorted. + /// Set this to true to prevent the indexer from sorting on the normalized form. /// Attributes can have the NOINDEX option, which means they will not be indexed. - /// Set this to true to prevent the indexer from sorting on the normalized form. + /// The tag separator. + /// If true, Keeps the original letter cases of the tags. /// Normalied form is the field sent to lower case with all diaretics removed + /// Keeps a suffix trie with all terms which match the suffix. /// The object. - public Schema AddVectorField(string name, VectorAlgo algorithm, Dictionary attributes, - bool sortable = false, bool noIndex = false, bool unNormalizedForm = false) + public Schema AddTagField(string name, bool sortable = false, bool unf = false, + bool noIndex = false, string separator = ",", + bool caseSensitive = false, bool withSuffixTrie = false) { - Fields.Add(new VectorField(name, algorithm, attributes, sortable, noIndex, unNormalizedForm)); + Fields.Add(new TagField(name, sortable, unf, noIndex, separator, caseSensitive, withSuffixTrie)); return this; } /// - /// Add a TAG field. + /// Add a Vector field to the schema. /// /// The field's name. /// The vector similarity algorithm to use. /// The algorithm attributes for the creation of the vector index. - /// If true, the field can be sorted. - /// Attributes can have the NOINDEX option, which means they will not be indexed. - /// Set this to true to prevent the indexer from sorting on the normalized form. - /// Normalied form is the field sent to lower case with all diaretics removed /// The object. - public Schema AddVectorField(FieldName name, VectorAlgo algorithm, Dictionary attributes, - bool sortable = false, bool noIndex = false, bool unNormalizedForm = false) + public Schema AddVectorField(string name, VectorAlgo algorithm, Dictionary? attributes = null) { - Fields.Add(new VectorField(name, algorithm, attributes, sortable, noIndex, unNormalizedForm)); + Fields.Add(new VectorField(name, algorithm, attributes)); return this; } } diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index cbca7286..20db480a 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -126,13 +126,13 @@ public RedisResult Info(RedisValue index) public bool Create(string indexName, FTCreateParams parameters, Schema schema) { var args = new List() { indexName }; - parameters.AddParams(args); + parameters.AddParams(args); // TODO: Think of a better implementation args.Add("SCHEMA"); foreach (var f in schema.Fields) { - f.SerializeRedisArgs(args); + f.AddSchemaArgs(args); } return _db.Execute(FT.CREATE, args).OKtoBoolean(); From 90de6ae24fdd0493073517750074f1d7d1823c36 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Thu, 22 Sep 2022 15:41:03 +0300 Subject: [PATCH 07/28] Add FT.SEARCH command --- src/NRedisStack/Search/Document.cs | 93 +++ .../Search/FT.CREATE/FTCreateParams.cs | 15 +- src/NRedisStack/Search/Query.cs | 680 ++++++++++++++++++ src/NRedisStack/Search/SearchCommands.cs | 18 + src/NRedisStack/Search/SearchResult.cs | 98 +++ tests/NRedisStack.Tests/Search/SearchTests.cs | 78 +- 6 files changed, 963 insertions(+), 19 deletions(-) create mode 100644 src/NRedisStack/Search/Document.cs create mode 100644 src/NRedisStack/Search/Query.cs create mode 100644 src/NRedisStack/Search/SearchResult.cs diff --git a/src/NRedisStack/Search/Document.cs b/src/NRedisStack/Search/Document.cs new file mode 100644 index 00000000..96880700 --- /dev/null +++ b/src/NRedisStack/Search/Document.cs @@ -0,0 +1,93 @@ +using System.Collections.Generic; +using StackExchange.Redis; + +namespace NRedisStack.Search +{ + /// + /// Document represents a single indexed document or entity in the engine + /// + public class Document + { + public string Id { get; } + public double Score { get; set;} + public byte[] Payload { get; } + public string[] ScoreExplained { get; private set; } // TODO: check if this is needed (Jedis does not have it) + internal readonly Dictionary _properties; + public Document(string id, double score, byte[] payload) : this(id, null, score, payload) { } + public Document(string id) : this(id, null, 1.0, null) { } + + public Document(string id, Dictionary fields, double score = 1.0) : this(id, fields, score, null) { } + + public Document(string id, Dictionary fields, double score, byte[] payload) + { + Id = id; + _properties = fields ?? new Dictionary(); + Score = score; + Payload = payload; + } + + public IEnumerable> GetProperties() => _properties; + + public static Document Load(string id, double score, byte[] payload, RedisValue[] fields) + { + Document ret = new Document(id, score, payload); + if (fields != null) + { + for (int i = 0; i < fields.Length; i += 2) + { + string fieldName = (string)fields[i]; + if (fieldName == "$") { + ret["json"] = fields[i + 1]; + } + else { + ret[fieldName] = fields[i + 1]; + } + } + } + return ret; + } + + public static Document Load(string id, double score, byte[] payload, RedisValue[] fields, string[] scoreExplained) + { + Document ret = Document.Load(id, score, payload, fields); + if (scoreExplained != null) + { + ret.ScoreExplained = scoreExplained; + } + return ret; + } + + public RedisValue this[string key] + { + get { return _properties.TryGetValue(key, out var val) ? val : default(RedisValue); } + internal set { _properties[key] = value; } + } + + public bool HasProperty(string key) => _properties.ContainsKey(key); + + internal static Document Parse(string docId, RedisResult result) + { + if (result == null || result.IsNull) return null; + var arr = (RedisResult[])result; + var doc = new Document(docId); + + for(int i = 0; i < arr.Length; ) + { + doc[(string)arr[i++]] = (RedisValue)arr[i++]; + } + return doc; + } + + public Document Set(string field, RedisValue value) + { + this[field] = value; + return this; + } + + public Document SetScore(double score) + { + Score = score; + return this; + } + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs b/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs index 32f1ca30..895c4cb6 100644 --- a/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs +++ b/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs @@ -27,7 +27,7 @@ public FTCreateParams() { } - public static FTCreateParams createParams() + public static FTCreateParams CreateParams() { return new FTCreateParams(); } @@ -36,7 +36,7 @@ public static FTCreateParams createParams() /// Currently supports HASH (default) and JSON. To index JSON, you must have the RedisJSON module /// installed. /// - public FTCreateParams on(IndexDataType dataType) + public FTCreateParams On(IndexDataType dataType) { this.dataType = dataType; return this; @@ -220,7 +220,7 @@ public FTCreateParams SkipInitialScan() public void AddParams(List args) { - if (dataType != null) + if (dataType != default(IndexDataType)) { args.Add("ON"); args.Add(dataType.AsArg()); @@ -252,7 +252,7 @@ public void AddParams(List args) args.Add(languageField); } - if (score != null) + if (score != default(double)) { args.Add(SearchArgs.SCORE); args.Add(score); @@ -279,7 +279,7 @@ public void AddParams(List args) args.Add(SearchArgs.NOOFFSETS); } - if (temporary != null) + if (temporary != default(long)) { args.Add(SearchArgs.TEMPORARY); args.Add(temporary); @@ -311,11 +311,6 @@ public void AddParams(List args) { args.Add(SearchArgs.SKIPINITIALSCAN); } - // here sepose to be: - // SCHEMA field_name[AS alias] TEXT | TAG | NUMERIC | GEO | VECTOR[SORTABLE[UNF]] - // [NOINDEX][field_name[AS alias] TEXT | TAG | NUMERIC | GEO | VECTOR[SORTABLE[UNF]][NOINDEX]...] - - } } } \ No newline at end of file diff --git a/src/NRedisStack/Search/Query.cs b/src/NRedisStack/Search/Query.cs new file mode 100644 index 00000000..a145eb39 --- /dev/null +++ b/src/NRedisStack/Search/Query.cs @@ -0,0 +1,680 @@ +using System.Collections.Generic; +using System.Globalization; +using NRedisStack.Search; +using StackExchange.Redis; + +namespace NRedisStack.Search +{ + /// + /// Query represents query parameters and filters to load results from the engine + /// + public sealed class Query + { + /// + /// Filter represents a filtering rules in a query + /// + public abstract class Filter + { + public string Property { get; } + + internal abstract void SerializeRedisArgs(List args); + + internal Filter(string property) + { + Property = property; + } + } + + /// + /// NumericFilter wraps a range filter on a numeric field. It can be inclusive or exclusive + /// + public class NumericFilter : Filter + { + private readonly double min, max; + private readonly bool exclusiveMin, exclusiveMax; + + public NumericFilter(string property, double min, bool exclusiveMin, double max, bool exclusiveMax) : base(property) + { + this.min = min; + this.max = max; + this.exclusiveMax = exclusiveMax; + this.exclusiveMin = exclusiveMin; + } + + public NumericFilter(string property, double min, double max) : this(property, min, false, max, false) { } + + internal override void SerializeRedisArgs(List args) + { + static RedisValue FormatNum(double num, bool exclude) //TODO: understand this: + { + if (!exclude || double.IsInfinity(num)) + { + return (RedisValue)num; // can use directly + } + // need to add leading bracket + return "(" + num.ToString("G17", NumberFormatInfo.InvariantInfo); + } + args.Add("FILTER"); + args.Add(Property); + args.Add(FormatNum(min, exclusiveMin)); + args.Add(FormatNum(max, exclusiveMax)); + } + } + + /// + /// GeoFilter encapsulates a radius filter on a geographical indexed fields + /// + public class GeoFilter : Filter + { + public static readonly string KILOMETERS = "km"; + public static readonly string METERS = "m"; + public static readonly string FEET = "ft"; + public static readonly string MILES = "mi"; + private readonly double lon, lat, radius; + private readonly string unit; // TODO: think about implementing this as an enum + + public GeoFilter(string property, double lon, double lat, double radius, string unit) : base(property) + { + this.lon = lon; + this.lat = lat; + this.radius = radius; + this.unit = unit; + } + + internal override void SerializeRedisArgs(List args) + { + args.Add("GEOFILTER"); + args.Add(Property); + args.Add(lon); + args.Add(lat); + args.Add(radius); + args.Add(unit); + } + } + + internal readonly struct Paging + { + public int Offset { get; } + public int Count { get; } + + public Paging(int offset, int count) + { + Offset = offset; + Count = count; + } + } + + public readonly struct HighlightTags + { + public HighlightTags(string open, string close) + { + Open = open; + Close = close; + } + public string Open { get; } + public string Close { get; } + } + + /// + /// The query's filter list. We only support AND operation on all those filters + /// + internal readonly List _filters = new List(); + + /// + /// The textual part of the query + /// + public string QueryString { get; } + + /// + /// The sorting parameters + /// + internal Paging _paging = new Paging(0, 10); + + /// + /// Set the query to verbatim mode, disabling stemming and query expansion + /// + public bool Verbatim { get; set; } + /// + /// Set the query not to return the contents of documents, and rather just return the ids + /// + public bool NoContent { get; set; } + /// + /// Set the query not to filter for stopwords. In general this should not be used + /// + public bool NoStopwords { get; set; } + /// + /// Set the query to return a factored score for each results. This is useful to merge results from multiple queries. + /// + public bool WithScores { get; set; } + /// + /// Set the query to return object payloads, if any were given + /// + public bool WithPayloads { get; set; } + + /// + /// Set the query language, for stemming purposes; see http://redisearch.io for documentation on languages and stemming + /// + public string Language { get; set; } + + internal string[] _fields = null; + internal string[] _keys = null; + internal string[] _returnFields = null; + internal FieldName[] _returnFieldsNames = null; + internal string[] _highlightFields = null; + internal string[] _summarizeFields = null; + internal HighlightTags? _highlightTags = null; + internal string _summarizeSeparator = null; + internal int _summarizeNumFragments = -1, _summarizeFragmentLen = -1; + + /// + /// Set the query payload to be evaluated by the scoring function + /// + public byte[] Payload { get; set; } + + // TODO: Check if I need to add here WITHSORTKEYS + + /// + /// Set the query parameter to sort by + /// + public string SortBy { get; set; } + + /// + /// Set the query parameter to sort by ASC by default + /// + public bool SortAscending { get; set; } = true; + + // highlight and summarize + internal bool _wantsHighlight = false, _wantsSummarize = false; + + /// + /// Set the query scoring. see https://oss.redislabs.com/redisearch/Scoring.html for documentation + /// + public string Scorer { get; set; } + public bool ExplainScore { get; set; } // TODO: Check if this is needed because Jedis doesn't have it + + private Dictionary _params = null; + private int _dialect = 0; + private int _slop = -1; + private long _timeout = -1; + private bool _inOrder = false; + private string _expander = null; + + public Query() : this("*") { } + + /// + /// Create a new index + /// + /// The query string to use for this query. + public Query(string queryString) + { + QueryString = queryString; + } + + internal void SerializeRedisArgs(List args) + { + args.Add(QueryString); + + if (Verbatim) + { + args.Add("VERBATIM"); + } + if (NoContent) + { + args.Add("NOCONTENT"); + } + if (NoStopwords) + { + args.Add("NOSTOPWORDS"); + } + if (WithScores) + { + args.Add("WITHSCORES"); + } + if (WithPayloads) + { + args.Add("WITHPAYLOADS"); + } + if (Language != null) + { + args.Add("LANGUAGE"); + args.Add(Language); + } + + if (Scorer != null) + { + args.Add("SCORER"); + args.Add(Scorer); + + if (ExplainScore) + { + args.Add("EXPLAINSCORE"); // TODO: Check Why Jedis doesn't have it + } + } + + if (_fields?.Length > 0) + { + args.Add("INFIELDS"); + args.Add(_fields.Length); + args.AddRange(_fields); + } + + if (SortBy != null) + { + args.Add("SORTBY"); + args.Add(SortBy); + args.Add((SortAscending ? "ASC" : "DESC")); + } + if (Payload != null) + { + args.Add("PAYLOAD"); + args.Add(Payload); + } + + if (_paging.Offset != 0 || _paging.Count != 10) + { + args.Add("LIMIT"); + args.Add(_paging.Offset); + args.Add(_paging.Count); + } + + if (_filters?.Count > 0) + { + foreach (var f in _filters) + { + f.SerializeRedisArgs(args); + } + } + + if (_wantsHighlight) + { + args.Add("HIGHLIGHT"); + if (_highlightFields != null) + { + args.Add("FIELDS"); + args.Add(_highlightFields.Length); + foreach (var s in _highlightFields) + { + args.Add(s); + } + } + if (_highlightTags != null) + { + args.Add("TAGS"); + var tags = _highlightTags.GetValueOrDefault(); + args.Add(tags.Open); + args.Add(tags.Close); + } + } + if (_wantsSummarize) + { + args.Add("SUMMARIZE"); + if (_summarizeFields != null) + { + args.Add("FIELDS"); + args.Add(_summarizeFields.Length); + foreach (var s in _summarizeFields) + { + args.Add(s); + } + } + if (_summarizeNumFragments != -1) + { + args.Add("FRAGS"); + args.Add(_summarizeNumFragments); + } + if (_summarizeFragmentLen != -1) + { + args.Add("LEN"); + args.Add(_summarizeFragmentLen); + } + if (_summarizeSeparator != null) + { + args.Add("SEPARATOR"); + args.Add(_summarizeSeparator); + } + } + + if (_keys != null && _keys.Length > 0) + { + args.Add("INKEYS"); + args.Add(_keys.Length); + + foreach (var key in _keys) + { + args.Add(key); + } + } + + if (_keys?.Length > 0) + { + args.Add("INKEYS"); + args.Add(_keys.Length); + args.AddRange(_keys); + } + if (_returnFields?.Length > 0) + { + args.Add("RETURN"); + args.Add(_returnFields.Length); + args.AddRange(_returnFields); + } + else if (_returnFieldsNames?.Length > 0) // TODO: understad this + { + args.Add("RETURN"); + int returnCountIndex = args.Count; + int returnCount = 0; + foreach (FieldName fn in _returnFieldsNames) { + returnCount += fn.AddCommandArguments(args); + } + + args.Insert(returnCountIndex, returnCount); + } + if (_params != null && _params.Count > 0) + { + args.Add("PARAMS"); + args.Add(_params.Count * 2); + foreach (var entry in _params) + { + args.Add(entry.Key); + args.Add(entry.Value); + } + } + + if (_dialect != 0) + { + args.Add("DIALECT"); + args.Add(_dialect); + } + + if (_slop >= 0) + { + args.Add("SLOP"); + args.Add(_slop); + } + + if (_timeout >= 0) + { + args.Add("TIMEOUT"); + args.Add(_timeout); + } + + if (_inOrder) + { + args.Add("INORDER"); + } + + if (_expander != null) + { + args.Add("EXPANDER"); + args.Add(_expander); + } + } + + // TODO: check if DelayedRawable is needed here (Jedis have it) + + /// + /// Limit the results to a certain offset and limit + /// + /// the first result to show, zero based indexing + /// how many results we want to show + /// the query itself, for builder-style syntax + public Query Limit(int offset, int count) + { + _paging = new Paging(offset, count); + return this; + } + + /// + /// Add a filter to the query's filter list + /// + /// either a numeric or geo filter object + /// the query itself + public Query AddFilter(Filter f) + { + _filters.Add(f); + return this; + } + + /// + /// Set the query payload to be evaluated by the scoring function + /// + /// the payload + /// the query itself + public Query SetPayload(byte[] payload) + { + Payload = payload; + return this; + } + + /// + /// Set the query to verbatim mode, disabling stemming and query expansion + /// + /// the query itself + public Query SetVerbatim(bool value = true) + { + Verbatim = value; + return this; + } + + /// + /// Set the query not to return the contents of documents, and rather just return the ids + /// + /// the query itself + public Query SetNoContent(bool value = true) + { + NoContent = value; + return this; + } + + /// + /// Set the query not to filter for stopwords. In general this should not be used + /// + /// the query itself + public Query SetNoStopwords(bool value = true) + { + NoStopwords = value; + return this; + } + + /// + /// Set the query to return a factored score for each results. This is useful to merge results from + /// multiple queries. + /// + /// the query itself + public Query SetWithScores(bool value = true) + { + WithScores = value; + return this; + } + + /// + /// Set the query to return object payloads, if any were given + /// + /// the query itself + public Query SetWithPayload() + { + WithPayloads = true; + return this; + } + + /// + /// Set the query language, for stemming purposes + /// + /// the language + /// the query itself + public Query SetLanguage(string language) + { + Language = language; + return this; + } + + /// + /// Set the query language, for stemming purposes + /// + /// + /// + public Query SetScorer(string scorer) + { + Scorer = scorer; + return this; + } + /// + /// Limit the query to results that are limited to a specific set of fields + /// + /// a list of TEXT fields in the schemas + /// the query object itself + public Query LimitFields(params string[] fields) + { + _fields = fields; + return this; + } + + /// + /// Limit the query to results that are limited to a specific set of keys + /// + /// a list of the TEXT fields in the schemas + /// the query object itself + public Query LimitKeys(params string[] keys) + { + _keys = keys; + return this; + } + + /// + /// Result's projection - the fields to return by the query + /// + /// fields a list of TEXT fields in the schemas + /// the query object itself + public Query ReturnFields(params string[] fields) + { + _returnFields = fields; + _returnFieldsNames = null; + return this; + } + + /// + /// Result's projection - the fields to return by the query + /// + /// field a list of TEXT fields in the schemas + /// the query object itself + public Query ReturnFields(params FieldName[] fields) + { + _returnFields = null; + _returnFieldsNames = fields; + return this; + } + + public Query HighlightFields(HighlightTags tags, params string[] fields) => HighlightFieldsImpl(tags, fields); + public Query HighlightFields(params string[] fields) => HighlightFieldsImpl(null, fields); + private Query HighlightFieldsImpl(HighlightTags? tags, string[] fields) + { + if (fields == null || fields.Length > 0) + { + _highlightFields = fields; + } + _highlightTags = tags; + _wantsHighlight = true; + return this; + } + + public Query SummarizeFields(int contextLen, int fragmentCount, string separator, params string[] fields) + { + if (fields == null || fields.Length > 0) + { + _summarizeFields = fields; + } + _summarizeFragmentLen = contextLen; + _summarizeNumFragments = fragmentCount; + _summarizeSeparator = separator; + _wantsSummarize = true; + return this; + } + + public Query SummarizeFields(params string[] fields) => SummarizeFields(-1, -1, null, fields); + + /// + /// Set the query to be sorted by a sortable field defined in the schema + /// + /// the sorting field's name + /// if set to true, the sorting order is ascending, else descending + /// the query object itself + public Query SetSortBy(string field, bool ascending = true) + { + SortBy = field; + SortAscending = ascending; + return this; + } + + /// + /// Parameters can be referenced in the query string by a $ , followed by the parameter name, + /// e.g., $user , and each such reference in the search query to a parameter name is substituted + /// by the corresponding parameter value. + /// + /// + /// can be String, long or float + /// The query object itself + public Query AddParam(String name, Object value) + { + if (_params == null) + { + _params = new Dictionary(); + } + _params.Add(name, value); + return this; + } + + /// + /// Set the dialect version to execute the query accordingly + /// + /// + /// the query object itself + public Query Dialect(int dialect) + { + _dialect = dialect; + return this; + } + + /// + /// Set the slop to execute the query accordingly + /// + /// + /// the query object itself + public Query Slop(int slop) + { + _slop = slop; + return this; + } + + /// + /// Set the timeout to execute the query accordingly + /// + /// + /// the query object itself + public Query Timeout(long timeout) + { + _timeout = timeout; + return this; + } + + /// + /// Set the query terms appear in the same order in the document as in the query, regardless of the offsets between them + /// + /// the query object + public Query SetInOrder() + { + this._inOrder = true; + return this; + } + + /// + /// Set the query to use a custom query expander instead of the stemmer + /// + /// + /// the query object itself + + public Query SetExpander(String field) + { + _expander = field; + return this; + } + } +} diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index 20db480a..1c144228 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -137,5 +137,23 @@ public bool Create(string indexName, FTCreateParams parameters, Schema schema) return _db.Execute(FT.CREATE, args).OKtoBoolean(); } + + /// + /// Search the index + /// + /// The index name + /// a object with the query string and optional parameters + /// a object with the results + public SearchResult Search(string indexName, Query q) + { + var args = new List{indexName}; + // { + // _boxedIndexName + // }; + q.SerializeRedisArgs(args); + + var resp = _db.Execute("FT.SEARCH", args).ToArray(); + return new SearchResult(resp, !q.NoContent, q.WithScores, q.WithPayloads, q.ExplainScore); + } } } \ No newline at end of file diff --git a/src/NRedisStack/Search/SearchResult.cs b/src/NRedisStack/Search/SearchResult.cs new file mode 100644 index 00000000..a8ea49d3 --- /dev/null +++ b/src/NRedisStack/Search/SearchResult.cs @@ -0,0 +1,98 @@ +using StackExchange.Redis; +using System.Collections.Generic; +using System.Linq; + +namespace NRedisStack.Search +{ + /// + /// SearchResult encapsulates the returned result from a search query. + /// It contains publically accessible fields for the total number of results, and an array of + /// objects conatining the actual returned documents. + /// + public class SearchResult + { + public long TotalResults { get; } + public List Documents { get; } + + internal SearchResult(RedisResult[] resp, bool hasContent, bool hasScores, bool hasPayloads, bool shouldExplainScore) + { + // Calculate the step distance to walk over the results. + // The order of results is id, score (if withScore), payLoad (if hasPayloads), fields + int step = 1; + int scoreOffset = 0; + int contentOffset = 1; + int payloadOffset = 0; + if (hasScores) + { + step++; + scoreOffset = 1; + contentOffset++; + + } + if (hasContent) + { + step++; + if (hasPayloads) + { + payloadOffset = scoreOffset + 1; + step++; + contentOffset++; + } + } + + // the first element is always the number of results + TotalResults = (long)resp[0]; + var docs = new List((resp.Length - 1) / step); + Documents = docs; + for (int i = 1; i < resp.Length; i += step) + { + var id = (string)resp[i]; + double score = 1.0; + byte[] payload = null; + RedisValue[] fields = null; + string[] scoreExplained = null; + if (hasScores) + { + if (shouldExplainScore) + { + var scoreResult = (RedisResult[])resp[i + scoreOffset]; + score = (double) scoreResult[0]; + var redisResultsScoreExplained = (RedisResult[]) scoreResult[1]; + scoreExplained = FlatRedisResultArray(redisResultsScoreExplained).ToArray(); + } + else + { + score = (double)resp[i + scoreOffset]; + } + } + if (hasPayloads) + { + payload = (byte[])resp[i + payloadOffset]; + } + + if (hasContent) + { + fields = (RedisValue[])resp[i + contentOffset]; + } + + docs.Add(Document.Load(id, score, payload, fields, scoreExplained)); + } + } + + static IEnumerable FlatRedisResultArray(RedisResult[] collection) + { + foreach (var o in collection) + { + if (o.Type == ResultType.MultiBulk) + { + foreach (string t in FlatRedisResultArray((RedisResult[])o)) + yield return t; + } + else + { + yield return o.ToString(); + } + } + } + } +} \ No newline at end of file diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index ad62fa21..71b79064 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -4,6 +4,7 @@ using Moq; using NRedisStack.Search.FT.CREATE; using NRedisStack.Search; +using static NRedisStack.Search.Schema; namespace NRedisStack.Tests.Search; @@ -26,7 +27,7 @@ public void TestCreate() db.Execute("FLUSHALL"); var ft = db.FT(); var schema = new Schema().AddTextField("first").AddTextField("last").AddNumericField("age"); - var parameters = FTCreateParams.createParams().Filter("@age>16").Prefix("student:", "pupil:"); + var parameters = FTCreateParams.CreateParams().Filter("@age>16").Prefix("student:", "pupil:"); Assert.True(ft.Create(index, parameters, schema)); @@ -38,19 +39,78 @@ public void TestCreate() db.HashSet("student:5555", new HashEntry[] { new("first", "Joen"), new("last", "Ko"), new("age", "20") }); db.HashSet("teacher:6666", new HashEntry[] { new("first", "Pat"), new("last", "Rod"), new("age", "20") }); - // var noFilters = ft.Search(index, new Query()); - // Assert.Equal(4, noFilters.getTotalResults()); + var noFilters = ft.Search(index, new Query()); + Assert.Equal(4, noFilters.TotalResults); - // var res1 = ft.Search(index, new Query("@first:Jo*")); - // Assert.Equal(2, res1.getTotalResults()); + var res1 = ft.Search(index, new Query("@first:Jo*")); + Assert.Equal(2, res1.TotalResults); - // var res2 = ft.Search(index, new Query("@first:Pat")); - // Assert.Equal(1, res2.getTotalResults()); + var res2 = ft.Search(index, new Query("@first:Pat")); + Assert.Equal(1, res2.TotalResults); - // var res3 = ft.Search(index, new Query("@last:Rod")); - // Assert.Equal(0, res3.getTotalResults()); + var res3 = ft.Search(index, new Query("@last:Rod")); + Assert.Equal(0, res3.TotalResults); } + [Fact] + public void CreateNoParams() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + + Schema sc = new Schema().AddTextField("first", 1.0).AddTextField("last", 1.0).AddNumericField("age"); + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + + db.HashSet("student:1111", new HashEntry[] { new("first", "Joe"), new("last", "Dod"), new("age", 18) }); + db.HashSet("student:3333", new HashEntry[] { new("first", "El"), new("last", "Mark"), new("age", 17) }); + db.HashSet("pupil:4444", new HashEntry[] { new("first", "Pat"), new("last", "Shu"), new("age", 21) }); + db.HashSet("student:5555", new HashEntry[] { new("first", "Joen"), new("last", "Ko"), new("age", 20) }); + + SearchResult noFilters = ft.Search(index, new Query()); + Assert.Equal(4, noFilters.TotalResults); + + SearchResult res1 = ft.Search(index, new Query("@first:Jo*")); + Assert.Equal(2, res1.TotalResults); + + SearchResult res2 = ft.Search(index, new Query("@first:Pat")); + Assert.Equal(1, res2.TotalResults); + + SearchResult res3 = ft.Search(index, new Query("@last:Rod")); + Assert.Equal(0, res3.TotalResults); + } + + [Fact] + public void CreateWithFieldNames() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema().AddField(new TextField(FieldName.Of("first").As("given"))) + .AddField(new TextField(FieldName.Of("last"))); + + Assert.True(ft.Create(index, FTCreateParams.CreateParams().Prefix("student:", "pupil:"), sc)); + + db.HashSet("profesor:5555", new HashEntry[] { new("first", "Albert"), new("last", "Blue"), new("age", "55") }); + db.HashSet("student:1111", new HashEntry[] { new("first", "Joe"), new("last", "Dod"), new("age", "18") }); + db.HashSet("pupil:2222", new HashEntry[] { new("first", "Jen"), new("last", "Rod"), new("age", "14") }); + db.HashSet("student:3333", new HashEntry[] { new("first", "El"), new("last", "Mark"), new("age", "17") }); + db.HashSet("pupil:4444", new HashEntry[] { new("first", "Pat"), new("last", "Shu"), new("age", "21") }); + db.HashSet("student:5555", new HashEntry[] { new("first", "Joen"), new("last", "Ko"), new("age", "20") }); + db.HashSet("teacher:6666", new HashEntry[] { new("first", "Pat"), new("last", "Rod"), new("age", "20") }); + + SearchResult noFilters = ft.Search(index, new Query()); + Assert.Equal(5, noFilters.TotalResults); + + SearchResult asOriginal = ft.Search(index, new Query("@first:Jo*")); + Assert.Equal(0, asOriginal.TotalResults); + + SearchResult asAttribute = ft.Search(index, new Query("@given:Jo*")); + Assert.Equal(2, asAttribute.TotalResults); + + SearchResult nonAttribute = ft.Search(index, new Query("@last:Rod")); + Assert.Equal(1, nonAttribute.TotalResults); + } [Fact] public void TestModulePrefixs() From 845196b6d5e8dd75217c8a5431dc5fd567971db1 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Thu, 22 Sep 2022 17:04:02 +0300 Subject: [PATCH 08/28] Add FT.ALTER command --- src/NRedisStack/Search/SearchCommands.cs | 19 ++++++---- tests/NRedisStack.Tests/Search/SearchTests.cs | 36 +++++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index 1c144228..d10ca631 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -99,15 +99,22 @@ public async Task AliasUpdateAsync(string alias, string index) /// /// Add a new attribute to the index /// - /// The index name to create. + /// The index name. /// If set, does not scan and index. - /// attribute to add. - /// attribute options. + /// the schema. /// if executed correctly, error otherwise - /// - public bool Alter(string alias, string index) + /// + public bool Alter(string index, Schema schema, bool skipInitialScan = false) { - return _db.Execute(FT.ALIASUPDATE, alias, index).OKtoBoolean(); + List args = new List(){index}; + if (skipInitialScan) args.Add("SKIPINITIALSCAN"); + args.Add("SCHEMA"); + args.Add("ADD"); + foreach (var f in schema.Fields) + { + f.AddSchemaArgs(args); + } + return _db.Execute(FT.ALTER, args).OKtoBoolean(); } public RedisResult Info(RedisValue index) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 71b79064..f762e264 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -112,6 +112,42 @@ public void CreateWithFieldNames() Assert.Equal(1, nonAttribute.TotalResults); } + [Fact] + public void AlterAdd() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema().AddTextField("title", 1.0); + + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + var fields = new HashEntry("title", "hello world"); + //fields.("title", "hello world"); + for (int i = 0; i < 100; i++) + { + db.HashSet($"doc{i}", fields.Name, fields.Value); + } + SearchResult res = ft.Search(index, new Query("hello world")); + Assert.Equal(100, res.TotalResults); + + Assert.True(ft.Alter(index, new Schema().AddTagField("tags").AddTextField("name", weight: 0.5))); + for (int i = 0; i < 100; i++) + { + var fields2 = new HashEntry[] { new("name", "name" + i), + new("tags", $"tagA,tagB,tag{i}") }; + // assertTrue(client.updateDocument(string.format("doc%d", i), 1.0, fields2)); + db.HashSet($"doc{i}", fields2); + } + SearchResult res2 = ft.Search(index, new Query("@tags:{tagA}")); + Assert.Equal(100, res2.TotalResults); + + //TODO: complete this test when I finish the command FT.INFO + // var info = ft.Info(index); + // Assert.Equal(index, info.get("index_name")); + // Assert.Equal("identifier", ((List)((List)info.get("attributes")).get(1)).get(0)); + // Assert.Equal("attribute", ((List)((List)info.get("attributes")).get(1)).get(2)); + } + [Fact] public void TestModulePrefixs() { From 5e55a8c237da29cf4ad897f9a21faefd44357bce Mon Sep 17 00:00:00 2001 From: Chayim Date: Tue, 6 Sep 2022 14:24:54 +0300 Subject: [PATCH 09/28] Linking to the latest pre-release (#30) * Linking to the latest pre-release * spelling and a link --- .github/wordlist.txt | 1 + README.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/wordlist.txt b/.github/wordlist.txt index d368bbab..6b418488 100644 --- a/.github/wordlist.txt +++ b/.github/wordlist.txt @@ -3,3 +3,4 @@ NRedisStack github yml Codecov +pre diff --git a/README.md b/README.md index fcc0608f..d8b9effd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ [![license](https://img.shields.io/github/license/redis/NRedisStack.svg)](https://raw.githubusercontent.com/redis/NRedisStack/master/LICENSE) [![.github/workflows/integration.yml](https://github.com/redis/NRedisStack/actions/workflows/integration.yml/badge.svg)](https://github.com/redis/NRedisStack/actions/workflows/integration.yml) -[![GitHub issues](https://img.shields.io/github/release/redis/NRedisStack.svg)](https://github.com/redis/NRedisStack/releases/latest) +[![pre-release](https://img.shields.io/github/v/release/redis/nredisstack?include_prereleases&label=prerelease)](https://github.com/redis/nredisstack/releases) + # NRedisStack From 083adb56fe954c547bc4e63a2ea2adad330d34c5 Mon Sep 17 00:00:00 2001 From: shacharPash <93581407+shacharPash@users.noreply.github.com> Date: Tue, 6 Sep 2022 15:22:06 +0300 Subject: [PATCH 10/28] Updading the version to v0.2.1 (#31) --- src/NRedisStack/NRedisStack.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/NRedisStack/NRedisStack.csproj b/src/NRedisStack/NRedisStack.csproj index cea8b902..112ea03e 100644 --- a/src/NRedisStack/NRedisStack.csproj +++ b/src/NRedisStack/NRedisStack.csproj @@ -7,9 +7,9 @@ Redis Open Source Redis OSS .Net Client for Redis Stack - 0.1.0 - 0.1.0 - 0.1.0 + 0.2.1 + 0.2.1 + 0.2.1 From 71f264d1758e65fd437541337fe890bb621124f0 Mon Sep 17 00:00:00 2001 From: Steve Lorello <42971704+slorello89@users.noreply.github.com> Date: Wed, 7 Sep 2022 09:14:20 -0400 Subject: [PATCH 11/28] adding JSON commands (#29) * adding JSON commands * Change commands names * some fixes * adding missing commands, removing visibility modifiers in interface * async breakout Co-authored-by: shacharPash --- src/NRedisStack/Auxiliary.cs | 14 + src/NRedisStack/Bloom/BloomCommands.cs | 6 +- src/NRedisStack/Json/IJsonCommands.cs | 498 +++++++++++++ src/NRedisStack/Json/JsonCommands.cs | 675 +++++++++++++++++- src/NRedisStack/Json/JsonType.cs | 13 + src/NRedisStack/Json/Literals/Commands.cs | 2 +- src/NRedisStack/ResponseParser.cs | 48 ++ tests/NRedisStack.Tests/Json/JsonTests.cs | 816 ++++++++++++++++++++-- 8 files changed, 2001 insertions(+), 71 deletions(-) create mode 100644 src/NRedisStack/Json/IJsonCommands.cs create mode 100644 src/NRedisStack/Json/JsonType.cs diff --git a/src/NRedisStack/Auxiliary.cs b/src/NRedisStack/Auxiliary.cs index 474489ce..29b40fea 100644 --- a/src/NRedisStack/Auxiliary.cs +++ b/src/NRedisStack/Auxiliary.cs @@ -10,5 +10,19 @@ public static List MergeArgs(RedisKey key, params RedisValue[] items) foreach (var item in items) args.Add(item); return args; } + + public static object[] AssembleNonNullArguments(params object?[] arguments) + { + var args = new List(); + foreach (var arg in arguments) + { + if (arg != null) + { + args.Add(arg); + } + } + + return args.ToArray(); + } } } \ No newline at end of file diff --git a/src/NRedisStack/Bloom/BloomCommands.cs b/src/NRedisStack/Bloom/BloomCommands.cs index dfc96deb..d0765adc 100644 --- a/src/NRedisStack/Bloom/BloomCommands.cs +++ b/src/NRedisStack/Bloom/BloomCommands.cs @@ -111,7 +111,7 @@ public bool[] Insert(RedisKey key, RedisValue[] items, int? capacity = null, throw new ArgumentOutOfRangeException(nameof(items)); var args = BloomAux.BuildInsertArgs(key, items, capacity, error, expansion, nocreate, nonscaling); - + return _db.Execute(BF.INSERT, args).ToBooleanArray(); } @@ -335,7 +335,7 @@ public async Task ReserveAsync(RedisKey key, double errorRate, long capaci /// Iterator value; either 0 or the iterator from a previous invocation of this command. /// Tuple of iterator and data. /// - public Tuple ScanDump(RedisKey key, long iterator) + public Tuple ScanDump(RedisKey key, long iterator) { return _db.Execute(BF.SCANDUMP, key, iterator).ToScanDumpTuple(); } @@ -347,7 +347,7 @@ public Tuple ScanDump(RedisKey key, long iterator) /// Iterator value; either 0 or the iterator from a previous invocation of this command. /// Tuple of iterator and data. /// - public async Task> ScanDumpAsync(RedisKey key, long iterator) + public async Task> ScanDumpAsync(RedisKey key, long iterator) { var result = await _db.ExecuteAsync(BF.SCANDUMP, key, iterator); return result.ToScanDumpTuple(); diff --git a/src/NRedisStack/Json/IJsonCommands.cs b/src/NRedisStack/Json/IJsonCommands.cs new file mode 100644 index 00000000..f8d5cd70 --- /dev/null +++ b/src/NRedisStack/Json/IJsonCommands.cs @@ -0,0 +1,498 @@ +using StackExchange.Redis; + +namespace NRedisStack; + +public interface IJsonCommands +{ + /// + /// Appends the provided items to the array at the provided path. + /// + /// The key to append to + /// The path to append to + /// the values to append + /// The new array sizes for the appended paths + /// + long?[] ArrAppend(RedisKey key, string? path = null, params object[] values); + + /// + /// Finds the index of the provided item within the provided range + /// + /// The key to look up. + /// The json path. + /// The value to find the index of. + /// The starting index within the array. Inclusive. + /// The ending index within the array. Exclusive + /// The index of the value for each array the path resolved to. + /// + long?[] ArrIndex(RedisKey key, string path, object value, long? start = null, long? stop = null); + + /// + /// Inserts the provided items at the provided index within a json array. + /// + /// The key to insert into. + /// The path of the array(s) within the key to insert into. + /// The index to insert at. + /// The values to insert + /// The new size of each array the item was inserted into. + /// + long?[] ArrInsert(RedisKey key, string path, long index, params object[] values); + + /// + /// Gets the length of the arrays resolved by the provided path. + /// + /// The key of the json object. + /// The path to the array(s) + /// The length of each array resolved by the json path. + /// + long?[] ArrLen(RedisKey key, string? path = null); + + /// + /// Pops an item from the array(s) at the provided index. Or the last element if no index is provided. + /// + /// The json key to use. + /// The path of the array(s). + /// The index to pop from + /// The items popped from the array + /// + RedisResult[] ArrPop(RedisKey key, string? path = null, long? index = null); + + /// + /// Trims the array(s) at the provided path, leaving the range between the specified indexes (inclusive). + /// + /// The key to trim from. + /// The path of the array(s) within the json object to trim. + /// the starting index to retain. + /// The ending index to retain. + /// The new length of the array(s) after they're trimmed. + /// + long?[] ArrTrim(RedisKey key, string path, long start, long stop); + + /// + /// Clear's container values(arrays/objects), and sets numeric values to 0. + /// + /// The key to clear. + /// The path to clear. + /// number of values cleared + /// + long Clear(RedisKey key, string? path = null); + + /// + /// Deletes a json value. + /// + /// The key to delete from. + /// The path to delete. + /// number of path's deleted + /// + long Del(RedisKey key, string? path = null); + + /// + /// Deletes a json value. + /// + /// The key to delete from. + /// The path to delete. + /// number of path's deleted + /// + long Forget(RedisKey key, string? path = null); + + /// + /// Gets the value stored at the key and path in redis. + /// + /// The key to retrieve. + /// the indentation string for nested levels + /// sets the string that's printed at the end of each line + /// sets the string that's put between a key and a value + /// the path to get. + /// The requested Items + /// + RedisResult Get(RedisKey key, RedisValue? indent = null, RedisValue? newLine = null, RedisValue? space = null, RedisValue? path = null); + + /// + /// Gets the values stored at the provided paths in redis. + /// + /// The key to pull from. + /// The paths within the key to pull. + /// the indentation string for nested levels + /// sets the string that's printed at the end of each line + /// sets the string that's put between a key and a value + /// + RedisResult Get(RedisKey key, string[] paths, RedisValue? indent = null, RedisValue? newLine = null, RedisValue? space = null); + + /// + /// Generically gets an Item stored in Redis. + /// + /// The key to retrieve + /// The path to retrieve + /// The type retrieved + /// The object requested + /// + T? Get(RedisKey key, string path = "$"); + + /// + /// retrieves a group of items stored in redis, appropriate if the path will resolve to multiple records. + /// + /// The key to pull from. + /// The path to pull. + /// The type. + /// An enumerable of the requested tyep + /// + IEnumerable GetEnumerable(RedisKey key, string path = "$"); + + /// + /// Gets the provided path from multiple keys + /// + /// The keys to retrieve from. + /// The path to retrieve + /// An array of RedisResults with the requested data. + /// + RedisResult[] MGet(RedisKey[] keys, string path); + + /// + /// Increments the fields at the provided path by the provided number. + /// + /// The key. + /// The path to increment. + /// The value to increment by. + /// The new values after being incremented, or null if the path resolved a non-numeric. + /// + double?[] NumIncrby(RedisKey key, string path, double value); + + /// + /// Gets the keys of the object at the provided path. + /// + /// the key of the json object. + /// The path of the object(s) + /// the keys of the resolved object(s) + /// + IEnumerable> ObjKeys(RedisKey key, string? path = null); + + /// + /// returns the number of keys in the object(s) at the provided path. + /// + /// The key of the json object. + /// The path of the object(s) to resolve. + /// The length of the object(s) keyspace. + /// + long?[] ObjLen(RedisKey key, string? path = null); + + /// + /// Gets the key in RESP(Redis Serialization Protocol) form. + /// + /// The key to get. + /// Path within the key to get. + /// the resultant resp + /// + RedisResult[] Resp(RedisKey key, string? path = null); + + /// + /// Set's the key/path to the provided value. + /// + /// The key. + /// The path to set within the key. + /// The value to set. + /// When to set the value. + /// The disposition of the command + /// + bool Set(RedisKey key, RedisValue path, object obj, When when = When.Always); + + /// + /// Set's the key/path to the provided value. + /// + /// The key. + /// The path to set within the key. + /// The value to set. + /// When to set the value. + /// The disposition of the command + /// + bool Set(RedisKey key, RedisValue path, RedisValue json, When when = When.Always); + + /// + /// Appends the provided string to the string(s) at the provided path. + /// + /// The key to append to. + /// The path of the string(s) to append to. + /// The value to append. + /// The new length of the string(s) appended to, those lengths will be null if the path did not resolve ot a string. + /// + long?[] StrAppend(RedisKey key, string value, string? path = null); + + /// + /// Check's the length of the string(s) at the provided path. + /// + /// The key of the json object. + /// The path of the string(s) within the json object. + /// The length of the string(s) appended to, those lengths will be null if the path did not resolve ot a string. + /// + public long?[] StrLen(RedisKey key, string? path = null); + + /// + /// Toggles the boolean value(s) at the provided path. + /// + /// The key of the json object. + /// The path of the value(s) to toggle. + /// the new value(s). Which will be null if the path did not resolve to a boolean. + /// + bool?[] Toggle(RedisKey key, string? path = null); + + /// + /// Gets the type(s) of the item(s) at the provided json path. + /// + /// The key of the JSON object. + /// The path to resolve. + /// An array of types. + /// + JsonType[] Type(RedisKey key, string? path = null); + + /// + /// Report a value's memory usage in bytes. path defaults to root if not provided. + /// + /// The object's key + /// The path within the object. + /// the value's size in bytes. + long DebugMemory(string key, string? path = null); + + /// + /// Appends the provided items to the array at the provided path. + /// + /// The key to append to + /// The path to append to + /// the values to append + /// The new array sizes for the appended paths + /// + Task ArrAppendAsync(RedisKey key, string? path = null, params object[] values); + + /// + /// Finds the index of the provided item within the provided range + /// + /// The key to look up. + /// The json path. + /// The value to find the index of. + /// The starting index within the array. Inclusive. + /// The ending index within the array. Exclusive + /// The index of the value for each array the path resolved to. + /// + Task ArrIndexAsync(RedisKey key, string path, object value, long? start = null, long? stop = null); + + /// + /// Inserts the provided items at the provided index within a json array. + /// + /// The key to insert into. + /// The path of the array(s) within the key to insert into. + /// The index to insert at. + /// The values to insert + /// The new size of each array the item was inserted into. + /// + Task ArrInsertAsync(RedisKey key, string path, long index, params object[] values); + + /// + /// Gets the length of the arrays resolved by the provided path. + /// + /// The key of the json object. + /// The path to the array(s) + /// The length of each array resolved by the json path. + /// + Task ArrLenAsync(RedisKey key, string? path = null); + + /// + /// Pops an item from the array(s) at the provided index. Or the last element if no index is provided. + /// + /// The json key to use. + /// The path of the array(s). + /// The index to pop from + /// The items popped from the array + /// + Task ArrPopAsync(RedisKey key, string? path = null, long? index = null); + + /// + /// Trims the array(s) at the provided path, leaving the range between the specified indexes (inclusive). + /// + /// The key to trim from. + /// The path of the array(s) within the json object to trim. + /// the starting index to retain. + /// The ending index to retain. + /// The new length of the array(s) after they're trimmed. + /// + Task ArrTrimAsync(RedisKey key, string path, long start, long stop); + + /// + /// Clear's container values(arrays/objects), and sets numeric values to 0. + /// + /// The key to clear. + /// The path to clear. + /// number of values cleared + /// + Task ClearAsync(RedisKey key, string? path = null); + + /// + /// Deletes a json value. + /// + /// The key to delete from. + /// The path to delete. + /// number of path's deleted + /// + Task DelAsync(RedisKey key, string? path = null); + + /// + /// Deletes a json value. + /// + /// The key to delete from. + /// The path to delete. + /// number of path's deleted + /// + Task ForgetAsync(RedisKey key, string? path = null); + + /// + /// Gets the value stored at the key and path in redis. + /// + /// The key to retrieve. + /// the indentation string for nested levels + /// sets the string that's printed at the end of each line + /// sets the string that's put between a key and a value + /// the path to get. + /// The requested Items + /// + Task GetAsync(RedisKey key, RedisValue? indent = null, RedisValue? newLine = null, RedisValue? space = null, RedisValue? path = null); + + /// + /// Gets the values stored at the provided paths in redis. + /// + /// The key to pull from. + /// The paths within the key to pull. + /// the indentation string for nested levels + /// sets the string that's printed at the end of each line + /// sets the string that's put between a key and a value + /// + Task GetAsync(RedisKey key, string[] paths, RedisValue? indent = null, RedisValue? newLine = null, RedisValue? space = null); + + /// + /// Generically gets an Item stored in Redis. + /// + /// The key to retrieve + /// The path to retrieve + /// The type retrieved + /// The object requested + /// + Task GetAsync(RedisKey key, string path = "$"); + + /// + /// retrieves a group of items stored in redis, appropriate if the path will resolve to multiple records. + /// + /// The key to pull from. + /// The path to pull. + /// The type. + /// An enumerable of the requested tyep + /// + Task> GetEnumerableAsync(RedisKey key, string path = "$"); + + /// + /// Gets the provided path from multiple keys + /// + /// The keys to retrieve from. + /// The path to retrieve + /// An array of RedisResults with the requested data. + /// + Task MGetAsync(RedisKey[] keys, string path); + + /// + /// Increments the fields at the provided path by the provided number. + /// + /// The key. + /// The path to increment. + /// The value to increment by. + /// The new values after being incremented, or null if the path resolved a non-numeric. + /// + Task NumIncrbyAsync(RedisKey key, string path, double value); + + /// + /// Gets the keys of the object at the provided path. + /// + /// the key of the json object. + /// The path of the object(s) + /// the keys of the resolved object(s) + /// + Task>> ObjKeysAsync(RedisKey key, string? path = null); + + /// + /// returns the number of keys in the object(s) at the provided path. + /// + /// The key of the json object. + /// The path of the object(s) to resolve. + /// The length of the object(s) keyspace. + /// + Task ObjLenAsync(RedisKey key, string? path = null); + + /// + /// Gets the key in RESP(Redis Serialization Protocol) form. + /// + /// The key to get. + /// Path within the key to get. + /// the resultant resp + /// + Task RespAsync(RedisKey key, string? path = null); + + /// + /// Set's the key/path to the provided value. + /// + /// The key. + /// The path to set within the key. + /// The value to set. + /// When to set the value. + /// The disposition of the command + /// + Task SetAsync(RedisKey key, RedisValue path, object obj, When when = When.Always); + + /// + /// Set's the key/path to the provided value. + /// + /// The key. + /// The path to set within the key. + /// The value to set. + /// When to set the value. + /// The disposition of the command + /// + Task SetAsync(RedisKey key, RedisValue path, RedisValue json, When when = When.Always); + + /// + /// Appends the provided string to the string(s) at the provided path. + /// + /// The key to append to. + /// The path of the string(s) to append to. + /// The value to append. + /// The new length of the string(s) appended to, those lengths will be null if the path did not resolve ot a string. + /// + Task StrAppendAsync(RedisKey key, string value, string? path = null); + + /// + /// Check's the length of the string(s) at the provided path. + /// + /// The key of the json object. + /// The path of the string(s) within the json object. + /// The length of the string(s) appended to, those lengths will be null if the path did not resolve ot a string. + /// + Task StrLenAsync(RedisKey key, string? path = null); + + /// + /// Toggles the boolean value(s) at the provided path. + /// + /// The key of the json object. + /// The path of the value(s) to toggle. + /// the new value(s). Which will be null if the path did not resolve to a boolean. + /// + Task ToggleAsync(RedisKey key, string? path = null); + + /// + /// Gets the type(s) of the item(s) at the provided json path. + /// + /// The key of the JSON object. + /// The path to resolve. + /// An array of types. + /// + Task TypeAsync(RedisKey key, string? path = null); + + /// + /// Report a value's memory usage in bytes. path defaults to root if not provided. + /// + /// The object's key + /// The path within the object. + /// the value's size in bytes. + Task DebugMemoryAsync(string key, string? path = null); +} \ No newline at end of file diff --git a/src/NRedisStack/Json/JsonCommands.cs b/src/NRedisStack/Json/JsonCommands.cs index 7782408d..78fa6627 100644 --- a/src/NRedisStack/Json/JsonCommands.cs +++ b/src/NRedisStack/Json/JsonCommands.cs @@ -1,48 +1,595 @@ using NRedisStack.Literals; using StackExchange.Redis; using System.Text.Json; -using System.Text.Json.Serialization; +using System.Text.Json.Nodes; +using static NRedisStack.Auxiliary; namespace NRedisStack; -public class JsonCommands +public class JsonCommands : IJsonCommands { IDatabase _db; public JsonCommands(IDatabase db) { _db = db; } - private readonly JsonSerializerOptions Options = new() + + /// + public RedisResult[] Resp(RedisKey key, string? path = null) { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - }; - public RedisResult Set(RedisKey key, RedisValue path, object obj, When when = When.Always) + RedisResult result; + if (string.IsNullOrEmpty(path)) + { + result = _db.Execute(JSON.RESP, key); + } + else + { + result = _db.Execute(JSON.RESP, key, path); + } + + if (result.IsNull) + { + return Array.Empty(); + } + + return (RedisResult[])result!; + } + + /// + public bool Set(RedisKey key, RedisValue path, object obj, When when = When.Always) { string json = JsonSerializer.Serialize(obj); return Set(key, path, json, when); } - public RedisResult Set(RedisKey key, RedisValue path, RedisValue json, When when = When.Always) + /// + public bool Set(RedisKey key, RedisValue path, RedisValue json, When when = When.Always) + { + var result = when switch + { + When.Exists => _db.Execute(JSON.SET, key, path, json, "XX"), + When.NotExists => _db.Execute(JSON.SET, key, path, json, "NX"), + _ => _db.Execute(JSON.SET, key, path, json) + }; + + if (result.IsNull || result.ToString() != "OK") + { + return false; + } + + return true; + } + + /// + public long?[] StrAppend(RedisKey key, string value, string? path = null) + { + RedisResult result; + if (path == null) + { + result = _db.Execute(JSON.STRAPPEND, key, JsonSerializer.Serialize(value)); + } + else + { + result = _db.Execute(JSON.STRAPPEND, key, path, JsonSerializer.Serialize(value)); + } + + return result.ToNullableLongArray(); + } + + /// + public long?[] StrLen(RedisKey key, string? path = null) + { + RedisResult result; + if (path != null) + { + result = _db.Execute(JSON.STRLEN, key, path); + } + else + { + result = _db.Execute(JSON.STRLEN, key); + } + + return result.ToNullableLongArray(); + } + + /// + public bool?[] Toggle(RedisKey key, string? path = null) + { + RedisResult result; + if (path != null) + { + result = _db.Execute(JSON.TOGGLE, key, path); + } + else + { + result = _db.Execute(JSON.TOGGLE, key, "$"); + } + + if (result.IsNull) + { + return Array.Empty(); + } + + if (result.Type == ResultType.Integer) + { + return new bool?[] { (long)result == 1 }; + } + + return ((RedisResult[])result!).Select(x => (bool?)((long)x == 1)).ToArray(); + } + + /// + public JsonType[] Type(RedisKey key, string? path = null) + { + RedisResult result; + if (path == null) + { + result = _db.Execute(JSON.TYPE, key); + } + else + { + result = _db.Execute(JSON.TYPE, key, path); + } + + if (result.Type == ResultType.MultiBulk) + { + return ((RedisResult[])result!).Select(x => Enum.Parse(x.ToString()!.ToUpper())).ToArray(); + } + + if (result.Type == ResultType.BulkString) + { + return new[] { Enum.Parse(result.ToString()!.ToUpper()) }; + } + + return Array.Empty(); + + } + + public long DebugMemory(string key, string? path = null) + { + if (path != null) + { + return (long)_db.Execute(JSON.DEBUG, JSON.MEMORY, key, path); + } + return (long)_db.Execute(JSON.DEBUG, JSON.MEMORY, key); + } + + public async Task ArrAppendAsync(RedisKey key, string? path = null, params object[] values) + { + if (values.Length < 1) + throw new ArgumentOutOfRangeException(nameof(values)); + + var args = new List { key }; + if (path != null) + { + args.Add(path); + } + + args.AddRange(values.Select(x => JsonSerializer.Serialize(x))); + + var result = await _db.ExecuteAsync(JSON.ARRAPPEND, args.ToArray()); + return result.ToNullableLongArray(); + } + + public async Task ArrIndexAsync(RedisKey key, string path, object value, long? start = null, long? stop = null) + { + if (start == null && stop != null) + throw new ArgumentException("stop cannot be defined without start"); + + var args = AssembleNonNullArguments(key, path, JsonSerializer.Serialize(value), start, stop); + var result = await _db.ExecuteAsync(JSON.ARRINDEX, args); + return result.ToNullableLongArray(); + } + + public async Task ArrInsertAsync(RedisKey key, string path, long index, params object[] values) + { + if (values.Length < 1) + throw new ArgumentOutOfRangeException(nameof(values)); + var args = new List { key, path, index }; + foreach (var val in values) + { + args.Add(JsonSerializer.Serialize(val)); + } + + var result = await _db.ExecuteAsync(JSON.ARRINSERT, args); + return result.ToNullableLongArray(); + } + + public async Task ArrLenAsync(RedisKey key, string? path = null) + { + var args = AssembleNonNullArguments(key, path); + var result = await _db.ExecuteAsync(JSON.ARRLEN, args); + return result.ToNullableLongArray(); + } + + public async Task ArrPopAsync(RedisKey key, string? path = null, long? index = null) + { + if (path == null && index != null) + throw new ArgumentException("index cannot be defined without path"); + + var args = AssembleNonNullArguments(key, path, index); + var res = await _db.ExecuteAsync(JSON.ARRPOP, args)!; + + if (res.Type == ResultType.MultiBulk) + { + return (RedisResult[])res!; + } + + if (res.Type == ResultType.BulkString) + { + return new[] { res }; + } + + return Array.Empty(); + } + + public async Task ArrTrimAsync(RedisKey key, string path, long start, long stop) => + (await _db.ExecuteAsync(JSON.ARRTRIM, key, path, start, stop)).ToNullableLongArray(); + + public async Task ClearAsync(RedisKey key, string? path = null) + { + var args = AssembleNonNullArguments(key, path); + return (long)await _db.ExecuteAsync(JSON.CLEAR, args); + } + + public async Task DelAsync(RedisKey key, string? path = null) + { + var args = AssembleNonNullArguments(key, path); + return (long)await _db.ExecuteAsync(JSON.DEL, args); + } + + public Task ForgetAsync(RedisKey key, string? path = null) => DelAsync(key, path); + + public Task GetAsync(RedisKey key, RedisValue? indent = null, RedisValue? newLine = null, RedisValue? space = null, + RedisValue? path = null) + { + List args = new List() { key }; + + if (indent != null) + { + args.Add(JsonArgs.INDENT); + args.Add(indent); + } + + if (newLine != null) + { + args.Add(JsonArgs.NEWLINE); + args.Add(newLine); + } + + if (space != null) + { + args.Add(JsonArgs.SPACE); + args.Add(space); + } + + if (path != null) + { + args.Add(path); + } + + return _db.ExecuteAsync(JSON.GET, args); + } + + public Task GetAsync(RedisKey key, string[] paths, RedisValue? indent = null, RedisValue? newLine = null, + RedisValue? space = null) + { + List args = new List() { key }; + + foreach (var path in paths) + { + args.Add(path); + } + + if (indent != null) + { + args.Add(JsonArgs.INDENT); + args.Add(indent); + } + + if (newLine != null) + { + args.Add(JsonArgs.NEWLINE); + args.Add(newLine); + } + + if (space != null) + { + args.Add(JsonArgs.SPACE); + args.Add(space); + } + + return _db.ExecuteAsync(JSON.GET, args); + } + + public async Task GetAsync(RedisKey key, string path = "$") + { + var res = await _db.ExecuteAsync(JSON.GET, key, path); + if (res.Type == ResultType.BulkString) + { + var arr = JsonSerializer.Deserialize(res.ToString()!); + if (arr?.Count > 0) + { + return JsonSerializer.Deserialize(JsonSerializer.Serialize(arr[0])); + } + } + + return default; + } + + public Task> GetEnumerableAsync(RedisKey key, string path = "$") + { + throw new NotImplementedException(); + } + + public async Task MGetAsync(RedisKey[] keys, string path) + { + var args = new List(); + foreach (var key in keys) + { + args.Add(key); + } + + args.Add(path); + var res = await _db.ExecuteAsync(JSON.MGET, args); + if (res.IsNull) + { + return Array.Empty(); + } + return (RedisResult[])res!; + } + + public async Task NumIncrbyAsync(RedisKey key, string path, double value) + { + var res = await _db.ExecuteAsync(JSON.NUMINCRBY, key, path, value); + return JsonSerializer.Deserialize(res.ToString()); + } + + public async Task>> ObjKeysAsync(RedisKey key, string? path = null) + { + var args = AssembleNonNullArguments(key, path); + return (await _db.ExecuteAsync(JSON.OBJKEYS, args)).ToHashSets(); + } + + public async Task ObjLenAsync(RedisKey key, string? path = null) + { + var args = AssembleNonNullArguments(key, path); + return (await _db.ExecuteAsync(JSON.OBJLEN, args)).ToNullableLongArray(); + } + + public async Task RespAsync(RedisKey key, string? path = null) + { + RedisResult result; + if (string.IsNullOrEmpty(path)) + { + result = await _db.ExecuteAsync(JSON.RESP, key); + } + else + { + result = await _db.ExecuteAsync(JSON.RESP, key, path); + } + + if (result.IsNull) + { + return Array.Empty(); + } + + return (RedisResult[])result!; + } + + public Task SetAsync(RedisKey key, RedisValue path, object obj, When when = When.Always) + { + string json = JsonSerializer.Serialize(obj); + return SetAsync(key, path, json, when); + } + + public async Task SetAsync(RedisKey key, RedisValue path, RedisValue json, When when = When.Always) + { + var t = when switch + { + When.Exists => _db.ExecuteAsync(JSON.SET, key, path, json, "XX"), + When.NotExists => _db.ExecuteAsync(JSON.SET, key, path, json, "NX"), + _ => _db.ExecuteAsync(JSON.SET, key, path, json) + }; + + var result = await t; + + if (result.IsNull || result.ToString() != "OK") + { + return false; + } + + return true; + } + + public async Task StrAppendAsync(RedisKey key, string value, string? path = null) + { + RedisResult result; + if (path == null) + { + result = await _db.ExecuteAsync(JSON.STRAPPEND, key, JsonSerializer.Serialize(value)); + } + else + { + result = await _db.ExecuteAsync(JSON.STRAPPEND, key, path, JsonSerializer.Serialize(value)); + } + + return result.ToNullableLongArray(); + } + + public async Task StrLenAsync(RedisKey key, string? path = null) + { + RedisResult result; + if (path != null) + { + result = await _db.ExecuteAsync(JSON.STRLEN, key, path); + } + else + { + result = await _db.ExecuteAsync(JSON.STRLEN, key); + } + + return result.ToNullableLongArray(); + } + + public async Task ToggleAsync(RedisKey key, string? path = null) + { + RedisResult result; + if (path != null) + { + result = await _db.ExecuteAsync(JSON.TOGGLE, key, path); + } + else + { + result = await _db.ExecuteAsync(JSON.TOGGLE, key, "$"); + } + + if (result.IsNull) + { + return Array.Empty(); + } + + if (result.Type == ResultType.Integer) + { + return new bool?[] { (long)result == 1 }; + } + + return ((RedisResult[])result!).Select(x => (bool?)((long)x == 1)).ToArray(); + } + + public async Task TypeAsync(RedisKey key, string? path = null) + { + RedisResult result; + if (path == null) + { + result = await _db.ExecuteAsync(JSON.TYPE, key); + } + else + { + result = await _db.ExecuteAsync(JSON.TYPE, key, path); + } + + if (result.Type == ResultType.MultiBulk) + { + return ((RedisResult[])result!).Select(x => Enum.Parse(x.ToString()!.ToUpper())).ToArray(); + } + + if (result.Type == ResultType.BulkString) + { + return new[] { Enum.Parse(result.ToString()!.ToUpper()) }; + } + + return Array.Empty(); + } + + public async Task DebugMemoryAsync(string key, string? path = null) + { + if (path != null) + { + return (long)await _db.ExecuteAsync(JSON.DEBUG, JSON.MEMORY, key, path); + } + return (long)await _db.ExecuteAsync(JSON.DEBUG, JSON.MEMORY, key); + } + + /// + public long?[] ArrAppend(RedisKey key, string? path = null, params object[] values) { - switch (when) + if (values.Length < 1) + throw new ArgumentOutOfRangeException(nameof(values)); + + var args = new List { key }; + if (path != null) { - case When.Exists: - return _db.Execute(JSON.SET, key, path, json, "XX"); - case When.NotExists: - return _db.Execute(JSON.SET, key, path, json, "NX"); - default: - return _db.Execute(JSON.SET, key, path, json); + args.Add(path); } + + args.AddRange(values.Select(x => JsonSerializer.Serialize(x))); + + var result = _db.Execute(JSON.ARRAPPEND, args.ToArray()); + return result.ToNullableLongArray(); } - public RedisResult Get(RedisKey key, - RedisValue? indent = null, - RedisValue? newLine = null, - RedisValue? space = null, - RedisValue? path = null) + /// + public long?[] ArrIndex(RedisKey key, string path, object value, long? start = null, long? stop = null) { + if (start == null && stop != null) + throw new ArgumentException("stop cannot be defined without start"); + + var args = AssembleNonNullArguments(key, path, JsonSerializer.Serialize(value), start, stop); + var result = _db.Execute(JSON.ARRINDEX, args); + return result.ToNullableLongArray(); + } - List args = new List(){key}; + /// + public long?[] ArrInsert(RedisKey key, string path, long index, params object[] values) + { + if (values.Length < 1) + throw new ArgumentOutOfRangeException(nameof(values)); + var args = new List { key, path, index }; + foreach (var val in values) + { + args.Add(JsonSerializer.Serialize(val)); + } + + var result = _db.Execute(JSON.ARRINSERT, args); + return result.ToNullableLongArray(); + } + + /// + public long?[] ArrLen(RedisKey key, string? path = null) + { + var args = AssembleNonNullArguments(key, path); + var result = _db.Execute(JSON.ARRLEN, args); + return result.ToNullableLongArray(); + } + + /// + public RedisResult[] ArrPop(RedisKey key, string? path = null, long? index = null) + { + if (path == null && index != null) + throw new ArgumentException("index cannot be defined without path"); + + var args = AssembleNonNullArguments(key, path, index); + var res = _db.Execute(JSON.ARRPOP, args)!; + + if (res.Type == ResultType.MultiBulk) + { + return (RedisResult[])res!; + } + + if (res.Type == ResultType.BulkString) + { + return new[] { res }; + } + + return Array.Empty(); + } + + /// + public long?[] ArrTrim(RedisKey key, string path, long start, long stop) => + _db.Execute(JSON.ARRTRIM, key, path, start, stop).ToNullableLongArray(); + + /// + public long Clear(RedisKey key, string? path = null) + { + var args = AssembleNonNullArguments(key, path); + return (long)_db.Execute(JSON.CLEAR, args); + } + + /// + public long Del(RedisKey key, string? path = null) + { + var args = AssembleNonNullArguments(key, path); + return (long)_db.Execute(JSON.DEL, args); + } + + /// + public long Forget(RedisKey key, string? path = null) => Del(key, path); + + /// + public RedisResult Get(RedisKey key, RedisValue? indent = null, RedisValue? newLine = null, RedisValue? space = null, RedisValue? path = null) + { + List args = new List() { key }; if (indent != null) { @@ -69,4 +616,92 @@ public RedisResult Get(RedisKey key, return _db.Execute(JSON.GET, args); } + + /// + public RedisResult Get(RedisKey key, string[] paths, RedisValue? indent = null, RedisValue? newLine = null, RedisValue? space = null) + { + List args = new List() { key }; + + foreach (var path in paths) + { + args.Add(path); + } + + if (indent != null) + { + args.Add(JsonArgs.INDENT); + args.Add(indent); + } + + if (newLine != null) + { + args.Add(JsonArgs.NEWLINE); + args.Add(newLine); + } + + if (space != null) + { + args.Add(JsonArgs.SPACE); + args.Add(space); + } + + return _db.Execute(JSON.GET, args); + } + + /// + public T? Get(RedisKey key, string path = "$") + { + var res = _db.Execute(JSON.GET, key, path); + if (res.Type == ResultType.BulkString) + { + var arr = JsonSerializer.Deserialize(res.ToString()!); + if (arr?.Count > 0) + { + return JsonSerializer.Deserialize(JsonSerializer.Serialize(arr[0])); + } + } + + return default; + } + + /// + public IEnumerable GetEnumerable(RedisKey key, string path = "$") + { + var res = _db.Execute(JSON.GET, key, path); + return JsonSerializer.Deserialize>(res.ToString()); + } + + /// + public RedisResult[] MGet(RedisKey[] keys, string path) + { + var args = new List(); + foreach (var key in keys) + { + args.Add(key); + } + + args.Add(path); + return (RedisResult[])_db.Execute(JSON.MGET, args)!; + } + + /// + public double?[] NumIncrby(RedisKey key, string path, double value) + { + var res = _db.Execute(JSON.NUMINCRBY, key, path, value); + return JsonSerializer.Deserialize(res.ToString()); + } + + /// + public IEnumerable> ObjKeys(RedisKey key, string? path = null) + { + var args = AssembleNonNullArguments(key, path); + return _db.Execute(JSON.OBJKEYS, args).ToHashSets(); + } + + /// + public long?[] ObjLen(RedisKey key, string? path = null) + { + var args = AssembleNonNullArguments(key, path); + return _db.Execute(JSON.OBJLEN, args).ToNullableLongArray(); + } } \ No newline at end of file diff --git a/src/NRedisStack/Json/JsonType.cs b/src/NRedisStack/Json/JsonType.cs new file mode 100644 index 00000000..6e5ac114 --- /dev/null +++ b/src/NRedisStack/Json/JsonType.cs @@ -0,0 +1,13 @@ +namespace NRedisStack; + +public enum JsonType +{ + UNKNOWN = 0, + NULL = 1, + BOOLEAN = 2, + INTEGER = 3, + NUMBER = 4, + STRING = 5, + ARRAY = 6, + OBJECT = 7 +} \ No newline at end of file diff --git a/src/NRedisStack/Json/Literals/Commands.cs b/src/NRedisStack/Json/Literals/Commands.cs index 159a4c81..c455e6ea 100644 --- a/src/NRedisStack/Json/Literals/Commands.cs +++ b/src/NRedisStack/Json/Literals/Commands.cs @@ -11,10 +11,10 @@ internal class JSON public const string CLEAR = "JSON.CLEAR"; public const string DEBUG = "JSON.DEBUG"; public const string DEBUG_HELP = "JSON.DEBUG HELP"; - public const string DEBUG_MEMORY = "JSON.DEBUG MEMORY"; public const string DEL = "JSON.DEL"; public const string FORGET = "JSON.FORGET"; public const string GET = "JSON.GET"; + public const string MEMORY = "MEMORY"; public const string MGET = "JSON.MGET"; public const string NUMINCRBY = "JSON.NUMINCRBY"; public const string NUMMULTBY = "JSON.NUMMULTBY"; diff --git a/src/NRedisStack/ResponseParser.cs b/src/NRedisStack/ResponseParser.cs index 6cf0c475..5e9f87d4 100644 --- a/src/NRedisStack/ResponseParser.cs +++ b/src/NRedisStack/ResponseParser.cs @@ -531,5 +531,53 @@ public static IReadOnlyList ToStringArray(this RedisResult result) Array.ForEach(redisResults, str => list.Add((string)str)); return list; } + + public static long?[] ToNullableLongArray(this RedisResult result) + { + if (result.IsNull) + { + return Array.Empty(); + } + + if (result.Type == ResultType.Integer) + { + return new[] { (long?)result }; + } + + return ((RedisResult[])result!).Select(x=>(long?)x).ToArray(); + } + + public static IEnumerable> ToHashSets(this RedisResult result) + { + if (result.IsNull) + { + return Array.Empty>(); + } + + var res = (RedisResult[])result!; + var sets = new List>(); + if (res.All(x => x.Type != ResultType.MultiBulk)) + { + var keys = res.Select(x => x.ToString()!); + sets.Add(keys.ToHashSet()); + return sets; + } + + foreach (var arr in res) + { + var set = new HashSet(); + if (arr.Type == ResultType.MultiBulk) + { + var resultArr = (RedisResult[])arr!; + foreach (var item in resultArr) + { + set.Add(item.ToString()!); + } + } + sets.Add(set); + } + + return sets; + } } } \ No newline at end of file diff --git a/tests/NRedisStack.Tests/Json/JsonTests.cs b/tests/NRedisStack.Tests/Json/JsonTests.cs index 33b0eaf7..b35fb9ab 100644 --- a/tests/NRedisStack.Tests/Json/JsonTests.cs +++ b/tests/NRedisStack.Tests/Json/JsonTests.cs @@ -1,3 +1,6 @@ +using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Nodes; using Xunit; using StackExchange.Redis; using Moq; @@ -9,67 +12,23 @@ namespace NRedisStack.Tests; public class JsonTests : AbstractNRedisStackTest, IDisposable { Mock _mock = new Mock(); - private readonly string key = "JSON_TESTS"; + private readonly string _testName = "JSON_TESTS"; public JsonTests(RedisFixture redisFixture) : base(redisFixture) { } public void Dispose() { - redisFixture.Redis.GetDatabase().KeyDelete(key); + redisFixture.Redis.GetDatabase().KeyDelete(_testName); } - // [Fact] - // public void TestJsonSet() - // { - // var obj = new Person { Name = "Shachar", Age = 23 }; - // _mock.Object.JSON().Set("Person:Shachar", "$", obj, When.Exists); - // _mock.Verify(x => x.Execute("JSON.SET", "Person:Shachar", "$", "{\"Name\":\"Shachar\",\"Age\":23}", "XX")); - // } - [Fact] public void TestJsonSetNotExist() { var obj = new Person { Name = "Shachar", Age = 23 }; + _mock.Setup(x => x.Execute(It.IsAny(), It.IsAny())).Returns((RedisResult.Create(new RedisValue("OK")))); _mock.Object.JSON().Set("Person:Shachar", "$", obj, When.NotExists); _mock.Verify(x => x.Execute("JSON.SET", "Person:Shachar", "$", "{\"Name\":\"Shachar\",\"Age\":23}", "NX")); } - //TODO: understand why this 2 tests are not pass what we do - //"dotnet test" but they pass when we do "dotnet test --filter ..." - // [Fact] - // public void TestSimpleJsonGet() - // { - // var obj = new Person { Name = "Shachar", Age = 23 }; - // IDatabase db = redisFixture.Redis.GetDatabase(); - // db.Execute("FLUSHALL"); - // var cf = db.JSON(); - - // json.Set(key, "$", obj); - // string expected = "{\"Name\":\"Shachar\",\"Age\":23}"; - // var result = json.Get(key).ToString(); - // if(result == null) - // throw new ArgumentNullException(nameof(result)); - - // Assert.Equal(result, expected); - // } - - // [Fact] - // public void TestJsonGet() - // { - // var obj = new Person { Name = "Shachar", Age = 23 }; - // IDatabase db = redisFixture.Redis.GetDatabase(); - // db.Execute("FLUSHALL"); - // var cf = db.JSON(); - - // json.Set(key, "$", obj); - - // var expected = "[222111\"Shachar\"222]"; - // var result = json.Get(key, "111", "222", "333", "$.Name"); - // // if(result == null) - // // throw new ArgumentNullException(nameof(result)); - // Assert.Equal(result.ToString(), expected); - // } - - [Fact] public void TestModulePrefixs() { @@ -102,6 +61,769 @@ public void TestModulePrefixs1() // ... conn.Dispose(); } + } + + [Fact] + public void TestResp() + { + //arrange + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(1); + + var key = keys[0]; + commands.Set(key, "$", new { name = "Steve", age = 33 }); + + //act + var respResult = commands.Resp(key); + + //assert + var i = 0; + Assert.Equal("{", respResult[i++]!.ToString()); + Assert.Equal("name", respResult[i++]!.ToString()); + Assert.Equal("Steve", respResult[i++]!.ToString()); + Assert.Equal("age", respResult[i++]!.ToString()); + Assert.Equal(33, (long)respResult[i]!); + conn.GetDatabase().KeyDelete(key); + } + + [Fact] + public async Task TestRespAsync() + { + //arrange + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(1); + + var key = keys[0]; + await commands.SetAsync(key, "$", new { name = "Steve", age = 33 }); + + //act + var respResult = await commands.RespAsync(key); + + //assert + var i = 0; + Assert.Equal("{", respResult[i++]!.ToString()); + Assert.Equal("name", respResult[i++]!.ToString()); + Assert.Equal("Steve", respResult[i++]!.ToString()); + Assert.Equal("age", respResult[i++]!.ToString()); + Assert.Equal(33, (long)respResult[i]!); + conn.GetDatabase().KeyDelete(key); + } + + [Fact] + public void TestStringAppend() + { + //arrange + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(2); + + var key = keys[0]; + commands.Set(key, "$", new { name = "Steve", sibling = new {name = "christopher"}, age = 33}); + var simpleStringKey = keys[1]; + commands.Set(simpleStringKey, "$", "\"foo\""); + + //act + var nullResult = commands.StrAppend(key, " Lorello", "$.age"); + var keyResult = commands.StrAppend(key, " Lorello", "$..name"); + var simpleKeyResult = commands.StrAppend(simpleStringKey, "bar"); + + //assert + var i = 0; + Assert.Equal(2, keyResult.Length); + Assert.Equal(13, keyResult[i++]); + Assert.Equal(19, keyResult[i]); + Assert.Null(nullResult[0]); + Assert.Equal(6, simpleKeyResult[0]); + } + + [Fact] + public async Task TestStringAppendAsync() + { + //arrange + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(2); + + var key = keys[0]; + await commands.SetAsync(key, "$", new { name = "Steve", sibling = new {name = "christopher"}, age = 33}); + var simpleStringKey = keys[1]; + await commands.SetAsync(simpleStringKey, "$", "\"foo\""); + + //act + var nullResult = await commands.StrAppendAsync(key, " Lorello", "$.age"); + var keyResult = await commands.StrAppendAsync(key, " Lorello", "$..name"); + var simpleKeyResult = await commands.StrAppendAsync(simpleStringKey, "bar"); + + //assert + var i = 0; + Assert.Equal(2, keyResult.Length); + Assert.Equal(13, keyResult[i++]); + Assert.Equal(19, keyResult[i]); + Assert.Null(nullResult[0]); + Assert.Equal(6, simpleKeyResult[0]); + } + + [Fact] + public void StringLength() + { + //arrange + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleStringKey = keys[1]; + + commands.Set(key, "$", new { name = "Steve", sibling = new {name = "christopher"}, age = 33}); + commands.Set(simpleStringKey, "$", "\"foo\""); + + var normalResult = commands.StrLen(key, "$..name"); + var nullResult = commands.StrLen(key, "$.age"); + var simpleResult = commands.StrLen(simpleStringKey); + + var i = 0; + Assert.Equal(5, normalResult[i++]); + Assert.Equal(11, normalResult[i]); + Assert.Null(nullResult[0]); + Assert.Equal(3,simpleResult[0]); + } + + [Fact] + public async Task StringLengthAsync() + { + //arrange + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleStringKey = keys[1]; + + await commands.SetAsync(key, "$", new { name = "Steve", sibling = new {name = "christopher"}, age = 33}); + await commands.SetAsync(simpleStringKey, "$", "\"foo\""); + + var normalResult = await commands.StrLenAsync(key, "$..name"); + var nullResult = await commands.StrLenAsync(key, "$.age"); + var simpleResult = await commands.StrLenAsync(simpleStringKey); + + var i = 0; + Assert.Equal(5, normalResult[i++]); + Assert.Equal(11, normalResult[i]); + Assert.Null(nullResult[0]); + Assert.Equal(3,simpleResult[0]); + } + + [Fact] + public void Toggle() + { + //arrange + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + + commands.Set(key, "$", new { @bool = true, other = new {@bool = false}, age = 33}); + commands.Set(simpleKey, "$", true); + + var result = commands.Toggle(key, "$..bool"); + var simpleResult = commands.Toggle(simpleKey); + + Assert.False(result[0]); + Assert.True(result[1]); + Assert.False(simpleResult[0]); + } + + [Fact] + public async Task ToggleAsync() + { + //arrange + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + + await commands.SetAsync(key, "$", new { @bool = true, other = new {@bool = false}, age = 33}); + await commands.SetAsync(simpleKey, "$", true); + + var result = await commands.ToggleAsync(key, "$..bool"); + var simpleResult = await commands.ToggleAsync(simpleKey); + + Assert.False(result[0]); + Assert.True(result[1]); + Assert.False(simpleResult[0]); + } + + [Fact] + public void Type() + { + //arrange + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + commands.Set(key, "$", new { name = "Steve", sibling = new {name = "christopher"}, age = 33, aDouble = 3.5}); + commands.Set(simpleKey, "$", "true"); + + var result = commands.Type(key, "$..name"); + Assert.Equal(JsonType.STRING, result[0]); + Assert.Equal(JsonType.STRING, result[1]); + result = commands.Type(key, "$..age"); + Assert.Equal(JsonType.INTEGER, result[0]); + result = commands.Type(key, "$..aDouble"); + Assert.Equal(JsonType.NUMBER, result[0]); + result = commands.Type(simpleKey); + Assert.Equal(JsonType.BOOLEAN, result[0]); + } + + [Fact] + public async Task TypeAsync() + { + //arrange + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + await commands.SetAsync(key, "$", new { name = "Steve", sibling = new {name = "christopher"}, age = 33, aDouble = 3.5}); + await commands.SetAsync(simpleKey, "$", "true"); + + var result = await commands.TypeAsync(key, "$..name"); + Assert.Equal(JsonType.STRING, result[0]); + Assert.Equal(JsonType.STRING, result[1]); + result = await commands.TypeAsync(key, "$..age"); + Assert.Equal(JsonType.INTEGER, result[0]); + result = await commands.TypeAsync(key, "$..aDouble"); + Assert.Equal(JsonType.NUMBER, result[0]); + result = await commands.TypeAsync(simpleKey); + Assert.Equal(JsonType.BOOLEAN, result[0]); + } + + [Fact] + public void ArrayAppend() + { + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(2); + var key = keys[0]; + var complexKey = keys[1]; + + commands.Set(key, "$", new { name = "Elizabeth", nickNames = new[] { "Beth" } }); + commands.Set(complexKey, "$", new { name = "foo", people = new[] { new { name = "steve" } } }); + var result = commands.ArrAppend(key, "$.nickNames", "Elle", "Liz","Betty"); + Assert.Equal(4, result[0]); + result = commands.ArrAppend(complexKey, "$.people", new { name = "bob" }); + Assert.Equal(2, result[0]); + } + + [Fact] + public async Task ArrayAppendAsync() + { + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(2); + var key = keys[0]; + var complexKey = keys[1]; + + await commands.SetAsync(key, "$", new { name = "Elizabeth", nickNames = new[] { "Beth" } }); + await commands.SetAsync(complexKey, "$", new { name = "foo", people = new[] { new { name = "steve" } } }); + var result = await commands.ArrAppendAsync(key, "$.nickNames", "Elle", "Liz","Betty"); + Assert.Equal(4, result[0]); + result = await commands.ArrAppendAsync(complexKey, "$.people", new { name = "bob" }); + Assert.Equal(2, result[0]); + } + + [Fact] + public void ArrayIndex() + { + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(1); + var key = keys[0]; + commands.Set(key, "$", new { name = "Elizabeth", nicknames = new[] { "Beth", "Betty", "Liz" }, sibling = new {name="Johnathan", nicknames = new [] {"Jon", "Johnny"}} }); + var res = commands.ArrIndex(key, "$..nicknames", "Betty", 0,5); + Assert.Equal(1,res[0]); + Assert.Equal(-1,res[1]); + } + + [Fact] + public async Task ArrayIndexAsync() + { + var conn = redisFixture.Redis; + var db = conn.GetDatabase(); + IJsonCommands commands = new JsonCommands(db); + var keys = CreateKeyNames(1); + var key = keys[0]; + await commands.SetAsync(key, "$", new { name = "Elizabeth", nicknames = new[] { "Beth", "Betty", "Liz" }, sibling = new {name="Johnathan", nicknames = new [] {"Jon", "Johnny"}} }); + var res = await commands.ArrIndexAsync(key, "$..nicknames", "Betty", 0,5); + Assert.Equal(1,res[0]); + Assert.Equal(-1,res[1]); + } + + [Fact] + public void ArrayInsert() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + + commands.Set(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + commands.Set(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = commands.ArrInsert(key, $"$.nicknames", 1, "Lys"); + Assert.Equal(4,result[0]); + result = commands.ArrInsert(simpleKey, "$", 1, "Lys"); + Assert.Equal(4, result[0]); + } + + [Fact] + public async Task ArrayInsertAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + + await commands.SetAsync(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + await commands.SetAsync(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = await commands.ArrInsertAsync(key, $"$.nicknames", 1, "Lys"); + Assert.Equal(4,result[0]); + result = await commands.ArrInsertAsync(simpleKey, "$", 1, "Lys"); + Assert.Equal(4, result[0]); + } + + [Fact] + public void ArrayLength() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + commands.Set(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + commands.Set(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = commands.ArrLen(key, $"$.nicknames"); + Assert.Equal(3, result[0]); + result = commands.ArrLen(simpleKey); + Assert.Equal(3, result[0]); + } + + [Fact] + public async Task ArrayLengthAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + await commands.SetAsync(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + await commands.SetAsync(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = await commands.ArrLenAsync(key, $"$.nicknames"); + Assert.Equal(3, result[0]); + result = await commands.ArrLenAsync(simpleKey); + Assert.Equal(3, result[0]); + } + + [Fact] + public void ArrayPop() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + commands.Set(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + commands.Set(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = commands.ArrPop(key, "$.nicknames", 1); + Assert.Equal("\"Ali\"", result[0].ToString()); + result = commands.ArrPop(key, "$.nicknames"); + Assert.Equal("\"Ally\"", result[0].ToString()); + result = commands.ArrPop(simpleKey); + Assert.Equal("\"Ally\"", result[0].ToString()); + } + + [Fact] + public void ArrayTrim() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + commands.Set(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + commands.Set(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = commands.ArrTrim(key, "$.nicknames", 0, 0); + Assert.Equal(1,result[0]); + result = commands.ArrTrim(simpleKey, "$", 0, 1); + Assert.Equal(2,result[0]); + } + + [Fact] + public async Task ArrayTrimAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + await commands.SetAsync(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + await commands.SetAsync(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = await commands.ArrTrimAsync(key, "$.nicknames", 0, 0); + Assert.Equal(1,result[0]); + result = await commands.ArrTrimAsync(simpleKey, "$", 0, 1); + Assert.Equal(2,result[0]); + } + + [Fact] + public void Clear() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + commands.Set(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + commands.Set(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = commands.Clear(key, "$.nicknames"); + Assert.Equal(1,result); + result = commands.Clear(simpleKey); + Assert.Equal(1,result); + } + + [Fact] + public async Task ClearAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + await commands.SetAsync(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + await commands.SetAsync(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = await commands.ClearAsync(key, "$.nicknames"); + Assert.Equal(1,result); + result = await commands.ClearAsync(simpleKey); + Assert.Equal(1,result); + } + + [Fact] + public void Del() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + commands.Set(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + commands.Set(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = commands.Del(key, "$.nicknames"); + Assert.Equal(1,result); + result = commands.Del(simpleKey); + Assert.Equal(1,result); + } + + [Fact] + public async Task DelAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + await commands.SetAsync(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + await commands.SetAsync(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = await commands.DelAsync(key, "$.nicknames"); + Assert.Equal(1,result); + result = await commands.DelAsync(simpleKey); + Assert.Equal(1,result); + } + + [Fact] + public void Forget() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + commands.Set(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + commands.Set(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = commands.Forget(key, "$.nicknames"); + Assert.Equal(1,result); + result = commands.Forget(simpleKey); + Assert.Equal(1,result); + } + + [Fact] + public async Task ForgetAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var simpleKey = keys[1]; + await commands.SetAsync(key, "$", new { name = "Alice", nicknames = new[] { "Al", "Ali", "Ally" } }); + await commands.SetAsync(simpleKey, "$", new[] { "Al", "Ali", "Ally" }); + + var result = await commands.ForgetAsync(key, "$.nicknames"); + Assert.Equal(1,result); + result = await commands.ForgetAsync(simpleKey); + Assert.Equal(1,result); + } + + [Fact] + public void Get() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var complexKey = keys[1]; + commands.Set(key, "$", new Person(){Age = 35, Name = "Alice"}); + commands.Set(complexKey, "$", new {a=new Person(){Age = 35, Name = "Alice"}, b = new {a = new Person(){Age = 35, Name = "Alice"}}}); + var result = commands.Get(key); + Assert.Equal("Alice", result!.Name); + Assert.Equal(35, result.Age); + var people = commands.GetEnumerable(complexKey, "$..a").ToArray(); + Assert.Equal(2, people.Length); + Assert.Equal("Alice", people[0]!.Name); + Assert.Equal(35, people[0]!.Age); + Assert.Equal("Alice", people[1]!.Name); + Assert.Equal(35, people[1]!.Age); + } + + [Fact] + public async Task GetAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key = keys[0]; + var complexKey = keys[1]; + await commands.SetAsync(key, "$", new Person(){Age = 35, Name = "Alice"}); + await commands.SetAsync(complexKey, "$", new {a=new Person(){Age = 35, Name = "Alice"}, b = new {a = new Person(){Age = 35, Name = "Alice"}}}); + var result = await commands.GetAsync(key); + Assert.Equal("Alice", result!.Name); + Assert.Equal(35, result.Age); + var people = commands.GetEnumerable(complexKey, "$..a").ToArray(); + Assert.Equal(2, people.Length); + Assert.Equal("Alice", people[0]!.Name); + Assert.Equal(35, people[0]!.Age); + Assert.Equal("Alice", people[1]!.Name); + Assert.Equal(35, people[1]!.Age); + } + + [Fact] + public void MGet() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key1 = keys[0]; + var key2 = keys[1]; + commands.Set(key1, "$", new { a = "hello" }); + commands.Set(key2, "$", new { a = "world" }); + var result = commands.MGet(keys.Select(x => new RedisKey(x)).ToArray(), "$.a"); + + Assert.Equal("[\"hello\"]", result[0].ToString()); + Assert.Equal("[\"world\"]", result[1].ToString()); + } + + [Fact] + public async Task MGetAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(2); + var key1 = keys[0]; + var key2 = keys[1]; + await commands.SetAsync(key1, "$", new { a = "hello" }); + await commands.SetAsync(key2, "$", new { a = "world" }); + var result = await commands.MGetAsync(keys.Select(x => new RedisKey(x)).ToArray(), "$.a"); + + Assert.Equal("[\"hello\"]", result[0].ToString()); + Assert.Equal("[\"world\"]", result[1].ToString()); + } + + [Fact] + public void NumIncrby() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(1); + var key = keys[0]; + commands.Set(key, "$", new { age = 33, a = new { age = 34 }, b = new {age = "cat"} }); + var result = commands.NumIncrby(key, "$..age", 2); + Assert.Equal(35, result[0]); + Assert.Equal(36, result[1]); + Assert.Null(result[2]); + } + + [Fact] + public async Task NumIncrbyAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(1); + var key = keys[0]; + await commands.SetAsync(key, "$", new { age = 33, a = new { age = 34 }, b = new {age = "cat"} }); + var result = await commands.NumIncrbyAsync(key, "$..age", 2); + Assert.Equal(35, result[0]); + Assert.Equal(36, result[1]); + Assert.Null(result[2]); + } + + [Fact] + public void ObjectKeys() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(3); + var key = keys[0]; + commands.Set(key, "$", new { a = 5, b = 10, c = "hello", d = new { a = new { a = 6, b = "hello" }, b = 7 } }); + var result = commands.ObjKeys(key).ToArray(); + Assert.Contains("a", result[0]); + Assert.Contains("b", result[0]); + Assert.Contains("c", result[0]); + Assert.Contains("d", result[0]); + result = commands.ObjKeys(key, "$..a").ToArray(); + Assert.Empty(result[0]); + Assert.Contains("a", result[1]); + Assert.Contains("b", result[1]); + } + + [Fact] + public async Task ObjectKeysAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(3); + var key = keys[0]; + await commands.SetAsync(key, "$", new { a = 5, b = 10, c = "hello", d = new { a = new { a = 6, b = "hello" }, b = 7 } }); + var result = (await commands.ObjKeysAsync(key)).ToArray(); + Assert.Contains("a", result[0]); + Assert.Contains("b", result[0]); + Assert.Contains("c", result[0]); + Assert.Contains("d", result[0]); + result = (await commands.ObjKeysAsync(key, "$..a")).ToArray(); + Assert.Empty(result[0]); + Assert.Contains("a", result[1]); + Assert.Contains("b", result[1]); + } + + [Fact] + public void ObjectLength() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(3); + var key = keys[0]; + commands.Set(key, "$", new { a = 5, b = 10, c = "hello", d = new { a = new { a = 6, b = "hello" }, b = 7 } }); + var result = commands.ObjLen(key); + Assert.Equal(4, result[0]); + result = commands.ObjLen(key, $"$..a"); + Assert.Null(result[0]); + Assert.Equal(2, result[1]); + Assert.Null(result[2]); + + } + + [Fact] + public async Task ObjectLengthAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(3); + var key = keys[0]; + await commands.SetAsync(key, "$", new { a = 5, b = 10, c = "hello", d = new { a = new { a = 6, b = "hello" }, b = 7 } }); + var result = await commands.ObjLenAsync(key); + Assert.Equal(4, result[0]); + result = await commands.ObjLenAsync(key, $"$..a"); + Assert.Null(result[0]); + Assert.Equal(2, result[1]); + Assert.Null(result[2]); + + } + + [Fact] + public void TestMultiPathGet() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(1); + var key = keys[0]; + commands.Set(key, "$", new { a = "hello", b = new { a = "world" } }); + var res = commands.Get(key, new[] { "$..a", "$.b" }).ToString(); + var obj = JsonSerializer.Deserialize(res); + Assert.True(obj.ContainsKey("$..a")); + Assert.True(obj.ContainsKey("$.b")); + if (obj["$..a"] is JsonArray arr) + { + Assert.Equal("hello", arr[0]!.ToString()); + Assert.Equal("world", arr[1]!.ToString()); + } + else + { + Assert.True(false, "$..a was not a json array"); + } + + Assert.True(obj["$.b"]![0]!["a"]!.ToString() == "world"); + } + + [Fact] + public async Task TestMultiPathGetAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(1); + var key = keys[0]; + await commands.SetAsync(key, "$", new { a = "hello", b = new { a = "world" } }); + var res = (await commands.GetAsync(key, new[] { "$..a", "$.b" })).ToString(); + var obj = JsonSerializer.Deserialize(res); + Assert.True(obj.ContainsKey("$..a")); + Assert.True(obj.ContainsKey("$.b")); + if (obj["$..a"] is JsonArray arr) + { + Assert.Equal("hello", arr[0]!.ToString()); + Assert.Equal("world", arr[1]!.ToString()); + } + else + { + Assert.True(false, "$..a was not a json array"); + } + + Assert.True(obj["$.b"]![0]!["a"]!.ToString() == "world"); + } + + [Fact] + public void Memory() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(1); + var key = keys[0]; + + commands.Set(key, "$", new {a="hello", b=new {a="world"}}); + var res = commands.DebugMemory(key); + Assert.Equal(45, res); + res = commands.DebugMemory("non-existent key"); + Assert.Equal(0,res); + } + + [Fact] + public async Task MemoryAsync() + { + IJsonCommands commands = new JsonCommands(redisFixture.Redis.GetDatabase()); + var keys = CreateKeyNames(1); + var key = keys[0]; + await commands.SetAsync(key, "$", new {a="hello", b=new {a="world"}}); + var res = await commands.DebugMemoryAsync(key); + Assert.Equal(45, res); + res = await commands.DebugMemoryAsync("non-existent key"); + Assert.Equal(0,res); } } \ No newline at end of file From 2648aad3046567eadda993db61c3cec32d447701 Mon Sep 17 00:00:00 2001 From: shacharPash <93581407+shacharPash@users.noreply.github.com> Date: Sun, 2 Oct 2022 14:32:56 +0300 Subject: [PATCH 12/28] Implement new TDIGEST commands and TDIGEST.CREATE compression (#33) * Update Tdigest Commands * Fix Rank Commands Tests --- src/NRedisStack/ResponseParser.cs | 19 +- .../Tdigest/DataTypes/TdigestInformation.cs | 7 +- src/NRedisStack/Tdigest/Literals/Commands.cs | 4 + src/NRedisStack/Tdigest/TdigestCommands.cs | 305 +++++++++++------- .../NRedisStack.Tests/Tdigest/TdigestTests.cs | 243 ++++++++++++-- 5 files changed, 426 insertions(+), 152 deletions(-) diff --git a/src/NRedisStack/ResponseParser.cs b/src/NRedisStack/ResponseParser.cs index 5e9f87d4..5c9be0e6 100644 --- a/src/NRedisStack/ResponseParser.cs +++ b/src/NRedisStack/ResponseParser.cs @@ -368,11 +368,11 @@ public static IReadOnlyList ToRuleArray(this RedisResult result) public static TdigestInformation ToTdigestInfo(this RedisResult result) //TODO: Think about a different implementation, because if the output of CMS.INFO changes or even just the names of the labels then the parsing will not work { - long compression, capacity, mergedNodes, unmergedNodes, totalCompressions; - double mergedWeight, unmergedWeight; + long compression, capacity, mergedNodes, unmergedNodes, totalCompressions, memoryUsage; + double mergedWeight, unmergedWeight, sumWeight; - compression = capacity = mergedNodes = unmergedNodes = totalCompressions = -1; - mergedWeight = unmergedWeight = -1.0; + compression = capacity = mergedNodes = unmergedNodes = totalCompressions = memoryUsage = -1; + mergedWeight = unmergedWeight = sumWeight = -1.0; RedisResult[] redisResults = result.ToArray(); @@ -395,20 +395,25 @@ public static IReadOnlyList ToRuleArray(this RedisResult result) unmergedNodes = (long)redisResults[i]; break; case "Merged weight": - mergedWeight = (double)redisResults[i]; break; case "Unmerged weight": unmergedWeight = (double)redisResults[i]; break; + case "Sum weights": + sumWeight = (double)redisResults[i]; + break; case "Total compressions": totalCompressions = (long)redisResults[i]; break; + case "Memory usage": + memoryUsage = (long)redisResults[i]; + break; } } return new TdigestInformation(compression, capacity, mergedNodes, unmergedNodes, - mergedWeight, unmergedWeight, totalCompressions); + mergedWeight, unmergedWeight, sumWeight, totalCompressions, memoryUsage); } public static TimeSeriesInformation ToTimeSeriesInfo(this RedisResult result) @@ -531,7 +536,7 @@ public static IReadOnlyList ToStringArray(this RedisResult result) Array.ForEach(redisResults, str => list.Add((string)str)); return list; } - + public static long?[] ToNullableLongArray(this RedisResult result) { if (result.IsNull) diff --git a/src/NRedisStack/Tdigest/DataTypes/TdigestInformation.cs b/src/NRedisStack/Tdigest/DataTypes/TdigestInformation.cs index fdd39b06..e57fa3b4 100644 --- a/src/NRedisStack/Tdigest/DataTypes/TdigestInformation.cs +++ b/src/NRedisStack/Tdigest/DataTypes/TdigestInformation.cs @@ -12,13 +12,14 @@ public class TdigestInformation public long UnmergedNodes { get; private set; } public double MergedWeight { get; private set; } public double UnmergedWeight { get; private set; } - + public double SumWeights { get; private set; } public long TotalCompressions { get; private set; } + public long MemoryUsage { get; private set; } internal TdigestInformation(long compression, long capacity, long mergedNodes, long unmergedNodes, double mergedWeight, - double unmergedWeight, long totalCompressions) + double unmergedWeight, double sumWeights, long totalCompressions, long memoryUsage) { Compression = compression; @@ -27,7 +28,9 @@ internal TdigestInformation(long compression, long capacity, long mergedNodes, UnmergedNodes = unmergedNodes; MergedWeight = mergedWeight; UnmergedWeight = unmergedWeight; + SumWeights = sumWeights; TotalCompressions = totalCompressions; + MemoryUsage = memoryUsage; } } } \ No newline at end of file diff --git a/src/NRedisStack/Tdigest/Literals/Commands.cs b/src/NRedisStack/Tdigest/Literals/Commands.cs index c9ef8167..66be06eb 100644 --- a/src/NRedisStack/Tdigest/Literals/Commands.cs +++ b/src/NRedisStack/Tdigest/Literals/Commands.cs @@ -13,5 +13,9 @@ internal class TDIGEST public const string CDF = "TDIGEST.CDF"; public const string TRIMMED_MEAN = "TDIGEST.TRIMMED_MEAN"; public const string INFO = "TDIGEST.INFO"; + public const string RANK = "TDIGEST.RANK"; + public const string REVRANK = "TDIGEST.REVRANK"; + public const string BYRANK = "TDIGEST.BYRANK"; + public const string BYREVRANK = "TDIGEST.BYREVRANK"; } } \ No newline at end of file diff --git a/src/NRedisStack/Tdigest/TdigestCommands.cs b/src/NRedisStack/Tdigest/TdigestCommands.cs index ebe413cd..3e373973 100644 --- a/src/NRedisStack/Tdigest/TdigestCommands.cs +++ b/src/NRedisStack/Tdigest/TdigestCommands.cs @@ -20,7 +20,7 @@ public TdigestCommands(IDatabase db) /// The weight of this observation. /// if executed correctly, error otherwise /// - public bool Add(RedisKey key, double item, double weight) + public bool Add(RedisKey key, double item, long weight) { if (weight < 0) throw new ArgumentOutOfRangeException(nameof(weight)); @@ -35,7 +35,7 @@ public bool Add(RedisKey key, double item, double weight) /// The weight of this observation. /// if executed correctly, error otherwise /// - public async Task AddAsync(RedisKey key, double item, double weight) + public async Task AddAsync(RedisKey key, double item, int weight) { if (weight < 0) throw new ArgumentOutOfRangeException(nameof(weight)); @@ -50,7 +50,7 @@ public async Task AddAsync(RedisKey key, double item, double weight) /// Tuple of the value of the observation and The weight of this observation. /// if executed correctly, error otherwise /// - public bool Add(RedisKey key, params Tuple[] valueWeight) + public bool Add(RedisKey key, params Tuple[] valueWeight) { if (valueWeight.Length < 1) throw new ArgumentOutOfRangeException(nameof(valueWeight)); @@ -73,7 +73,7 @@ public bool Add(RedisKey key, params Tuple[] valueWeight) /// Tuple of the value of the observation and The weight of this observation. /// if executed correctly, error otherwise /// - public async Task AddAsync(RedisKey key, params Tuple[] valueWeight) + public async Task AddAsync(RedisKey key, params Tuple[] valueWeight) { if (valueWeight.Length < 1) throw new ArgumentOutOfRangeException(nameof(valueWeight)); @@ -93,25 +93,28 @@ public async Task AddAsync(RedisKey key, params Tuple[] va /// Estimate the fraction of all observations added which are <= value. /// /// The name of the sketch. - /// upper limit of observation value. + /// upper limit of observation value. /// double-reply - estimation of the fraction of all observations added which are <= value /// - public double CDF(RedisKey key, double value) + public double[] CDF(RedisKey key, params double[] values) { - return _db.Execute(TDIGEST.CDF, key, value).ToDouble(); + var args = new List(values.Length +1) { key }; + foreach(var value in values) args.Add(value); + return _db.Execute(TDIGEST.CDF, args).ToDoubleArray(); } /// /// Estimate the fraction of all observations added which are <= value. /// /// The name of the sketch. - /// upper limit of observation value. + /// upper limit of observation value. /// double-reply - estimation of the fraction of all observations added which are <= value /// - public async Task CDFAsync(RedisKey key, double value) + public async Task CDFAsync(RedisKey key, params double[] values) { - var result = await _db.ExecuteAsync(TDIGEST.CDF, key, value); - return result.ToDouble(); + var args = new List(values.Length +1) { key }; + foreach(var value in values) args.Add(value); + return (await _db.ExecuteAsync(TDIGEST.CDF, args)).ToDoubleArray(); } /// @@ -169,7 +172,8 @@ public async Task InfoAsync(RedisKey key) /// public double Max(RedisKey key) { - return _db.Execute(TDIGEST.MAX, key).ToDouble(); + var result = _db.Execute(TDIGEST.MAX, key); + return result.ToDouble(); } /// @@ -180,7 +184,8 @@ public double Max(RedisKey key) /// public async Task MaxAsync(RedisKey key) { - return (await _db.ExecuteAsync(TDIGEST.MAX, key)).ToDouble(); + var result = await _db.ExecuteAsync(TDIGEST.MAX, key); + return result.ToDouble(); } /// @@ -206,111 +211,76 @@ public async Task MinAsync(RedisKey key) } /// - /// Get the minimum observation value from the sketch. - /// - /// TSketch to copy observation values to (a t-digest data structure). - /// Sketch to copy observation values from (a t-digest data structure). - /// if executed correctly, error otherwise - /// - public bool Merge(RedisKey destinationKey, RedisKey sourceKey) - { - return _db.Execute(TDIGEST.MERGE, destinationKey, sourceKey).OKtoBoolean(); - } - - /// - /// Get the minimum observation value from the sketch. - /// - /// TSketch to copy observation values to (a t-digest data structure). - /// Sketch to copy observation values from (a t-digest data structure). - /// if executed correctly, error otherwise - /// - public async Task MergeAsync(RedisKey destinationKey, RedisKey sourceKey) - { - var result = await _db.ExecuteAsync(TDIGEST.MERGE, destinationKey, sourceKey); - return result.OKtoBoolean(); - } - - /// - /// Get the minimum observation value from the sketch. + /// Merges all of the values from 'from' keys to 'destination-key' sketch /// /// TSketch to copy observation values to (a t-digest data structure). + /// The compression parameter. + /// If destination already exists, it is overwritten. /// Sketch to copy observation values from (a t-digest data structure). /// if executed correctly, error otherwise /// - public bool Merge(RedisKey destinationKey, params RedisKey[] sourceKeys) + public bool Merge(RedisKey destinationKey, long compression = default(long), bool overide = false, params RedisKey[] sourceKeys) { if (sourceKeys.Length < 1) throw new ArgumentOutOfRangeException(nameof(sourceKeys)); - var args = sourceKeys.ToList(); - args.Insert(0, destinationKey); - - return _db.Execute(TDIGEST.MERGE, args).OKtoBoolean(); - } + int numkeys = sourceKeys.Length; + var args = new List() { destinationKey, numkeys}; + foreach(var key in sourceKeys) + { + args.Add(key); + } - /// - /// Get the minimum observation value from the sketch. - /// - /// TSketch to copy observation values to (a t-digest data structure). - /// Sketch to copy observation values from (a t-digest data structure). - /// if executed correctly, error otherwise - /// - public async Task MergeAsync(RedisKey destinationKey, params RedisKey[] sourceKeys) - { - if (sourceKeys.Length < 1) throw new ArgumentOutOfRangeException(nameof(sourceKeys)); + if (compression != default(long)) + { + args.Add("COMPRESSION"); + args.Add(compression); + } - var args = sourceKeys.ToList(); - args.Insert(0, destinationKey); + if(overide) + { + args.Add("OVERRIDE"); + } - var result = await _db.ExecuteAsync(TDIGEST.MERGE, args); - return result.OKtoBoolean(); + return _db.Execute(TDIGEST.MERGE, args).OKtoBoolean(); } /// - /// Merges all of the values from 'from' keys to 'destination-key' sketch. + /// Merges all of the values from 'from' keys to 'destination-key' sketch /// /// TSketch to copy observation values to (a t-digest data structure). - /// Number of sketch(es) to copy observation values from. /// The compression parameter. + /// If destination already exists, it is overwritten. /// Sketch to copy observation values from (a t-digest data structure). /// if executed correctly, error otherwise - /// - public bool MergeStore(RedisKey destinationKey, long numkeys, long compression = 100, params RedisKey[] sourceKeys) + /// + public async Task MergeAsync(RedisKey destinationKey, long compression = default(long), bool overide = false, params RedisKey[] sourceKeys) { if (sourceKeys.Length < 1) throw new ArgumentOutOfRangeException(nameof(sourceKeys)); - var args = new List { destinationKey, numkeys }; - foreach (var key in sourceKeys) args.Add(key); - args.Add(TdigestArgs.COMPRESSION); - args.Add(compression); - - return _db.Execute(TDIGEST.MERGESTORE, args).OKtoBoolean(); - } + int numkeys = sourceKeys.Length; + var args = new List() { destinationKey, numkeys}; + foreach(var key in sourceKeys) + { + args.Add(key); + } - /// - /// Merges all of the values from 'from' keys to 'destination-key' sketch. - /// - /// TSketch to copy observation values to (a t-digest data structure). - /// Number of sketch(es) to copy observation values from. - /// The compression parameter. - /// Sketch to copy observation values from (a t-digest data structure). - /// if executed correctly, error otherwise - /// - public async Task MergeStoreAsync(RedisKey destinationKey, long numkeys, long compression = 100, params RedisKey[] sourceKeys) - { - if (sourceKeys.Length < 1) throw new ArgumentOutOfRangeException(nameof(sourceKeys)); + if (compression != default(long)) + { + args.Add("COMPRESSION"); + args.Add(compression); + } - var args = new List { destinationKey, numkeys }; - foreach (var key in sourceKeys) args.Add(key); - args.Add(TdigestArgs.COMPRESSION); - args.Add(compression); + if(overide) + { + args.Add("OVERRIDE"); + } - var result = await _db.ExecuteAsync(TDIGEST.MERGESTORE, args); - return result.OKtoBoolean(); + return (await _db.ExecuteAsync(TDIGEST.MERGE, args)).OKtoBoolean(); } /// /// Returns estimates of one or more cutoffs such that a specified fraction of the observations - ///added to this t-digest would be less than or equal to each of the specified cutoffs. + /// added to this t-digest would be less than or equal to each of the specified cutoffs. /// /// The name of the sketch (a t-digest data structure). /// The desired fraction (between 0 and 1 inclusively). @@ -344,13 +314,145 @@ public async Task QuantileAsync(RedisKey key, params double[] quantile return (await _db.ExecuteAsync(TDIGEST.QUANTILE, args)).ToDoubleArray(); } + /// + /// Retrieve the estimated rank of value (the number of observations in the sketch + /// that are smaller than value + half the number of observations that are equal to value). + /// + /// The name of the sketch (a t-digest data structure). + /// input value, for which the rank will be determined. + /// an array of results populated with rank_1, rank_2, ..., rank_N. + /// + public long[] Rank(RedisKey key, params long[] values) + { + if (values.Length < 1) throw new ArgumentOutOfRangeException(nameof(values)); + + var args = new List(values.Length + 1) { key }; + foreach (var v in values) args.Add(v); + return _db.Execute(TDIGEST.RANK, args).ToLongArray(); + } + + /// + /// Retrieve the estimated rank of value (the number of observations in the sketch + /// that are smaller than value + half the number of observations that are equal to value). + /// + /// The name of the sketch (a t-digest data structure). + /// input value, for which the rank will be determined. + /// an array of results populated with rank_1, rank_2, ..., rank_N. + /// + public async Task RankAsync(RedisKey key, params long[] values) + { + if (values.Length < 1) throw new ArgumentOutOfRangeException(nameof(values)); + + var args = new List(values.Length + 1) { key }; + foreach (var v in values) args.Add(v); + return (await _db.ExecuteAsync(TDIGEST.RANK, args)).ToLongArray(); + } + + /// + /// Retrieve the estimated rank of value (the number of observations in the sketch + /// that are larger than value + half the number of observations that are equal to value). + /// + /// The name of the sketch (a t-digest data structure). + /// input value, for which the rank will be determined. + /// an array of results populated with rank_1, rank_2, ..., rank_N. + /// + public long[] RevRank(RedisKey key, params long[] values) + { + if (values.Length < 1) throw new ArgumentOutOfRangeException(nameof(values)); + + var args = new List(values.Length + 1) { key }; + foreach (var v in values) args.Add(v); + return _db.Execute(TDIGEST.REVRANK, args).ToLongArray(); + } + + /// + /// Retrieve the estimated rank of value (the number of observations in the sketch + /// that are larger than value + half the number of observations that are equal to value). + /// + /// The name of the sketch (a t-digest data structure). + /// input value, for which the rank will be determined. + /// an array of results populated with rank_1, rank_2, ..., rank_N. + /// + public async Task RevRankAsync(RedisKey key, params long[] values) + { + if (values.Length < 1) throw new ArgumentOutOfRangeException(nameof(values)); + + var args = new List(values.Length + 1) { key }; + foreach (var v in values) args.Add(v); + return ( await _db.ExecuteAsync(TDIGEST.REVRANK, args)).ToLongArray(); + } + + /// + /// Retrieve an estimation of the value with the given the rank. + /// + /// The name of the sketch (a t-digest data structure). + /// input rank, for which the value will be determined. + /// an array of results populated with value_1, value_2, ..., value_N. + /// + public double[] ByRank(RedisKey key, params long[] ranks) + { + if (ranks.Length < 1) throw new ArgumentOutOfRangeException(nameof(ranks)); + + var args = new List(ranks.Length + 1) { key }; + foreach (var v in ranks) args.Add(v); + return _db.Execute(TDIGEST.BYRANK, args).ToDoubleArray(); + } + + /// + /// Retrieve an estimation of the value with the given the rank. + /// + /// The name of the sketch (a t-digest data structure). + /// input rank, for which the value will be determined. + /// an array of results populated with value_1, value_2, ..., value_N. + /// + public async Task ByRankAsync(RedisKey key, params long[] ranks) + { + if (ranks.Length < 1) throw new ArgumentOutOfRangeException(nameof(ranks)); + + var args = new List(ranks.Length + 1) { key }; + foreach (var v in ranks) args.Add(v); + return (await _db.ExecuteAsync(TDIGEST.BYRANK, args)).ToDoubleArray(); + } + + /// + /// Retrieve an estimation of the value with the given the reverse rank. + /// + /// The name of the sketch (a t-digest data structure). + /// input reverse rank, for which the value will be determined. + /// an array of results populated with value_1, value_2, ..., value_N. + /// + public double[] ByRevRank(RedisKey key, params long[] ranks) + { + if (ranks.Length < 1) throw new ArgumentOutOfRangeException(nameof(ranks)); + + var args = new List(ranks.Length + 1) { key }; + foreach (var v in ranks) args.Add(v); + return _db.Execute(TDIGEST.BYREVRANK, args).ToDoubleArray(); + } + + /// + /// Retrieve an estimation of the value with the given the reverse rank. + /// + /// The name of the sketch (a t-digest data structure). + /// input reverse rank, for which the value will be determined. + /// an array of results populated with value_1, value_2, ..., value_N. + /// + public async Task ByRevRankAsync(RedisKey key, params long[] ranks) + { + if (ranks.Length < 1) throw new ArgumentOutOfRangeException(nameof(ranks)); + + var args = new List(ranks.Length + 1) { key }; + foreach (var v in ranks) args.Add(v); + return ( await _db.ExecuteAsync(TDIGEST.BYREVRANK, args)).ToDoubleArray(); + } + /// /// Reset the sketch - empty the sketch and re-initialize it /// /// The name of the sketch (a t-digest data structure). /// if executed correctly, error otherwise. /// - public bool Reset(RedisKey key, params double[] quantile) + public bool Reset(RedisKey key) { return _db.Execute(TDIGEST.RESET, key).OKtoBoolean(); } @@ -361,7 +463,7 @@ public bool Reset(RedisKey key, params double[] quantile) /// The name of the sketch (a t-digest data structure). /// if executed correctly, error otherwise. /// - public async Task ResetAsync(RedisKey key, params double[] quantile) + public async Task ResetAsync(RedisKey key) { return (await _db.ExecuteAsync(TDIGEST.RESET, key)).OKtoBoolean(); } @@ -372,8 +474,8 @@ public async Task ResetAsync(RedisKey key, params double[] quantile) /// The name of the sketch (a t-digest data structure). /// Exclude observation values lower than this quantile. /// Exclude observation values higher than this quantile. - /// estimation of the mean value. Will return DBL_MAX if the sketch is empty. - /// + /// estimation of the mean value. Will return NaN if the sketch is empty. + /// public double TrimmedMean(RedisKey key, double lowCutQuantile, double highCutQuantile) { return _db.Execute(TDIGEST.TRIMMED_MEAN, key, lowCutQuantile, highCutQuantile).ToDouble(); @@ -385,24 +487,11 @@ public double TrimmedMean(RedisKey key, double lowCutQuantile, double highCutQua /// The name of the sketch (a t-digest data structure). /// Exclude observation values lower than this quantile. /// Exclude observation values higher than this quantile. - /// estimation of the mean value. Will return DBL_MAX if the sketch is empty. - /// + /// estimation of the mean value. Will return NaN if the sketch is empty. + /// public async Task TrimmedMeanAsync(RedisKey key, double lowCutQuantile, double highCutQuantile) { return (await _db.ExecuteAsync(TDIGEST.TRIMMED_MEAN, key, lowCutQuantile, highCutQuantile)).ToDouble(); } - - - - - - - - - - - - - } } diff --git a/tests/NRedisStack.Tests/Tdigest/TdigestTests.cs b/tests/NRedisStack.Tests/Tdigest/TdigestTests.cs index f79cf474..9201c204 100644 --- a/tests/NRedisStack.Tests/Tdigest/TdigestTests.cs +++ b/tests/NRedisStack.Tests/Tdigest/TdigestTests.cs @@ -90,6 +90,166 @@ public async Task TestCreateAndInfoAsync() } } + [Fact] + public void TestRank() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var tdigest = db.TDIGEST(); + + Assert.True(tdigest.Create("t-digest", 500)); + var tuples = new Tuple[20]; + for (int i = 0; i < 20; i++) + { + tuples[i] = new(i, 1); + } + Assert.True(tdigest.Add("t-digest", tuples)); + Assert.Equal(-1, tdigest.Rank("t-digest", -1)[0]); + Assert.Equal(1, tdigest.Rank("t-digest", 0)[0]); + Assert.Equal(11, tdigest.Rank("t-digest", 10)[0]); + Assert.Equal(new long[3] { -1, 20, 10 }, tdigest.Rank("t-digest", -20, 20, 9)); + } + + [Fact] + public async Task TestRankAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var tdigest = db.TDIGEST(); + + Assert.True(tdigest.Create("t-digest", 500)); + var tuples = new Tuple[20]; + for (int i = 0; i < 20; i++) + { + tuples[i] = new(i, 1); + } + Assert.True(tdigest.Add("t-digest", tuples)); + Assert.Equal(-1, (await tdigest.RankAsync("t-digest", -1))[0]); + Assert.Equal(1, (await tdigest.RankAsync("t-digest", 0))[0]); + Assert.Equal(11, (await tdigest.RankAsync("t-digest", 10))[0]); + Assert.Equal(new long[3] { -1, 20, 10 }, await tdigest.RankAsync("t-digest", -20, 20, 9)); + } + + [Fact] + public void TestRevRank() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var tdigest = db.TDIGEST(); + + Assert.True(tdigest.Create("t-digest", 500)); + var tuples = new Tuple[20]; + for (int i = 0; i < 20; i++) + { + tuples[i] = new(i, 1); + } + + Assert.True(tdigest.Add("t-digest", tuples)); + Assert.Equal(-1, tdigest.RevRank("t-digest", 20)[0]); + Assert.Equal(20, tdigest.RevRank("t-digest", 0)[0]); + Assert.Equal(new long[3] { -1, 20, 10 }, tdigest.RevRank("t-digest", 21, 0, 10)); + } + + [Fact] + public async Task TestRevRankAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var tdigest = db.TDIGEST(); + + Assert.True(tdigest.Create("t-digest", 500)); + var tuples = new Tuple[20]; + for (int i = 0; i < 20; i++) + { + tuples[i] = new(i, 1); + } + + Assert.True(tdigest.Add("t-digest", tuples)); + Assert.Equal(-1, (await tdigest.RevRankAsync("t-digest", 20))[0]); + Assert.Equal(20, (await tdigest.RevRankAsync("t-digest", 0))[0]); + Assert.Equal(new long[3] { -1, 20, 10 }, await tdigest.RevRankAsync("t-digest", 21, 0, 10)); + } + + // TODO: fix those tests: + [Fact] + public void TestByRank() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var tdigest = db.TDIGEST(); + + Assert.True(tdigest.Create("t-digest", 500)); + var tuples = new Tuple[10]; + for (int i = 1; i <= 10; i++) + { + tuples[i - 1] = new(i, 1); + } + Assert.True(tdigest.Add("t-digest", tuples)); + Assert.Equal(1, tdigest.ByRank("t-digest", 0)[0]); + Assert.Equal(10, tdigest.ByRank("t-digest", 9)[0]); + Assert.True(double.IsInfinity(tdigest.ByRank("t-digest", 100)[0])); + //Assert.Throws(() => tdigest.ByRank("t-digest", -1)[0]); + } + + [Fact] + public async Task TestByRankAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var tdigest = db.TDIGEST(); + + Assert.True(tdigest.Create("t-digest", 500)); + var tuples = new Tuple[10]; + for (int i = 1; i <= 10; i++) + { + tuples[i - 1] = new(i, 1); + } + Assert.True(tdigest.Add("t-digest", tuples)); + Assert.Equal(1, (await tdigest.ByRankAsync("t-digest", 0))[0]); + Assert.Equal(10, (await tdigest.ByRankAsync("t-digest", 9))[0]); + Assert.True(double.IsInfinity((await tdigest.ByRankAsync("t-digest", 100))[0])); + } + + [Fact] + public void TestByRevRank() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var tdigest = db.TDIGEST(); + + Assert.True(tdigest.Create("t-digest", 500)); + var tuples = new Tuple[10]; + for (int i = 1; i <= 10; i++) + { + tuples[i - 1] = new(i, 1); + } + Assert.True(tdigest.Add("t-digest", tuples)); + Assert.Equal(10, tdigest.ByRevRank("t-digest", 0)[0]); + Assert.Equal(2, tdigest.ByRevRank("t-digest", 9)[0]); + Assert.True(double.IsInfinity(-tdigest.ByRevRank("t-digest", 100)[0])); + //Assert.Throws(() => tdigest.ByRank("t-digest", -1)[0]); + } + + [Fact] + public async Task TestByRevRankAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var tdigest = db.TDIGEST(); + + Assert.True(tdigest.Create("t-digest", 500)); + var tuples = new Tuple[10]; + for (int i = 1; i <= 10; i++) + { + tuples[i - 1] = new(i, 1); + } + Assert.True(tdigest.Add("t-digest", tuples)); + Assert.Equal(10, (await tdigest.ByRevRankAsync("t-digest", 0))[0]); + Assert.Equal(2, (await tdigest.ByRevRankAsync("t-digest", 9))[0]); + Assert.True(double.IsInfinity(-(await tdigest.ByRevRankAsync("t-digest", 100))[0])); + } + + [Fact] public void TestReset() { @@ -164,7 +324,6 @@ public async Task TestAddAsync() AssertMergedUnmergedNodes(tdigest, "tdadd", 0, 5); } - [Fact] public void TestMerge() { @@ -175,13 +334,13 @@ public void TestMerge() tdigest.Create("td2", 100); tdigest.Create("td4m", 100); - Assert.True(tdigest.Merge("td2", "td4m")); + Assert.True(tdigest.Merge("td2", sourceKeys: "td4m")); AssertMergedUnmergedNodes(tdigest, "td2", 0, 0); tdigest.Add("td2", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); tdigest.Add("td4m", DefinedValueWeight(1, 100), DefinedValueWeight(1, 100)); - Assert.True(tdigest.Merge("td2", "td4m")); + Assert.True(tdigest.Merge("td2", sourceKeys: "td4m")); AssertMergedUnmergedNodes(tdigest, "td2", 3, 2); } @@ -196,54 +355,52 @@ public async Task TestMergeAsync() await tdigest.CreateAsync("td2", 100); await tdigest.CreateAsync("td4m", 100); - Assert.True(await tdigest.MergeAsync("td2", "td4m")); + Assert.True(await tdigest.MergeAsync("td2", sourceKeys: "td4m")); AssertMergedUnmergedNodes(tdigest, "td2", 0, 0); await tdigest.AddAsync("td2", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); await tdigest.AddAsync("td4m", DefinedValueWeight(1, 100), DefinedValueWeight(1, 100)); - Assert.True(await tdigest.MergeAsync("td2", "td4m")); + Assert.True(await tdigest.MergeAsync("td2", sourceKeys: "td4m")); AssertMergedUnmergedNodes(tdigest, "td2", 3, 2); } [Fact] - public void TestMergeStore() + public void MergeMultiAndParams() { IDatabase db = redisFixture.Redis.GetDatabase(); db.Execute("FLUSHALL"); var tdigest = db.TDIGEST(); - tdigest.Create("from1", 100); tdigest.Create("from2", 200); - tdigest.Add("from1", 1, 1); - tdigest.Add("from2", 1, 10); + tdigest.Add("from1", 1d, 1); + tdigest.Add("from2", 1d, 10); - Assert.True(tdigest.MergeStore("to", 2, 100, "from1", "from2")); + Assert.True(tdigest.Merge("to", 2, sourceKeys: new RedisKey[] { "from1", "from2" })); AssertTotalWeight(tdigest, "to", 11d); - Assert.True(tdigest.MergeStore("to50", 2, 50, "from1", "from2")); - Assert.Equal(50, tdigest.Info("to50").Compression); + Assert.True(tdigest.Merge("to", 50, true, "from1", "from2")); + Assert.Equal(50, tdigest.Info("to").Compression); } [Fact] - public async Task TestMergeStoreAsync() + public async Task MergeMultiAndParamsAsync() { IDatabase db = redisFixture.Redis.GetDatabase(); db.Execute("FLUSHALL"); var tdigest = db.TDIGEST(); + tdigest.Create("from1", 100); + tdigest.Create("from2", 200); - await tdigest.CreateAsync("from1", 100); - await tdigest.CreateAsync("from2", 200); - - await tdigest.AddAsync("from1", 1, 1); - await tdigest.AddAsync("from2", 1, 10); + tdigest.Add("from1", 1d, 1); + tdigest.Add("from2", 1d, 10); - Assert.True(await tdigest.MergeStoreAsync("to", 2, 100, "from1", "from2")); + Assert.True(await tdigest.MergeAsync("to", 2, sourceKeys: new RedisKey[] { "from1", "from2" })); AssertTotalWeight(tdigest, "to", 11d); - Assert.True(await tdigest.MergeStoreAsync("to50", 2, 50, "from1", "from2")); - Assert.Equal(50, (await tdigest.InfoAsync("to50")).Compression); + Assert.True(await tdigest.MergeAsync("to", 50, true, "from1", "from2")); + Assert.Equal(50, tdigest.Info("to").Compression); } [Fact] @@ -254,11 +411,14 @@ public void TestCDF() var tdigest = db.TDIGEST(); tdigest.Create("tdcdf", 100); - Assert.Equal(double.NaN, tdigest.CDF("tdcdf", 50)); + foreach (var item in tdigest.CDF("tdcdf", 50)) + { + Assert.Equal(double.NaN, item); + } tdigest.Add("tdcdf", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); tdigest.Add("tdcdf", DefinedValueWeight(100, 1), DefinedValueWeight(100, 1)); - Assert.Equal(0.6, tdigest.CDF("tdcdf", 50)); + Assert.Equal(new double[] { 0.6 }, tdigest.CDF("tdcdf", 50)); } [Fact] @@ -269,11 +429,14 @@ public async Task TestCDFAsync() var tdigest = db.TDIGEST(); await tdigest.CreateAsync("tdcdf", 100); - Assert.Equal(double.NaN, await tdigest.CDFAsync("tdcdf", 50)); + foreach (var item in (await tdigest.CDFAsync("tdcdf", 50))) + { + Assert.Equal(double.NaN, item); + } await tdigest.AddAsync("tdcdf", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); await tdigest.AddAsync("tdcdf", DefinedValueWeight(100, 1), DefinedValueWeight(100, 1)); - Assert.Equal(0.6, await tdigest.CDFAsync("tdcdf", 50)); + Assert.Equal(new double[] { 0.6 }, await tdigest.CDFAsync("tdcdf", 50)); } [Fact] @@ -316,8 +479,8 @@ public void TestMinAndMax() var tdigest = db.TDIGEST(); tdigest.Create(key, 100); - Assert.Equal(double.MaxValue, tdigest.Min(key)); - Assert.Equal(-double.MaxValue, tdigest.Max(key)); + Assert.Equal(double.NaN, tdigest.Min(key)); + Assert.Equal(double.NaN, tdigest.Max(key)); tdigest.Add(key, DefinedValueWeight(2, 1)); tdigest.Add(key, DefinedValueWeight(5, 1)); @@ -333,8 +496,8 @@ public async Task TestMinAndMaxAsync() var tdigest = db.TDIGEST(); await tdigest.CreateAsync(key, 100); - Assert.Equal(double.MaxValue, await tdigest.MinAsync(key)); - Assert.Equal(-double.MaxValue, await tdigest.MaxAsync(key)); + Assert.Equal(double.NaN, await tdigest.MinAsync(key)); + Assert.Equal(double.NaN, await tdigest.MaxAsync(key)); await tdigest.AddAsync(key, DefinedValueWeight(2, 1)); await tdigest.AddAsync(key, DefinedValueWeight(5, 1)); @@ -353,7 +516,7 @@ public void TestTrimmedMean() for (int i = 0; i < 20; i++) { - tdigest.Add(key, new Tuple(i, 1)); + tdigest.Add(key, new Tuple(i, 1)); } Assert.Equal(9.5, tdigest.TrimmedMean(key, 0.1, 0.9)); @@ -373,7 +536,7 @@ public async Task TestTrimmedMeanAsync() for (int i = 0; i < 20; i++) { - await tdigest.AddAsync(key, new Tuple(i, 1)); + await tdigest.AddAsync(key, new Tuple(i, 1)); } Assert.Equal(9.5, await tdigest.TrimmedMeanAsync(key, 0.1, 0.9)); @@ -418,15 +581,25 @@ public void TestModulePrefixs1() } - static Tuple RandomValueWeight() + static Tuple RandomValueWeight() { Random random = new Random(); - return new Tuple(random.NextDouble() * 10000, random.NextDouble() * 500 + 1); + return new Tuple(random.NextDouble() * 10000, random.NextInt64() + 1); + } + + static Tuple[] RandomValueWeightArray(int count) + { + var arr = new Tuple[count]; + for (int i = 0; i < count; i++) + { + arr[i] = RandomValueWeight(); + } + return arr; } - static Tuple DefinedValueWeight(double value, double weight) + static Tuple DefinedValueWeight(double value, long weight) { - return new Tuple(value, weight); + return new Tuple(value, weight); } } From a8846bc4bb436843110f1daaf919e9a777f09dcd Mon Sep 17 00:00:00 2001 From: shacharPash Date: Sun, 2 Oct 2022 15:01:06 +0300 Subject: [PATCH 13/28] Test Commit --- tests/NRedisStack.Tests/Tdigest/TdigestTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/NRedisStack.Tests/Tdigest/TdigestTests.cs b/tests/NRedisStack.Tests/Tdigest/TdigestTests.cs index 9201c204..2e46c406 100644 --- a/tests/NRedisStack.Tests/Tdigest/TdigestTests.cs +++ b/tests/NRedisStack.Tests/Tdigest/TdigestTests.cs @@ -170,7 +170,6 @@ public async Task TestRevRankAsync() Assert.Equal(new long[3] { -1, 20, 10 }, await tdigest.RevRankAsync("t-digest", 21, 0, 10)); } - // TODO: fix those tests: [Fact] public void TestByRank() { From b7bd388fc1158c0d86835129ee3ef8eaeff12d62 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Sun, 2 Oct 2022 17:43:50 +0300 Subject: [PATCH 14/28] Add Async Commands & tests --- src/NRedisStack/Search/SearchCommands.cs | 70 +++++++++- tests/NRedisStack.Tests/Search/SearchTests.cs | 122 ++++++++++++++++++ 2 files changed, 190 insertions(+), 2 deletions(-) diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index d10ca631..3bddf2eb 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -22,6 +22,16 @@ public RedisResult[] _List() return _db.Execute(FT._LIST).ToArray(); } + /// + /// Returns a list of all existing indexes. + /// + /// Array with index names. + /// + public async Task _ListAsync() + { + return (await _db.ExecuteAsync(FT._LIST)).ToArray(); + } + // TODO: Aggregate /// @@ -106,7 +116,7 @@ public async Task AliasUpdateAsync(string alias, string index) /// public bool Alter(string index, Schema schema, bool skipInitialScan = false) { - List args = new List(){index}; + List args = new List() { index }; if (skipInitialScan) args.Add("SKIPINITIALSCAN"); args.Add("SCHEMA"); args.Add("ADD"); @@ -117,6 +127,28 @@ public bool Alter(string index, Schema schema, bool skipInitialScan = false) return _db.Execute(FT.ALTER, args).OKtoBoolean(); } + /// + /// Add a new attribute to the index + /// + /// The index name. + /// If set, does not scan and index. + /// the schema. + /// if executed correctly, error otherwise + /// + public async Task AlterAsync(string index, Schema schema, bool skipInitialScan = false) + { + List args = new List() { index }; + if (skipInitialScan) args.Add("SKIPINITIALSCAN"); + args.Add("SCHEMA"); + args.Add("ADD"); + foreach (var f in schema.Fields) + { + f.AddSchemaArgs(args); + } + return (await _db.ExecuteAsync(FT.ALTER, args)).OKtoBoolean(); + } + + // TODO: finish this & add summary public RedisResult Info(RedisValue index) { return _db.Execute(FT.INFO, index); @@ -145,6 +177,26 @@ public bool Create(string indexName, FTCreateParams parameters, Schema schema) return _db.Execute(FT.CREATE, args).OKtoBoolean(); } + /// + /// Create an index with the given specification. + /// + /// The index name. + /// Command's parameters. + /// The index schema. + /// if executed correctly, error otherwise + /// + public async Task CreateAsync(string indexName, FTCreateParams parameters, Schema schema) + { + var args = new List() { indexName }; + parameters.AddParams(args); // TODO: Think of a better implementation + args.Add("SCHEMA"); + foreach (var f in schema.Fields) + { + f.AddSchemaArgs(args); + } + return (await _db.ExecuteAsync(FT.CREATE, args)).OKtoBoolean(); + } + /// /// Search the index /// @@ -153,7 +205,7 @@ public bool Create(string indexName, FTCreateParams parameters, Schema schema) /// a object with the results public SearchResult Search(string indexName, Query q) { - var args = new List{indexName}; + var args = new List { indexName }; // { // _boxedIndexName // }; @@ -162,5 +214,19 @@ public SearchResult Search(string indexName, Query q) var resp = _db.Execute("FT.SEARCH", args).ToArray(); return new SearchResult(resp, !q.NoContent, q.WithScores, q.WithPayloads, q.ExplainScore); } + + /// + /// Search the index + /// + /// The index name + /// a object with the query string and optional parameters + /// a object with the results + public async Task SearchAsync(string indexName, Query q) + { + var args = new List { indexName }; + q.SerializeRedisArgs(args); + var resp = (await _db.ExecuteAsync("FT.SEARCH", args)).ToArray(); + return new SearchResult(resp, !q.NoContent, q.WithScores, q.WithPayloads, q.ExplainScore); + } } } \ No newline at end of file diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index f762e264..8a77cc65 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -52,6 +52,32 @@ public void TestCreate() Assert.Equal(0, res3.TotalResults); } + [Fact] + public async Task TestCreateAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + var schema = new Schema().AddTextField("first").AddTextField("last").AddNumericField("age"); + var parameters = FTCreateParams.CreateParams().Filter("@age>16").Prefix("student:", "pupil:"); + Assert.True(await ft.CreateAsync(index, parameters, schema)); + db.HashSet("profesor:5555", new HashEntry[] { new("first", "Albert"), new("last", "Blue"), new("age", "55") }); + db.HashSet("student:1111", new HashEntry[] { new("first", "Joe"), new("last", "Dod"), new("age", "18") }); + db.HashSet("pupil:2222", new HashEntry[] { new("first", "Jen"), new("last", "Rod"), new("age", "14") }); + db.HashSet("student:3333", new HashEntry[] { new("first", "El"), new("last", "Mark"), new("age", "17") }); + db.HashSet("pupil:4444", new HashEntry[] { new("first", "Pat"), new("last", "Shu"), new("age", "21") }); + db.HashSet("student:5555", new HashEntry[] { new("first", "Joen"), new("last", "Ko"), new("age", "20") }); + db.HashSet("teacher:6666", new HashEntry[] { new("first", "Pat"), new("last", "Rod"), new("age", "20") }); + var noFilters = ft.Search(index, new Query()); + Assert.Equal(4, noFilters.TotalResults); + var res1 = ft.Search(index, new Query("@first:Jo*")); + Assert.Equal(2, res1.TotalResults); + var res2 = ft.Search(index, new Query("@first:Pat")); + Assert.Equal(1, res2.TotalResults); + var res3 = ft.Search(index, new Query("@last:Rod")); + Assert.Equal(0, res3.TotalResults); + } + [Fact] public void CreateNoParams() { @@ -80,6 +106,34 @@ public void CreateNoParams() Assert.Equal(0, res3.TotalResults); } + [Fact] + public async Task CreateNoParamsAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + + Schema sc = new Schema().AddTextField("first", 1.0).AddTextField("last", 1.0).AddNumericField("age"); + Assert.True(await ft.CreateAsync(index, FTCreateParams.CreateParams(), sc)); + + db.HashSet("student:1111", new HashEntry[] { new("first", "Joe"), new("last", "Dod"), new("age", 18) }); + db.HashSet("student:3333", new HashEntry[] { new("first", "El"), new("last", "Mark"), new("age", 17) }); + db.HashSet("pupil:4444", new HashEntry[] { new("first", "Pat"), new("last", "Shu"), new("age", 21) }); + db.HashSet("student:5555", new HashEntry[] { new("first", "Joen"), new("last", "Ko"), new("age", 20) }); + + SearchResult noFilters = ft.Search(index, new Query()); + Assert.Equal(4, noFilters.TotalResults); + + SearchResult res1 = ft.Search(index, new Query("@first:Jo*")); + Assert.Equal(2, res1.TotalResults); + + SearchResult res2 = ft.Search(index, new Query("@first:Pat")); + Assert.Equal(1, res2.TotalResults); + + SearchResult res3 = ft.Search(index, new Query("@last:Rod")); + Assert.Equal(0, res3.TotalResults); + } + [Fact] public void CreateWithFieldNames() { @@ -112,6 +166,38 @@ public void CreateWithFieldNames() Assert.Equal(1, nonAttribute.TotalResults); } + [Fact] + public async Task CreateWithFieldNamesAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema().AddField(new TextField(FieldName.Of("first").As("given"))) + .AddField(new TextField(FieldName.Of("last"))); + + Assert.True(await ft.CreateAsync(index, FTCreateParams.CreateParams().Prefix("student:", "pupil:"), sc)); + + db.HashSet("profesor:5555", new HashEntry[] { new("first", "Albert"), new("last", "Blue"), new("age", "55") }); + db.HashSet("student:1111", new HashEntry[] { new("first", "Joe"), new("last", "Dod"), new("age", "18") }); + db.HashSet("pupil:2222", new HashEntry[] { new("first", "Jen"), new("last", "Rod"), new("age", "14") }); + db.HashSet("student:3333", new HashEntry[] { new("first", "El"), new("last", "Mark"), new("age", "17") }); + db.HashSet("pupil:4444", new HashEntry[] { new("first", "Pat"), new("last", "Shu"), new("age", "21") }); + db.HashSet("student:5555", new HashEntry[] { new("first", "Joen"), new("last", "Ko"), new("age", "20") }); + db.HashSet("teacher:6666", new HashEntry[] { new("first", "Pat"), new("last", "Rod"), new("age", "20") }); + + SearchResult noFilters = await ft.SearchAsync(index, new Query()); + Assert.Equal(5, noFilters.TotalResults); + + SearchResult asOriginal = await ft.SearchAsync(index, new Query("@first:Jo*")); + Assert.Equal(0, asOriginal.TotalResults); + + SearchResult asAttribute = await ft.SearchAsync(index, new Query("@given:Jo*")); + Assert.Equal(2, asAttribute.TotalResults); + + SearchResult nonAttribute = await ft.SearchAsync(index, new Query("@last:Rod")); + Assert.Equal(1, nonAttribute.TotalResults); + } + [Fact] public void AlterAdd() { @@ -148,6 +234,42 @@ public void AlterAdd() // Assert.Equal("attribute", ((List)((List)info.get("attributes")).get(1)).get(2)); } + [Fact] + public async Task AlterAddAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema().AddTextField("title", 1.0); + + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + var fields = new HashEntry("title", "hello world"); + //fields.("title", "hello world"); + for (int i = 0; i < 100; i++) + { + db.HashSet($"doc{i}", fields.Name, fields.Value); + } + SearchResult res = ft.Search(index, new Query("hello world")); + Assert.Equal(100, res.TotalResults); + + Assert.True(await ft.AlterAsync(index, new Schema().AddTagField("tags").AddTextField("name", weight: 0.5))); + for (int i = 0; i < 100; i++) + { + var fields2 = new HashEntry[] { new("name", "name" + i), + new("tags", $"tagA,tagB,tag{i}") }; + // assertTrue(client.updateDocument(string.format("doc%d", i), 1.0, fields2)); + db.HashSet($"doc{i}", fields2); + } + SearchResult res2 = ft.Search(index, new Query("@tags:{tagA}")); + Assert.Equal(100, res2.TotalResults); + + //TODO: complete this test when I finish the command FT.INFO + // var info = ft.Info(index); + // Assert.Equal(index, info.get("index_name")); + // Assert.Equal("identifier", ((List)((List)info.get("attributes")).get(1)).get(0)); + // Assert.Equal("attribute", ((List)((List)info.get("attributes")).get(1)).get(2)); + } + [Fact] public void TestModulePrefixs() { From f77ca6f4203e6f0ba9bc3f9c2ef5f71bd2738ab1 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Sun, 2 Oct 2022 19:41:34 +0300 Subject: [PATCH 15/28] Working on FT.INFO Command --- src/NRedisStack/ResponseParser.cs | 15 ++ .../Search/DataTypes/InfoResult.cs | 142 ++++++++++++++++++ .../Search/DataTypes/SearchInformation.cs | 60 ++++---- src/NRedisStack/Search/SearchCommands.cs | 26 +++- tests/NRedisStack.Tests/Search/SearchTests.cs | 15 +- 5 files changed, 216 insertions(+), 42 deletions(-) create mode 100644 src/NRedisStack/Search/DataTypes/InfoResult.cs diff --git a/src/NRedisStack/ResponseParser.cs b/src/NRedisStack/ResponseParser.cs index 5c9be0e6..5700191b 100644 --- a/src/NRedisStack/ResponseParser.cs +++ b/src/NRedisStack/ResponseParser.cs @@ -485,6 +485,21 @@ public static TimeSeriesInformation ToTimeSeriesInfo(this RedisResult result) lastTimestamp, retentionTime, chunkCount, chunkSize, labels, sourceKey, rules, duplicatePolicy, keySelfName, chunks); } + public static Dictionary ToFtInfoAsDictionary(this RedisResult value) + { + var res = (RedisResult[])value; + var info = new Dictionary(); + for (int i = 0; i < res.Length; i += 2) + { + var val = res[i + 1]; + if (val.Type != ResultType.MultiBulk) + { + info.Add((string)res[i], (RedisValue)val); + } + } + return info; + } + public static IReadOnlyList ToTimeSeriesChunkArray(this RedisResult result) { RedisResult[] redisResults = (RedisResult[])result; diff --git a/src/NRedisStack/Search/DataTypes/InfoResult.cs b/src/NRedisStack/Search/DataTypes/InfoResult.cs new file mode 100644 index 00000000..b43ce81f --- /dev/null +++ b/src/NRedisStack/Search/DataTypes/InfoResult.cs @@ -0,0 +1,142 @@ +using System.Collections.Generic; +using StackExchange.Redis; + +namespace NRedisStack.Search.DataTypes +{ + public class InfoResult + { + private readonly Dictionary _all = new Dictionary(); + + public string IndexName => GetString("index_name"); + public Dictionary IndexOption => GetRedisResultDictionary("index_options"); + public Dictionary IndexDefinition => GetRedisResultsDictionary("index_definition"); + + public Dictionary Attributes => GetRedisResultsDictionary("attributes"); // TODO: check if this is correct + + public long NumDocs => GetLong("num_docs"); + + public string MaxDocId => GetString("max_doc_id"); + + public long NumTerms => GetLong("num_terms"); + + public long NumRecords => GetLong("num_records"); + + public double InvertedSzMebibytes => GetDouble("inverted_sz_mb"); + + public double VectorIndexSzMebibytes => GetDouble("vector_index_sz_mb"); // TODO: check if double or long + + public double TotalInvertedIndexBlocks => GetDouble("total_inverted_index_blocks"); + + // public double InvertedCapOvh => GetDouble("inverted_cap_ovh"); + + public double OffsetVectorsSzMebibytes => GetDouble("offset_vectors_sz_mb"); + + public double DocTableSizeMebibytes => GetDouble("doc_table_size_mb"); + + public double SortableValueSizeMebibytes => GetDouble("sortable_value_size_mb"); + + public double KeyTableSizeMebibytes => GetDouble("key_table_size_mb"); + + // public double SkipIndexSizeMebibytes => GetDouble("skip_index_size_mb"); + + // public double ScoreIndexSizeMebibytes => GetDouble("score_index_size_mb"); + + public double RecordsPerDocAvg => GetDouble("records_per_doc_avg"); + + public double BytesPerRecordAvg => GetDouble("bytes_per_record_avg"); + + public double OffsetsPerTermAvg => GetDouble("offsets_per_term_avg"); + + public double OffsetBitsPerRecordAvg => GetDouble("offset_bits_per_record_avg"); + + public long HashIndexingFailures => GetLong("hash_indexing_failures"); + + public double TotalIndexingTime => GetDouble("total_indexing_time"); + + public long Indexing => GetLong("indexing"); + + public double PercentIndexed => GetDouble("percent_indexed"); + + public long NumberOfUses => GetLong("number_of_uses"); + + + public Dictionary GcStats => GetRedisResultDictionary("gc_stats"); + + public Dictionary CursorStats => GetRedisResultDictionary("cursor_stats"); + + public InfoResult(RedisResult result) + { + var results = (RedisResult[])result; + + for (var i = 0; i < results.Length; i += 2) + { + var key = (string)results[i]; + var value = results[i + 1]; + + _all.Add(key, value); + } + } + + private string GetString(string key) => _all.TryGetValue(key, out var value) ? (string)value : default; + + private long GetLong(string key) => _all.TryGetValue(key, out var value) ? (long)value : default; + + private double GetDouble(string key) + { + if (_all.TryGetValue(key, out var value)) + { + if ((string)value == "-nan") + { + return default; + } + else + { + return (double)value; + } + } + else + { + return default; + } + } + + private Dictionary GetRedisResultDictionary(string key) + { + if (_all.TryGetValue(key, out var value)) + { + var values = (RedisResult[])value; + var result = new Dictionary(); + + for (var ii = 0; ii < values.Length; ii += 2) + { + result.Add((string)values[ii], values[ii + 1]); + } + + return result; + } + else + { + return default; + } + } + + private Dictionary GetRedisResultsDictionary(string key) + { + if (_all.TryGetValue(key, out var value)) + { + var result = new Dictionary(); + + foreach (RedisResult[] fv in (RedisResult[])value) + { + result.Add((string)fv[0], fv); + } + + return result; + } + else + { + return default; + } + } + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/DataTypes/SearchInformation.cs b/src/NRedisStack/Search/DataTypes/SearchInformation.cs index 3f4f34b2..26c6cb63 100644 --- a/src/NRedisStack/Search/DataTypes/SearchInformation.cs +++ b/src/NRedisStack/Search/DataTypes/SearchInformation.cs @@ -1,34 +1,34 @@ -namespace NRedisStack.Search.DataTypes -{ - /// - /// This class represents the response for SEARCH.INFO command. - /// This object has Read-only properties and cannot be generated outside a SEARCH.INFO response. - /// - public class SearchInformation - { - // TODO: work on it with someone from Search team - // public string IndexName { get; private set; } - // public string[] IndexOptions { get; private set; } - // public long IndexDefinition { get; private set; } - // public long UnmergedNodes { get; private set; } - // public double MergedWeight { get; private set; } - // public double UnmergedWeight { get; private set; } +// namespace NRedisStack.Search.DataTypes +// { +// /// +// /// This class represents the response for SEARCH.INFO command. +// /// This object has Read-only properties and cannot be generated outside a SEARCH.INFO response. +// /// +// public class SearchInformation +// { +// // TODO: work on it with someone from Search team +// // public string IndexName { get; private set; } +// // public string[] IndexOptions { get; private set; } +// // public long IndexDefinition { get; private set; } +// // public long UnmergedNodes { get; private set; } +// // public double MergedWeight { get; private set; } +// // public double UnmergedWeight { get; private set; } - // public long TotalCompressions { get; private set; } +// // public long TotalCompressions { get; private set; } - // internal SearchInformation(long compression, long capacity, long mergedNodes, - // long unmergedNodes, double mergedWeight, - // double unmergedWeight, long totalCompressions) +// // internal SearchInformation(long compression, long capacity, long mergedNodes, +// // long unmergedNodes, double mergedWeight, +// // double unmergedWeight, long totalCompressions) - // { - // Compression = compression; - // Capacity = capacity; - // MergedNodes = mergedNodes; - // UnmergedNodes = unmergedNodes; - // MergedWeight = mergedWeight; - // UnmergedWeight = unmergedWeight; - // TotalCompressions = totalCompressions; - // } - } -} \ No newline at end of file +// // { +// // Compression = compression; +// // Capacity = capacity; +// // MergedNodes = mergedNodes; +// // UnmergedNodes = unmergedNodes; +// // MergedWeight = mergedWeight; +// // UnmergedWeight = unmergedWeight; +// // TotalCompressions = totalCompressions; +// // } +// } +// } \ No newline at end of file diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index 3bddf2eb..2fc4b1e3 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -1,5 +1,6 @@ using NRedisStack.Literals; using NRedisStack.Search; +using NRedisStack.Search.DataTypes; using NRedisStack.Search.FT.CREATE; using StackExchange.Redis; namespace NRedisStack @@ -148,11 +149,26 @@ public async Task AlterAsync(string index, Schema schema, bool skipInitial return (await _db.ExecuteAsync(FT.ALTER, args)).OKtoBoolean(); } - // TODO: finish this & add summary - public RedisResult Info(RedisValue index) - { - return _db.Execute(FT.INFO, index); - } + // /// + // /// Return information and statistics on the index. + // /// + // /// The name of the index. + // /// Dictionary of key and value with information about the index + // /// + // public Dictionary Info(RedisValue index) + // { + // return _db.Execute(FT.INFO, index).ToFtInfoAsDictionary(); + // } + + /// + /// Return information and statistics on the index. + /// + /// The name of the index. + /// Dictionary of key and value with information about the index + /// + public InfoResult Info(RedisValue index) => + new InfoResult(_db.Execute("FT.INFO", index)); + /// /// Create an index with the given specification. diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 8a77cc65..c7948dea 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -227,11 +227,12 @@ public void AlterAdd() SearchResult res2 = ft.Search(index, new Query("@tags:{tagA}")); Assert.Equal(100, res2.TotalResults); - //TODO: complete this test when I finish the command FT.INFO - // var info = ft.Info(index); - // Assert.Equal(index, info.get("index_name")); - // Assert.Equal("identifier", ((List)((List)info.get("attributes")).get(1)).get(0)); - // Assert.Equal("attribute", ((List)((List)info.get("attributes")).get(1)).get(2)); + //TODO: complete this test + var info = ft.Info(index); + Assert.Equal(index, info.IndexName); + //var result = info["attributes"]; + // Assert.Equal("identifier", (info.Attributes[]); + // Assert.Equal("attribute", ((RedisValue[])((RedisValue[])info["attributes"])[1])[2]); } [Fact] @@ -266,8 +267,8 @@ public async Task AlterAddAsync() //TODO: complete this test when I finish the command FT.INFO // var info = ft.Info(index); // Assert.Equal(index, info.get("index_name")); - // Assert.Equal("identifier", ((List)((List)info.get("attributes")).get(1)).get(0)); - // Assert.Equal("attribute", ((List)((List)info.get("attributes")).get(1)).get(2)); + // Assert.Equal("identifier", ((RedisValue[])((RedisValue[])info.get("attributes"))[1])[0]); + // Assert.Equal("attribute", ((RedisValue[])((RedisValue[])info.get("attributes"))[1])[2]); } [Fact] From 6475a3d372fdab077f575f07eae3372f4be32d8f Mon Sep 17 00:00:00 2001 From: shacharPash Date: Mon, 3 Oct 2022 14:34:20 +0300 Subject: [PATCH 16/28] Finish FT.INFO Command --- .../Search/DataTypes/InfoResult.cs | 49 ++++++++++++++++++- src/NRedisStack/Search/SearchCommands.cs | 9 ++++ tests/NRedisStack.Tests/Search/SearchTests.cs | 17 +++---- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/src/NRedisStack/Search/DataTypes/InfoResult.cs b/src/NRedisStack/Search/DataTypes/InfoResult.cs index b43ce81f..1040f440 100644 --- a/src/NRedisStack/Search/DataTypes/InfoResult.cs +++ b/src/NRedisStack/Search/DataTypes/InfoResult.cs @@ -11,7 +11,9 @@ public class InfoResult public Dictionary IndexOption => GetRedisResultDictionary("index_options"); public Dictionary IndexDefinition => GetRedisResultsDictionary("index_definition"); - public Dictionary Attributes => GetRedisResultsDictionary("attributes"); // TODO: check if this is correct + // public Dictionary Attributes => GetRedisResultsDictionary("attributes"); // TODO: check if this is correct + public Dictionary[] Attributes => GetRedisResultDictionaryArray("attributes"); // TODO: check if this is correct + public long NumDocs => GetLong("num_docs"); @@ -138,5 +140,50 @@ private Dictionary GetRedisResultsDictionary(string key) return default; } } + + private Dictionary[] GetRedisResultDictionaryArray(string key) + { + if (_all.TryGetValue(key, out var value)) + { + var values = (RedisResult[])value; + var result = new Dictionary[values.Length]; + for (int i = 0; i < values.Length; i++) + { + var fv = (RedisResult[])values[i]; + var dict = new Dictionary(); + for (int j = 0; j < fv.Length; j += 2) + { + dict.Add((string)fv[j], fv[j + 1]); + } + result[i] = dict; + } + return result; + } + + else + { + return default; + } + } + // private Dictionary[] GetRedisResultsDictionaryTry(string key) + // { + // if (_all.TryGetValue(key, out var value)) + // { + // var result = new List>(); + + // int i = 0; + // foreach (RedisResult[] fv in (RedisResult[])value) + // { + // var res = GetRedisResultDictionary((string)fv[i++]); + // result.Add(res); + // } + + // return result.ToArray(); + // } + // else + // { + // return default; + // } + // } } } \ No newline at end of file diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index 2fc4b1e3..e50d7afa 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -169,6 +169,15 @@ public async Task AlterAsync(string index, Schema schema, bool skipInitial public InfoResult Info(RedisValue index) => new InfoResult(_db.Execute("FT.INFO", index)); + /// + /// Return information and statistics on the index. + /// + /// The name of the index. + /// Dictionary of key and value with information about the index + /// + public async Task InfoAsync(RedisValue index) => + new InfoResult(await _db.ExecuteAsync("FT.INFO", index)); + /// /// Create an index with the given specification. diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index c7948dea..f129d49a 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -227,12 +227,11 @@ public void AlterAdd() SearchResult res2 = ft.Search(index, new Query("@tags:{tagA}")); Assert.Equal(100, res2.TotalResults); - //TODO: complete this test var info = ft.Info(index); Assert.Equal(index, info.IndexName); - //var result = info["attributes"]; - // Assert.Equal("identifier", (info.Attributes[]); - // Assert.Equal("attribute", ((RedisValue[])((RedisValue[])info["attributes"])[1])[2]); + Assert.Equal("title", (info.Attributes[0]["identifier"]).ToString()); + Assert.Equal("TAG", (info.Attributes[1]["type"]).ToString()); + Assert.Equal("name", (info.Attributes[2]["attribute"]).ToString()); } [Fact] @@ -264,11 +263,11 @@ public async Task AlterAddAsync() SearchResult res2 = ft.Search(index, new Query("@tags:{tagA}")); Assert.Equal(100, res2.TotalResults); - //TODO: complete this test when I finish the command FT.INFO - // var info = ft.Info(index); - // Assert.Equal(index, info.get("index_name")); - // Assert.Equal("identifier", ((RedisValue[])((RedisValue[])info.get("attributes"))[1])[0]); - // Assert.Equal("attribute", ((RedisValue[])((RedisValue[])info.get("attributes"))[1])[2]); + var info = await ft.InfoAsync(index); + Assert.Equal(index, info.IndexName); + Assert.Equal("title", (info.Attributes[0]["identifier"]).ToString()); + Assert.Equal("TAG", (info.Attributes[1]["type"]).ToString()); + Assert.Equal("name", (info.Attributes[2]["attribute"]).ToString()); } [Fact] From 66491b1d60ecc1bed9f081b51a419e800f257459 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Mon, 3 Oct 2022 18:19:09 +0300 Subject: [PATCH 17/28] Add AggregationRequest Class for FT.AGGREGATE Command --- src/NRedisStack/Search/AggregationRequest.cs | 217 ++++++++++++++++++ .../Search/FT.CREATE/FTCreateParams.cs | 1 - src/NRedisStack/Search/Group.cs | 54 +++++ src/NRedisStack/Search/Limit.cs | 33 +++ src/NRedisStack/Search/Reducer.cs | 85 +++++++ src/NRedisStack/Search/SortedField.cs | 32 +++ 6 files changed, 421 insertions(+), 1 deletion(-) create mode 100644 src/NRedisStack/Search/AggregationRequest.cs create mode 100644 src/NRedisStack/Search/Group.cs create mode 100644 src/NRedisStack/Search/Limit.cs create mode 100644 src/NRedisStack/Search/Reducer.cs create mode 100644 src/NRedisStack/Search/SortedField.cs diff --git a/src/NRedisStack/Search/AggregationRequest.cs b/src/NRedisStack/Search/AggregationRequest.cs new file mode 100644 index 00000000..ed781bcd --- /dev/null +++ b/src/NRedisStack/Search/AggregationRequest.cs @@ -0,0 +1,217 @@ +using System; +using System.Collections.Generic; +using System.Text; +// using NRediSearch.Aggregation.Reducers; +using StackExchange.Redis; + +namespace NRedisStack.Search.Aggregation +{ + public class AggregationRequest + { + private List args = new List(); // Check if Readonly + private bool isWithCursor = false; + + public AggregationRequest(string query) + { + args.Add(query); + } + + public AggregationRequest() : this("*") { } + + // public AggregationRequest load(params string[] fields) + // { + // return load(FieldName.Convert(fields)); + // } + + public AggregationRequest Load(params FieldName[] fields) + { + args.Add("LOAD"); + int loadCountIndex = args.Count(); + args.Add(null); + int loadCount = 0; + foreach (FieldName fn in fields) + { + loadCount += fn.AddCommandArguments(args); + } + args.Insert(loadCountIndex, loadCount.ToString()); + return this; + } + + public AggregationRequest LoadAll() + { + args.Add("LOAD"); + args.Add("*"); + return this; + } + + public AggregationRequest Limit(int offset, int count) + { + Limit limit = new Limit(offset, count); + limit.SerializeRedisArgs(args); + return this; + } + + public AggregationRequest Limit(int count) + { + return Limit(0, count); + } + + public AggregationRequest SortBy(params SortedField[] Fields) + { + args.Add("SORTBY"); + args.Add(Fields.Length * 2); + foreach (SortedField field in Fields) + { + args.Add(field.FieldName); + args.Add(field.Order); + } + + return this; + } + + public AggregationRequest SortBy(int max, params SortedField[] Fields) + { + SortBy(Fields); + if (max > 0) + { + args.Add("MAX"); + args.Add(max); + } + return this; + } + + public AggregationRequest SortByAsc(string field) + { + return SortBy(SortedField.Asc(field)); + } + + public AggregationRequest SortByDesc(string field) + { + return SortBy(SortedField.Desc(field)); + } + + public AggregationRequest Apply(string projection, string alias) + { + args.Add("APPLY"); + args.Add(projection); + args.Add("AS"); + args.Add(alias); + return this; + } + + public AggregationRequest GroupBy(IList fields, IList reducers) + { + // string[] fieldsArr = new string[fields.size()]; + Group g = new Group(fields); + foreach (Reducer r in reducers) + { + g.Reduce(r); + } + GroupBy(g); + return this; + } + + public AggregationRequest GroupBy(string field, params Reducer[] reducers) + { + return GroupBy(new string[] { field }, reducers); + } + + public AggregationRequest GroupBy(Group group) + { + args.Add("GROUPBY"); + group.SerializeRedisArgs(args); + return this; + } + + public AggregationRequest Filter(string expression) + { + args.Add("FILTER"); + args.Add(expression); + return this; + } + + public AggregationRequest Cursor(int count, long maxIdle) + { + isWithCursor = true; + if (count > 0) + { + args.Add("WITHCURSOR"); + args.Add("COUNT"); + args.Add(count); + if (maxIdle < long.MaxValue && maxIdle >= 0) + { + args.Add("MAXIDLE"); + args.Add(maxIdle); + } + } + return this; + } + + public AggregationRequest Verbatim() + { + args.Add("VERBATIM"); + return this; + } + + public AggregationRequest Timeout(long timeout) + { + if (timeout >= 0) + { + args.Add("TIMEOUT"); + args.Add(timeout); + } + return this; + } + + public AggregationRequest Params(Dictionary nameValue) + { + if (nameValue.Count >= 1) + { + args.Add("PARAMS"); + args.Add(nameValue.Count * 2); + foreach (var entry in nameValue) + { + args.Add(entry.Key); + args.Add(entry.Value); + } + } + + return this; + } + + public AggregationRequest Dialect(int dialect) + { + args.Add("DIALECT"); + args.Add(dialect); + return this; + } + + public List GetArgs() + { + return args; + } + + public void SerializeRedisArgs(List redisArgs) + { + foreach (var s in GetArgs()) + { + redisArgs.Add(s); + } + } + + // public string getArgsstring() + // { + // StringBuilder sj = new StringBuilder(" "); + // foreach (var s in GetArgs()) + // { + // sj.Add(s.ToString()); + // } + // return sj.tostring(); + // } + + public bool IsWithCursor() + { + return isWithCursor; + } + } +} diff --git a/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs b/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs index 895c4cb6..f5573675 100644 --- a/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs +++ b/src/NRedisStack/Search/FT.CREATE/FTCreateParams.cs @@ -1,7 +1,6 @@ using NRedisStack.Extensions; using NRedisStack.Literals; using NRedisStack.Literals.Enums; -// TODO: look at NRediSearch namespace NRedisStack.Search.FT.CREATE { public class FTCreateParams diff --git a/src/NRedisStack/Search/Group.cs b/src/NRedisStack/Search/Group.cs new file mode 100644 index 00000000..36bca902 --- /dev/null +++ b/src/NRedisStack/Search/Group.cs @@ -0,0 +1,54 @@ +using System.Collections.Generic; + +namespace NRedisStack.Search.Aggregation +{ + public class Group + { + + private readonly IList _reducers = new List(); + private readonly IList _fields; + private Limit _limit = new Limit(0, 0); + + public Group(params string[] fields) => _fields = fields; + public Group(IList fields) => _fields = fields; + + internal Group Limit(Limit limit) + { + _limit = limit; + return this; + } + + internal Group Reduce(Reducer r) + { + _reducers.Add(r); + return this; + } + + internal void SerializeRedisArgs(List args) + { + args.Add(_fields.Count); + foreach (var field in _fields) + args.Add(field); + foreach (var r in _reducers) + { + args.Add("REDUCE"); + args.Add(r.Name); + r.SerializeRedisArgs(args); + var alias = r.Alias; + if (!string.IsNullOrEmpty(alias)) + { + args.Add("AS"); + args.Add(alias); + } + } + _limit.SerializeRedisArgs(args); + } + + public List getArgs() + { + List args = new List(); + SerializeRedisArgs(args); + return args; + } + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/Limit.cs b/src/NRedisStack/Search/Limit.cs new file mode 100644 index 00000000..84297c0e --- /dev/null +++ b/src/NRedisStack/Search/Limit.cs @@ -0,0 +1,33 @@ +using System.Collections.Generic; + +namespace NRedisStack.Search.Aggregation +{ + internal readonly struct Limit + { + public static Limit NO_LIMIT = new Limit(0, 0); + private readonly int _offset, _count; + + public Limit(int offset, int count) + { + _offset = offset; + _count = count; + } + +// public void addArgs(List args) { +// if (count == 0) { +// return; +// } +// args.add("LIMIT"); +// args.add(Integer.toString(offset)); +// args.add(Integer.toString(count)); +// } + + internal void SerializeRedisArgs(List args) + { + if (_count == 0) return; + args.Add("LIMIT"); + args.Add(_offset); + args.Add(_count); + } + } +} diff --git a/src/NRedisStack/Search/Reducer.cs b/src/NRedisStack/Search/Reducer.cs new file mode 100644 index 00000000..68294250 --- /dev/null +++ b/src/NRedisStack/Search/Reducer.cs @@ -0,0 +1,85 @@ +using System.Collections.Generic; + +namespace NRedisStack.Search.Aggregation +{ + public abstract class Reducer + { + + public override string ToString() => Name; + + // internal Reducer(string field) => _field = field; + + /// + /// The name of the reducer + /// + public abstract string Name { get; } + + public string? Alias { get; set; } + private readonly string _field; + + + protected Reducer(string field) + { + _field = field; + Alias = null; + } + + //protected Reducer() : this(field: null) { } + + protected virtual int GetOwnArgsCount() => _field == null ? 0 : 1; + protected virtual void AddOwnArgs(List args) + { + if (_field != null) args.Add(_field); + } + + /** + * @return The name of the reducer + */ + // public abstract string getName(); + + // public string getAlias() + // { + // return Alias; + // } + + // public Reducer setAlias(string alias) + // { + // this.Alias = alias; + // return this; + // } + + // public final Reducer as(string alias) { + // return setAlias(alias); + // } + + public Reducer As(string alias) + { + Alias = alias; + return this; + } + public Reducer SetAliasAsField() + { + if (string.IsNullOrEmpty(_field)) throw new InvalidOperationException("Cannot set to field name since no field exists"); + return As(_field); + } + + internal void SerializeRedisArgs(List args) + { + int count = GetOwnArgsCount(); + args.Add(count); + int before = args.Count; + AddOwnArgs(args); + int after = args.Count; + if (count != (after - before)) + throw new InvalidOperationException($"Reducer '{ToString()}' incorrectly reported the arg-count as {count}, but added {after - before}"); + } + + public List GetArgs() + { + List args = new List(); + SerializeRedisArgs(args); + return args; + } +} + +} \ No newline at end of file diff --git a/src/NRedisStack/Search/SortedField.cs b/src/NRedisStack/Search/SortedField.cs new file mode 100644 index 00000000..6aa3022a --- /dev/null +++ b/src/NRedisStack/Search/SortedField.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; + +namespace NRedisStack.Search.Aggregation +{ + public class SortedField + { + + public enum SortOrder + { + ASC, DESC + } + + public string FieldName { get; } + public SortOrder Order { get; } + + public SortedField(String fieldName, SortOrder order) + { + this.FieldName = fieldName; + this.Order = order; + } + + public static SortedField Asc(String field) + { + return new SortedField(field, SortOrder.ASC); + } + + public static SortedField Desc(String field) + { + return new SortedField(field, SortOrder.DESC); + } + } +} \ No newline at end of file From 0368e23f91a642d0690041e4d7c7a5df7c7a313b Mon Sep 17 00:00:00 2001 From: shacharPash Date: Sun, 9 Oct 2022 11:09:19 +0300 Subject: [PATCH 18/28] Add FT.AGGREGATE Command + Tests --- src/NRedisStack/Search/AggregationResult.cs | 48 +++ src/NRedisStack/Search/Reducers.cs | 103 +++++++ src/NRedisStack/Search/Row.cs | 22 ++ src/NRedisStack/Search/SearchCommands.cs | 57 +++- tests/NRedisStack.Tests/Search/SearchTests.cs | 287 +++++++++++++++++- 5 files changed, 514 insertions(+), 3 deletions(-) create mode 100644 src/NRedisStack/Search/AggregationResult.cs create mode 100644 src/NRedisStack/Search/Reducers.cs create mode 100644 src/NRedisStack/Search/Row.cs diff --git a/src/NRedisStack/Search/AggregationResult.cs b/src/NRedisStack/Search/AggregationResult.cs new file mode 100644 index 00000000..03545895 --- /dev/null +++ b/src/NRedisStack/Search/AggregationResult.cs @@ -0,0 +1,48 @@ +using System.Collections.Generic; +using StackExchange.Redis; + +namespace NRedisStack.Search.Aggregation +{ + public sealed class AggregationResult + { + public long TotalResults { get; } + private readonly Dictionary[] _results; + public long CursorId { get; } + + + internal AggregationResult(RedisResult result, long cursorId = -1) + { + var arr = (RedisResult[])result; + + // the first element is always the number of results + TotalResults = (long)arr[0]; + + _results = new Dictionary[arr.Length - 1]; + for (int i = 1; i < arr.Length; i++) + { + var raw = (RedisResult[])arr[i]; + var cur = new Dictionary(); + for (int j = 0; j < raw.Length;) + { + var key = (string)raw[j++]; + var val = raw[j++]; + if (val.Type != ResultType.MultiBulk) + cur.Add(key, (RedisValue)val); + } + _results[i - 1] = cur; + } + + CursorId = cursorId; + } + public IReadOnlyList> GetResults() => _results; + + public Dictionary this[int index] + => index >= _results.Length ? null : _results[index]; + + public Row GetRow(int index) + { + if (index >= _results.Length) return default; + return new Row(_results[index]); + } + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/Reducers.cs b/src/NRedisStack/Search/Reducers.cs new file mode 100644 index 00000000..6e8254dd --- /dev/null +++ b/src/NRedisStack/Search/Reducers.cs @@ -0,0 +1,103 @@ +using System.Collections.Generic; +using NRedisStack.Search.Aggregation; + +namespace NRedisStack.Search.Aggregation +{ + public static class Reducers + { + public static Reducer Count() => CountReducer.Instance; + private sealed class CountReducer : Reducer + { + internal static readonly Reducer Instance = new CountReducer(); + private CountReducer() : base(null) { } + public override string Name => "COUNT"; + } + + private sealed class SingleFieldReducer : Reducer + { + public override string Name { get; } + + internal SingleFieldReducer(string name, string field) : base(field) + { + Name = name; + } + } + + public static Reducer CountDistinct(string field) => new SingleFieldReducer("COUNT_DISTINCT", field); + + public static Reducer CountDistinctish(string field) => new SingleFieldReducer("COUNT_DISTINCTISH", field); + + public static Reducer Sum(string field) => new SingleFieldReducer("SUM", field); + + public static Reducer Min(string field) => new SingleFieldReducer("MIN", field); + + public static Reducer Max(string field) => new SingleFieldReducer("MAX", field); + + public static Reducer Avg(string field) => new SingleFieldReducer("AVG", field); + + public static Reducer StdDev(string field) => new SingleFieldReducer("STDDEV", field); + + public static Reducer Quantile(string field, double percentile) => new QuantileReducer(field, percentile); + + private sealed class QuantileReducer : Reducer + { + private readonly double _percentile; + public QuantileReducer(string field, double percentile) : base(field) + { + _percentile = percentile; + } + protected override int GetOwnArgsCount() => base.GetOwnArgsCount() + 1; + protected override void AddOwnArgs(List args) + { + base.AddOwnArgs(args); + args.Add(_percentile); + } + public override string Name => "QUANTILE"; + } + public static Reducer FirstValue(string field, SortedField sortBy) => new FirstValueReducer(field, sortBy); + private sealed class FirstValueReducer : Reducer + { + private readonly SortedField? _sortBy; + public FirstValueReducer(string field, SortedField? sortBy) : base(field) + { + _sortBy = sortBy; + } + public override string Name => "FIRST_VALUE"; + + // TODO: Check if needed + // protected override int GetOwnArgsCount() => base.GetOwnArgsCount() + (_sortBy.HasValue ? 3 : 0); + protected override void AddOwnArgs(List args) + { + base.AddOwnArgs(args); + if (_sortBy != null) + { + var sortBy = _sortBy; + args.Add("BY"); + args.Add(sortBy.FieldName); + args.Add(sortBy.Order.ToString()); + } + } + } + public static Reducer FirstValue(string field) => new FirstValueReducer(field, null); + + public static Reducer ToList(string field) => new SingleFieldReducer("TOLIST", field); + + public static Reducer RandomSample(string field, int size) => new RandomSampleReducer(field, size); + + private sealed class RandomSampleReducer : Reducer + { + private readonly int _size; + public RandomSampleReducer(string field, int size) : base(field) + { + _size = size; + } + public override string Name => "RANDOM_SAMPLE"; + protected override int GetOwnArgsCount() => base.GetOwnArgsCount() + 1; + protected override void AddOwnArgs(List args) + { + base.AddOwnArgs(args); + args.Add(_size); + } + } + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/Row.cs b/src/NRedisStack/Search/Row.cs new file mode 100644 index 00000000..2a8a8aa9 --- /dev/null +++ b/src/NRedisStack/Search/Row.cs @@ -0,0 +1,22 @@ +using System.Collections.Generic; +using StackExchange.Redis; + +namespace NRedisStack.Search.Aggregation +{ + public readonly struct Row + { + private readonly Dictionary _fields; + + internal Row(Dictionary fields) + { + _fields = fields; + } + + public bool ContainsKey(string key) => _fields.ContainsKey(key); + public RedisValue this[string key] => _fields.TryGetValue(key, out var result) ? result : RedisValue.Null; + + public string GetString(string key) => _fields.TryGetValue(key, out var result) ? (string)result : default; + public long GetLong(string key) => _fields.TryGetValue(key, out var result) ? (long)result : default; + public double GetDouble(string key) => _fields.TryGetValue(key, out var result) ? (double)result : default; + } +} \ No newline at end of file diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index e50d7afa..bc91a304 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -1,5 +1,6 @@ using NRedisStack.Literals; using NRedisStack.Search; +using NRedisStack.Search.Aggregation; using NRedisStack.Search.DataTypes; using NRedisStack.Search.FT.CREATE; using StackExchange.Redis; @@ -33,7 +34,61 @@ public async Task _ListAsync() return (await _db.ExecuteAsync(FT._LIST)).ToArray(); } - // TODO: Aggregate + /// + /// Run a search query on an index, and perform aggregate transformations on the results. + /// + /// The index name. + /// The query + /// An object + /// + public AggregationResult Aggregate(string index, AggregationRequest query) + { + List args = new List { index }; + //query.SerializeRedisArgs(args); + foreach(var arg in query.GetArgs()) + { + args.Add(arg); + } + var result = _db.Execute(FT.AGGREGATE, args); + if (query.IsWithCursor()) + { + var results = (RedisResult[])result; + + return new AggregationResult(results[0], (long)results[1]); + } + else + { + return new AggregationResult(result); + } + } + + /// + /// Run a search query on an index, and perform aggregate transformations on the results. + /// + /// The index name. + /// The query + /// An object + /// + public async Task AggregateAsync(string index, AggregationRequest query) + { + List args = new List { index }; + //query.SerializeRedisArgs(args); + foreach(var arg in query.GetArgs()) + { + args.Add(arg); + } + var result = await _db.ExecuteAsync(FT.AGGREGATE, args); + if (query.IsWithCursor()) + { + var results = (RedisResult[])result; + + return new AggregationResult(results[0], (long)results[1]); + } + else + { + return new AggregationResult(result); + } + } /// /// Add an alias to an index. diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index f129d49a..aefde4c5 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -5,21 +5,304 @@ using NRedisStack.Search.FT.CREATE; using NRedisStack.Search; using static NRedisStack.Search.Schema; +using NRedisStack.Search.Aggregation; namespace NRedisStack.Tests.Search; public class SearchTests : AbstractNRedisStackTest, IDisposable { Mock _mock = new Mock(); - private readonly string key = "SEARCH_TESTS"; + // private readonly string key = "SEARCH_TESTS"; private readonly string index = "TEST_INDEX"; public SearchTests(RedisFixture redisFixture) : base(redisFixture) { } public void Dispose() { - redisFixture.Redis.GetDatabase().KeyDelete(key); + redisFixture.Redis.GetDatabase().KeyDelete(index); } + private void AddDocument(IDatabase db, Document doc) + { + string key = doc.Id; + var properties = doc.GetProperties(); + // HashEntry[] hash = new HashEntry[properties.Count()]; + // for(int i = 0; i < properties.Count(); i++) + // { + // var property = properties.ElementAt(i); + // hash[i] = new HashEntry(property.Key, property.Value); + // } + // db.HashSet(key, hash); + var nameValue = new List() { key }; + foreach (var item in properties) + { + nameValue.Add(item.Key); + nameValue.Add(item.Value); + } + db.Execute("HSET", nameValue); + + } + + private void AddDocument(IDatabase db, string key, Dictionary objDictionary) + { + Dictionary strDictionary = new Dictionary(); + // HashEntry[] hash = new HashEntry[objDictionary.Count()]; + // for(int i = 0; i < objDictionary.Count(); i++) + // { + // var property = objDictionary.ElementAt(i); + // hash[i] = new HashEntry(property.Key, property.Value.ToString()); + // } + // db.HashSet(key, hash); + var nameValue = new List(); + foreach (var item in objDictionary) + { + nameValue.Add(item.Key); + nameValue.Add(item.Value); + } + db.Execute("HSET", nameValue); + } + + [Fact] + public void TestAggregationRequestVerbatim() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema(); + sc.AddTextField("name", 1.0, sortable: true); + ft.Create(index, FTCreateParams.CreateParams(), sc); + AddDocument(db, new Document("data1").Set("name", "hello kitty")); + + AggregationRequest r = new AggregationRequest("kitti"); + + AggregationResult res = ft.Aggregate(index, r); + Assert.Equal(1, res.TotalResults); + + r = new AggregationRequest("kitti") + .Verbatim(); + + res = ft.Aggregate(index, r); + Assert.Equal(0, res.TotalResults); + } + + [Fact] + public async Task TestAggregationRequestVerbatimAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema(); + sc.AddTextField("name", 1.0, sortable: true); + ft.Create(index, FTCreateParams.CreateParams(), sc); + AddDocument(db, new Document("data1").Set("name", "hello kitty")); + + AggregationRequest r = new AggregationRequest("kitti"); + + AggregationResult res = await ft.AggregateAsync(index, r); + Assert.Equal(1, res.TotalResults); + + r = new AggregationRequest("kitti") + .Verbatim(); + + res = await ft.AggregateAsync(index, r); + Assert.Equal(0, res.TotalResults); + } + + [Fact] + public void TestAggregationRequestTimeout() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema(); + sc.AddTextField("name", 1.0, sortable: true); + sc.AddNumericField("count", sortable: true); + ft.Create(index, FTCreateParams.CreateParams(), sc); + AddDocument(db, new Document("data1").Set("name", "abc").Set("count", 10)); + AddDocument(db, new Document("data2").Set("name", "def").Set("count", 5)); + AddDocument(db, new Document("data3").Set("name", "def").Set("count", 25)); + + AggregationRequest r = new AggregationRequest() + .GroupBy("@name", Reducers.Sum("@count").As("sum")) + .Timeout(5000); + + AggregationResult res = ft.Aggregate(index, r); + Assert.Equal(2, res.TotalResults); + } + + [Fact] + public async Task TestAggregationRequestTimeoutAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema(); + sc.AddTextField("name", 1.0, sortable: true); + sc.AddNumericField("count", sortable: true); + ft.Create(index, FTCreateParams.CreateParams(), sc); + AddDocument(db, new Document("data1").Set("name", "abc").Set("count", 10)); + AddDocument(db, new Document("data2").Set("name", "def").Set("count", 5)); + AddDocument(db, new Document("data3").Set("name", "def").Set("count", 25)); + + AggregationRequest r = new AggregationRequest() + .GroupBy("@name", Reducers.Sum("@count").As("sum")) + .Timeout(5000); + + AggregationResult res = await ft.AggregateAsync(index, r); + Assert.Equal(2, res.TotalResults); + } + + // // TODO: underastant why its not working + // [Fact] + // public void TestAggregations() + // { + // IDatabase db = redisFixture.Redis.GetDatabase(); + // db.Execute("FLUSHALL"); + // var ft = db.FT(); + // Schema sc = new Schema(); + // sc.AddTextField("name", 1.0, true); + // sc.AddNumericField("count", true); + // ft.Create(index, FTCreateParams.CreateParams(), sc); + // // client.AddDocument(new Document("data1").Set("name", "abc").Set("count", 10)); + // // client.AddDocument(new Document("data2").Set("name", "def").Set("count", 5)); + // // client.AddDocument(new Document("data3").Set("name", "def").Set("count", 25)); + // AddDocument(db, new Document("data1").Set("name", "abc").Set("count", 10)); + // AddDocument(db, new Document("data2").Set("name", "def").Set("count", 5)); + // AddDocument(db, new Document("data3").Set("name", "def").Set("count", 25)); + + // AggregationRequest r = new AggregationRequest() + // .GroupBy("@name", Reducers.Sum("@count").As ("sum")) + // .SortBy(10, SortedField.Desc("@sum")); + + // // actual search + // // var resBefore = db.Execute("FT.AGGREGATE", index, "*", "GROUPBY", "1", "@name", "REDUCE", "SUM", "1", "@count", "AS", "sum", "SORTBY", "2", "@sum", "DESC", "MAX", "10"); + // //var res = new AggregationResult(resBefore); + + // var res = ft.Aggregate(index, r); + // Assert.Equal(2, res.TotalResults); + + // Row r1 = res.GetRow(0); + // Assert.NotNull(r1); + // Assert.Equal("def", r1.GetString("name")); + // Assert.Equal(30, r1.GetLong("sum")); + // Assert.Equal(30, r1.GetDouble("sum"), 0); + + // Assert.Equal(0L, r1.GetLong("nosuchcol")); + // Assert.Equal(0.0, r1.GetDouble("nosuchcol"), 0); + // Assert.Equal("", r1.GetString("nosuchcol")); + + // Row r2 = res.GetRow(1); + // Assert.NotNull(r2); + // Assert.Equal("abc", r2.GetString("name")); + // Assert.Equal(10, r2.GetLong("sum")); + // } + + [Fact] + public void TestAggregationRequestParamsDialect() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema(); + sc.AddTextField("name", 1.0, sortable: true); + sc.AddNumericField("count", sortable: true); + ft.Create(index, FTCreateParams.CreateParams(), sc); + AddDocument(db, new Document("data1").Set("name", "abc").Set("count", 10)); + AddDocument(db, new Document("data2").Set("name", "def").Set("count", 5)); + AddDocument(db, new Document("data3").Set("name", "def").Set("count", 25)); + + Dictionary parameters = new Dictionary(); + parameters.Add("name", "abc"); + + AggregationRequest r = new AggregationRequest("$name") + .GroupBy("@name", Reducers.Sum("@count").As("sum")) + .Params(parameters) + .Dialect(2); // From documentation - To use PARAMS, DIALECT must be set to 2 + + AggregationResult res = ft.Aggregate(index, r); + Assert.Equal(1, res.TotalResults); + + Row r1 = res.GetRow(0); + Assert.NotNull(r1); + Assert.Equal("abc", r1.GetString("name")); + Assert.Equal(10, r1.GetLong("sum")); + } + + [Fact] + public async Task TestAggregationRequestParamsDialectAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema(); + sc.AddTextField("name", 1.0, sortable: true); + sc.AddNumericField("count", sortable: true); + ft.Create(index, FTCreateParams.CreateParams(), sc); + AddDocument(db, new Document("data1").Set("name", "abc").Set("count", 10)); + AddDocument(db, new Document("data2").Set("name", "def").Set("count", 5)); + AddDocument(db, new Document("data3").Set("name", "def").Set("count", 25)); + + Dictionary parameters = new Dictionary(); + parameters.Add("name", "abc"); + + AggregationRequest r = new AggregationRequest("$name") + .GroupBy("@name", Reducers.Sum("@count").As("sum")) + .Params(parameters) + .Dialect(2); // From documentation - To use PARAMS, DIALECT must be set to 2 + + AggregationResult res = await ft.AggregateAsync(index, r); + Assert.Equal(1, res.TotalResults); + + Row r1 = res.GetRow(0); + Assert.NotNull(r1); + Assert.Equal("abc", r1.GetString("name")); + Assert.Equal(10, r1.GetLong("sum")); + } + + // // TODO: underastant why its not working + // [Fact] + // public void TestApplyAndFilterAggregations() + // { + // IDatabase db = redisFixture.Redis.GetDatabase(); + // db.Execute("FLUSHALL"); + // var ft = db.FT(); + // Schema sc = new Schema(); + // sc.AddTextField("name", 1.0, sortable: true); + // sc.AddNumericField("subj1", sortable: true); + // sc.AddNumericField("subj2", sortable: true); + // ft.Create(index, FTCreateParams.CreateParams(), sc); + // // client.AddDocument(db, new Document("data1").Set("name", "abc").Set("subj1", 20).Set("subj2", 70)); + // // client.AddDocument(db, new Document("data2").Set("name", "def").Set("subj1", 60).Set("subj2", 40)); + // // client.AddDocument(db, new Document("data3").Set("name", "ghi").Set("subj1", 50).Set("subj2", 80)); + // // client.AddDocument(db, new Document("data4").Set("name", "abc").Set("subj1", 30).Set("subj2", 20)); + // // client.AddDocument(db, new Document("data5").Set("name", "def").Set("subj1", 65).Set("subj2", 45)); + // // client.AddDocument(db, new Document("data6").Set("name", "ghi").Set("subj1", 70).Set("subj2", 70)); + // AddDocument(db, new Document("data1").Set("name", "abc").Set("subj1", 20).Set("subj2", 70)); + // AddDocument(db, new Document("data2").Set("name", "def").Set("subj1", 60).Set("subj2", 40)); + // AddDocument(db, new Document("data3").Set("name", "ghi").Set("subj1", 50).Set("subj2", 80)); + // AddDocument(db, new Document("data4").Set("name", "abc").Set("subj1", 30).Set("subj2", 20)); + // AddDocument(db, new Document("data5").Set("name", "def").Set("subj1", 65).Set("subj2", 45)); + // AddDocument(db, new Document("data6").Set("name", "ghi").Set("subj1", 70).Set("subj2", 70)); + + // AggregationRequest r = new AggregationRequest().Apply("(@subj1+@subj2)/2", "attemptavg") + // .GroupBy("@name", Reducers.Avg("@attemptavg").As("avgscore")) + // .Filter("@avgscore>=50") + // .SortBy(10, SortedField.Asc("@name")); + + // // actual search + // AggregationResult res = ft.Aggregate(index, r); + // Assert.Equal(3, res.TotalResults); + + // Row r1 = res.GetRow(0); + // Assert.NotNull(r1); + // Assert.Equal("def", r1.GetString("name")); + // Assert.Equal(52.5, r1.GetDouble("avgscore"), 0); + + // Row r2 = res.GetRow(1); + // Assert.NotNull(r2); + // Assert.Equal("ghi", r2.GetString("name")); + // Assert.Equal(67.5, r2.GetDouble("avgscore"), 0); + // } + [Fact] public void TestCreate() { From cba57f0bd9f20295fca0850064416e579eb391ed Mon Sep 17 00:00:00 2001 From: shacharPash Date: Sun, 9 Oct 2022 11:23:04 +0300 Subject: [PATCH 19/28] Adding TODOs to commands that have not yet been implemented --- src/NRedisStack/Search/SearchCommands.cs | 76 +++++++++++++++--------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index bc91a304..786e4f1a 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -204,35 +204,8 @@ public async Task AlterAsync(string index, Schema schema, bool skipInitial return (await _db.ExecuteAsync(FT.ALTER, args)).OKtoBoolean(); } - // /// - // /// Return information and statistics on the index. - // /// - // /// The name of the index. - // /// Dictionary of key and value with information about the index - // /// - // public Dictionary Info(RedisValue index) - // { - // return _db.Execute(FT.INFO, index).ToFtInfoAsDictionary(); - // } - - /// - /// Return information and statistics on the index. - /// - /// The name of the index. - /// Dictionary of key and value with information about the index - /// - public InfoResult Info(RedisValue index) => - new InfoResult(_db.Execute("FT.INFO", index)); - - /// - /// Return information and statistics on the index. - /// - /// The name of the index. - /// Dictionary of key and value with information about the index - /// - public async Task InfoAsync(RedisValue index) => - new InfoResult(await _db.ExecuteAsync("FT.INFO", index)); - + // TODO: FT.CONFIG_GET + // TODO: FT.CONFIG_SET /// /// Create an index with the given specification. @@ -277,6 +250,46 @@ public async Task CreateAsync(string indexName, FTCreateParams parameters, return (await _db.ExecuteAsync(FT.CREATE, args)).OKtoBoolean(); } + // TODO: FT.CURSOR DEL + // TODO: FT.CURSOR READ + // TODO: FT.DICTADD + // TODO: FT.DICTDEL + // TODO: FT.DICTDUMP + // TODO: FT.DROPINDEX + // TODO: FT.EXPLAIN + // TODO: FT.EXPLAINCLI + + // /// + // /// Return information and statistics on the index. + // /// + // /// The name of the index. + // /// Dictionary of key and value with information about the index + // /// + // public Dictionary Info(RedisValue index) + // { + // return _db.Execute(FT.INFO, index).ToFtInfoAsDictionary(); + // } + + /// + /// Return information and statistics on the index. + /// + /// The name of the index. + /// Dictionary of key and value with information about the index + /// + public InfoResult Info(RedisValue index) => + new InfoResult(_db.Execute("FT.INFO", index)); + + /// + /// Return information and statistics on the index. + /// + /// The name of the index. + /// Dictionary of key and value with information about the index + /// + public async Task InfoAsync(RedisValue index) => + new InfoResult(await _db.ExecuteAsync("FT.INFO", index)); + + // TODO: FT.PROFILE + /// /// Search the index /// @@ -308,5 +321,10 @@ public async Task SearchAsync(string indexName, Query q) var resp = (await _db.ExecuteAsync("FT.SEARCH", args)).ToArray(); return new SearchResult(resp, !q.NoContent, q.WithScores, q.WithPayloads, q.ExplainScore); } + + // TODO: FT.SPELLCHECK + // TODO: FT.SYNDUMP + // TODO: FT.SYNUPDATE + // TODO: FT.TAGVALS } } \ No newline at end of file From fc5c9a37a37e5cea5265040a4074d9c7cd62e367 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Sun, 9 Oct 2022 15:19:40 +0300 Subject: [PATCH 20/28] Add FT.CONFIG GET/SET Commands + Tests --- src/NRedisStack/ResponseParser.cs | 12 +++ src/NRedisStack/Search/Literals/Commands.cs | 1 + src/NRedisStack/Search/SearchCommands.cs | 48 +++++++++- tests/NRedisStack.Tests/Search/SearchTests.cs | 90 +++++++++++++++++++ 4 files changed, 149 insertions(+), 2 deletions(-) diff --git a/src/NRedisStack/ResponseParser.cs b/src/NRedisStack/ResponseParser.cs index 5700191b..6f161b0e 100644 --- a/src/NRedisStack/ResponseParser.cs +++ b/src/NRedisStack/ResponseParser.cs @@ -500,6 +500,18 @@ public static Dictionary ToFtInfoAsDictionary(this RedisResu return info; } + public static Dictionary ToConfigDictionary(this RedisResult value) + { + var res = (RedisResult[])value; + var dict = new Dictionary(); + foreach (var pair in res) + { + var arr = (RedisResult[])pair; + dict.Add(arr[0].ToString(), arr[1].ToString()); + } + return dict; + } + public static IReadOnlyList ToTimeSeriesChunkArray(this RedisResult result) { RedisResult[] redisResults = (RedisResult[])result; diff --git a/src/NRedisStack/Search/Literals/Commands.cs b/src/NRedisStack/Search/Literals/Commands.cs index 505ad479..d88f7892 100644 --- a/src/NRedisStack/Search/Literals/Commands.cs +++ b/src/NRedisStack/Search/Literals/Commands.cs @@ -8,6 +8,7 @@ internal class FT public const string ALIASDEL = "FT.ALIASDEL"; public const string ALIASUPDATE = "FT.ALIASUPDATE"; public const string ALTER = "FT.ALTER"; + public const string CONFIG = "FT.CONFIG"; public const string CONFIG_GET = "FT.CONFIG GET"; public const string CONFIG_HELP = "FT.CONFIG HELP"; public const string CONFIG_SET = "FT.CONFIG SET"; diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index 786e4f1a..f6baf0f4 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -204,8 +204,52 @@ public async Task AlterAsync(string index, Schema schema, bool skipInitial return (await _db.ExecuteAsync(FT.ALTER, args)).OKtoBoolean(); } - // TODO: FT.CONFIG_GET - // TODO: FT.CONFIG_SET + /// + /// Retrieve configuration options. + /// + /// is name of the configuration option, or '*' for all. + /// An array reply of the configuration name and value. + /// + public Dictionary ConfigGet(string option) + { + var result = _db.Execute(FT.CONFIG, "GET", option); + return result.ToConfigDictionary(); // TODO: fix all tests to be like this + } + + /// + /// Retrieve configuration options. + /// + /// is name of the configuration option, or '*' for all. + /// An array reply of the configuration name and value. + /// + public async Task> ConfigGetAsync(string option) + { + return (await _db.ExecuteAsync(FT.CONFIG, "GET", option)).ToConfigDictionary(); + } + + /// + /// Describe configuration options. + /// + /// is name of the configuration option, or '*' for all. + /// is value of the configuration option. + /// if executed correctly, error otherwise. + /// + public bool ConfigSet(string option, string value) + { + return _db.Execute(FT.CONFIG, "SET", option, value).OKtoBoolean(); + } + + /// + /// Describe configuration options. + /// + /// is name of the configuration option, or '*' for all. + /// is value of the configuration option. + /// if executed correctly, error otherwise. + /// + public async Task ConfigSetAsync(string option, string value) + { + return (await _db.ExecuteAsync(FT.CONFIG, "SET", option, value)).OKtoBoolean(); + } /// /// Create an index with the given specification. diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index aefde4c5..d50a5dae 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -553,6 +553,96 @@ public async Task AlterAddAsync() Assert.Equal("name", (info.Attributes[2]["attribute"]).ToString()); } + [Fact] + public void TestConfig() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Assert.True(ft.ConfigSet("TIMEOUT", "100")); + Dictionary configMap = ft.ConfigGet("*"); + Assert.Equal("100", configMap["TIMEOUT"].ToString()); + } + + [Fact] + public async Task TestConfigAsnyc() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Assert.True(await ft.ConfigSetAsync("TIMEOUT", "100")); + Dictionary configMap = await ft.ConfigGetAsync("*"); + Assert.Equal("100", configMap["TIMEOUT"].ToString()); + } + + [Fact] + public void configOnTimeout() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Assert.True(ft.ConfigSet("ON_TIMEOUT", "fail")); + Assert.Equal("fail", ft.ConfigGet("ON_TIMEOUT")["ON_TIMEOUT"]); + + try { ft.ConfigSet("ON_TIMEOUT", "null"); } catch (RedisServerException) { } + } + + [Fact] + public async Task configOnTimeoutAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Assert.True(await ft.ConfigSetAsync("ON_TIMEOUT", "fail")); + Assert.Equal("fail", (await ft.ConfigGetAsync("ON_TIMEOUT"))["ON_TIMEOUT"]); + + try { ft.ConfigSet("ON_TIMEOUT", "null"); } catch (RedisServerException) { } + } + + [Fact] + public void TestDialectConfig() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + // confirm default + var result = ft.ConfigGet("DEFAULT_DIALECT"); + Assert.Equal("1", result["DEFAULT_DIALECT"]); // TODO: should be "1" ? + + Assert.True(ft.ConfigSet("DEFAULT_DIALECT", "2")); + Assert.Equal("2", ft.ConfigGet("DEFAULT_DIALECT")["DEFAULT_DIALECT"]); + try { ft.ConfigSet("DEFAULT_DIALECT", "0"); } catch (RedisServerException) { } + try { ft.ConfigSet("DEFAULT_DIALECT", "3"); } catch (RedisServerException) { } + + // Assert.Throws(() => ft.ConfigSet("DEFAULT_DIALECT", "0")); + // Assert.Throws(() => ft.ConfigSet("DEFAULT_DIALECT", "3")); + + // Restore to default + Assert.True(ft.ConfigSet("DEFAULT_DIALECT", "1")); + } + + [Fact] + public async Task TestDialectConfigAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + // confirm default + var result = await ft.ConfigGetAsync("DEFAULT_DIALECT"); + Assert.Equal("1", result["DEFAULT_DIALECT"]); // TODO: should be "1" ? + + Assert.True(await ft.ConfigSetAsync("DEFAULT_DIALECT", "2")); + Assert.Equal("2", (await ft.ConfigGetAsync("DEFAULT_DIALECT"))["DEFAULT_DIALECT"]); + try { await ft.ConfigSetAsync("DEFAULT_DIALECT", "0"); } catch (RedisServerException) { } + try { await ft.ConfigSetAsync("DEFAULT_DIALECT", "3"); } catch (RedisServerException) { } + + // Assert.Throws(() => ft.ConfigSet("DEFAULT_DIALECT", "0")); + // Assert.Throws(() => ft.ConfigSet("DEFAULT_DIALECT", "3")); + + // Restore to default + Assert.True(ft.ConfigSet("DEFAULT_DIALECT", "1")); + } + [Fact] public void TestModulePrefixs() { From dac3fa49d9d22fcd434edb24103b1842f1a9c306 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Tue, 11 Oct 2022 13:42:03 +0300 Subject: [PATCH 21/28] Add FT.CURSOR READ/DEL Commands --- src/NRedisStack/Search/Literals/Commands.cs | 1 + src/NRedisStack/Search/SearchCommands.cs | 62 ++++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/NRedisStack/Search/Literals/Commands.cs b/src/NRedisStack/Search/Literals/Commands.cs index d88f7892..4c57bb4f 100644 --- a/src/NRedisStack/Search/Literals/Commands.cs +++ b/src/NRedisStack/Search/Literals/Commands.cs @@ -13,6 +13,7 @@ internal class FT public const string CONFIG_HELP = "FT.CONFIG HELP"; public const string CONFIG_SET = "FT.CONFIG SET"; public const string CREATE = "FT.CREATE"; + public const string CURSOR = "FT.CURSOR"; public const string CURSOR_DEL = "FT.CURSOR DEL"; public const string CURSOR_READ = "FT.CURSOR READ"; public const string DICTADD = "FT.DICTADD"; diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index f6baf0f4..b680bb35 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -294,8 +294,66 @@ public async Task CreateAsync(string indexName, FTCreateParams parameters, return (await _db.ExecuteAsync(FT.CREATE, args)).OKtoBoolean(); } - // TODO: FT.CURSOR DEL - // TODO: FT.CURSOR READ + // TODO: FT.CURSOR DEL test + /// + /// Delete a cursor from the index. + /// + /// The index name + /// The cursor's ID. + /// if it has been deleted, if it did not exist. + /// + public bool CursorDel(string indexName, long cursorId) + { + return _db.Execute(FT.CURSOR, "DEL", indexName, cursorId).OKtoBoolean(); + } + + /// + /// Delete a cursor from the index. + /// + /// The index name + /// The cursor's ID. + /// if it has been deleted, if it did not exist. + /// + public async Task CursorDelAsync(string indexName, long cursorId) + { + return (await _db.ExecuteAsync(FT.CURSOR, "DEL", indexName, cursorId)).OKtoBoolean(); + } + // TODO: FT.CURSOR READ test + + /// + /// Read next results from an existing cursor. + /// + /// The index name + /// The cursor's ID. + /// Limit the amount of returned results. + /// A AggregationResult object with the results + /// + public AggregationResult CursorRead(string indexName, long cursorId, int? count = null) + { + RedisResult[] resp = ((count == null) ? _db.Execute(FT.CURSOR, "READ", indexName, cursorId) + : _db.Execute(FT.CURSOR, "READ", indexName, cursorId, "COUNT", count)) + .ToArray(); + + return new AggregationResult(resp[0], (long)resp[1]); + } + + /// + /// Read next results from an existing cursor. + /// + /// The index name + /// The cursor's ID. + /// Limit the amount of returned results. + /// A AggregationResult object with the results + /// + public async Task CursorReadAsync(string indexName, long cursorId, int? count = null) + { + RedisResult[] resp = (await ((count == null) ? _db.ExecuteAsync(FT.CURSOR, "READ", indexName, cursorId) + : _db.ExecuteAsync(FT.CURSOR, "READ", indexName, cursorId, "COUNT", count))) + .ToArray(); + + return new AggregationResult(resp[0], (long)resp[1]); + } + // TODO: FT.DICTADD // TODO: FT.DICTDEL // TODO: FT.DICTDUMP From 49ba5bbafb701185185ae99dc3c82e7175f55053 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Tue, 11 Oct 2022 14:52:19 +0300 Subject: [PATCH 22/28] Add FT.DICT(ADD/DEL/DUMP) Commands --- src/NRedisStack/Search/SearchCommands.cs | 59 ++++++++++++++ tests/NRedisStack.Tests/Search/SearchTests.cs | 79 +++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index b680bb35..24fe5d7a 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -355,8 +355,67 @@ public async Task CursorReadAsync(string indexName, long curs } // TODO: FT.DICTADD + /// + /// Add terms to a dictionary. + /// + /// The dictionary name + /// Terms to add to the dictionary.. + /// The number of new terms that were added. + /// + public long DictAdd(string dict, params string[] terms) + { + if(terms.Length < 1) + { + throw new ArgumentOutOfRangeException("At least one term must be provided"); + } + + var args = new List(terms.Length + 1) { dict }; + foreach (var t in terms) + { + args.Add(t); + } + + return _db.Execute(FT.DICTADD, args).ToLong(); + } + + /// + /// Delete terms from a dictionary. + /// + /// The dictionary name + /// Terms to delete to the dictionary.. + /// The number of new terms that were deleted. + /// + public long DictDel(string dict, params string[] terms) + { + if(terms.Length < 1) + { + throw new ArgumentOutOfRangeException("At least one term must be provided"); + } + + var args = new List(terms.Length + 1) { dict }; + foreach (var t in terms) + { + args.Add(t); + } + + return _db.Execute(FT.DICTDEL, args).ToLong(); + } + + /// + /// Dump all terms in the given dictionary. + /// + /// The dictionary name + /// An array, where each element is term. + /// + public RedisResult[] DictDump(string dict) + { + return _db.Execute(FT.DICTDUMP, dict).ToArray(); + } + + // TODO: FT.DICTDEL // TODO: FT.DICTDUMP + // TODO: FT.DROPINDEX // TODO: FT.EXPLAIN // TODO: FT.EXPLAINCLI diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index d50a5dae..f3dfc891 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -643,6 +643,85 @@ public async Task TestDialectConfigAsync() Assert.True(ft.ConfigSet("DEFAULT_DIALECT", "1")); } + [Fact] + public async Task TestCursor() // TODO: finish this test and understand whats the problem with aggregate (maybe related to the dispose?) + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema(); + sc.AddTextField("name", 1.0, sortable: true); + sc.AddNumericField("count", sortable: true); + ft.Create(index, FTCreateParams.CreateParams(), sc); + AddDocument(db, new Document("data1").Set("name", "abc").Set("count", 10)); + AddDocument(db, new Document("data2").Set("name", "def").Set("count", 5)); + AddDocument(db, new Document("data3").Set("name", "def").Set("count", 25)); + + AggregationRequest r = new AggregationRequest() + .GroupBy("@name", Reducers.Sum("@count").As("sum")) + .SortBy(10, SortedField.Desc("@sum")) + .Cursor(1, 3000); + + // actual search + AggregationResult res = ft.Aggregate(index, r); + Row? row = res.GetRow(0); + Assert.NotNull(row); + Assert.Equal("def", row.Value.GetString("name")); + Assert.Equal(30, row.Value.GetLong("sum")); + Assert.Equal(30.0, row.Value.GetDouble("sum")); + + Assert.Equal(0L, row.Value.GetLong("nosuchcol")); + Assert.Equal(0.0, row.Value.GetDouble("nosuchcol")); + Assert.Null(row.Value.GetString("nosuchcol")); + + res = ft.CursorRead(index, res.CursorId, 1); + Row? row2 = res.GetRow(0); + + Assert.NotNull(row2); + Assert.Equal("abc", row2.Value.GetString("name")); + Assert.Equal(10, row2.Value.GetLong("sum")); + + Assert.True(ft.CursorDel(index, res.CursorId)); + + try + { + ft.CursorRead(index, res.CursorId, 1); + Assert.True(false); + } + catch (RedisException) { } + + _ = new AggregationRequest() + .GroupBy("@name", Reducers.Sum("@count").As("sum")) + .SortBy(10, SortedField.Desc("@sum")) + .Cursor(1, 1000); + + await Task.Delay(1000).ConfigureAwait(false); + + try + { + ft.CursorRead(index, res.CursorId, 1); + Assert.True(false); + } + catch (RedisException) { } + } + + [Fact] + public void TestDictionary() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Assert.Equal(3L, ft.DictAdd("dict", "bar", "foo", "hello world")); + var dumResult = ft.DictDump("dict"); + int i = 0; + + Assert.Equal("bar",dumResult[i++].ToString()); + Assert.Equal("foo",dumResult[i++].ToString()); + Assert.Equal("hello world",dumResult[i].ToString()); + Assert.Equal(3L, ft.DictDel("dict", "foo", "bar", "hello world")); + Assert.Equal(ft.DictDump("dict").Length, 0); + } + [Fact] public void TestModulePrefixs() { From 45309c30d269c76b08d6191e9a6d899f023ff53e Mon Sep 17 00:00:00 2001 From: shacharPash Date: Tue, 11 Oct 2022 14:57:16 +0300 Subject: [PATCH 23/28] Add FT.DICT(ADD/DEL/DUMP) Async Commands + Tests --- src/NRedisStack/Search/SearchCommands.cs | 60 +++++++++++++++++-- tests/NRedisStack.Tests/Search/SearchTests.cs | 23 ++++++- 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index 24fe5d7a..27950793 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -354,7 +354,6 @@ public async Task CursorReadAsync(string indexName, long curs return new AggregationResult(resp[0], (long)resp[1]); } - // TODO: FT.DICTADD /// /// Add terms to a dictionary. /// @@ -378,6 +377,29 @@ public long DictAdd(string dict, params string[] terms) return _db.Execute(FT.DICTADD, args).ToLong(); } + /// + /// Add terms to a dictionary. + /// + /// The dictionary name + /// Terms to add to the dictionary.. + /// The number of new terms that were added. + /// + public async Task DictAddAsync(string dict, params string[] terms) + { + if(terms.Length < 1) + { + throw new ArgumentOutOfRangeException("At least one term must be provided"); + } + + var args = new List(terms.Length + 1) { dict }; + foreach (var t in terms) + { + args.Add(t); + } + + return (await _db.ExecuteAsync(FT.DICTADD, args)).ToLong(); + } + /// /// Delete terms from a dictionary. /// @@ -401,6 +423,29 @@ public long DictDel(string dict, params string[] terms) return _db.Execute(FT.DICTDEL, args).ToLong(); } + /// + /// Delete terms from a dictionary. + /// + /// The dictionary name + /// Terms to delete to the dictionary.. + /// The number of new terms that were deleted. + /// + public async Task DictDelAsync(string dict, params string[] terms) + { + if(terms.Length < 1) + { + throw new ArgumentOutOfRangeException("At least one term must be provided"); + } + + var args = new List(terms.Length + 1) { dict }; + foreach (var t in terms) + { + args.Add(t); + } + + return (await _db.ExecuteAsync(FT.DICTDEL, args)).ToLong(); + } + /// /// Dump all terms in the given dictionary. /// @@ -412,9 +457,16 @@ public RedisResult[] DictDump(string dict) return _db.Execute(FT.DICTDUMP, dict).ToArray(); } - - // TODO: FT.DICTDEL - // TODO: FT.DICTDUMP + /// + /// Dump all terms in the given dictionary. + /// + /// The dictionary name + /// An array, where each element is term. + /// + public async Task DictDumpAsync(string dict) + { + return (await _db.ExecuteAsync(FT.DICTDUMP, dict)).ToArray(); + } // TODO: FT.DROPINDEX // TODO: FT.EXPLAIN diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index f3dfc891..3a219d6a 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -711,17 +711,38 @@ public void TestDictionary() IDatabase db = redisFixture.Redis.GetDatabase(); db.Execute("FLUSHALL"); var ft = db.FT(); + Assert.Equal(3L, ft.DictAdd("dict", "bar", "foo", "hello world")); + var dumResult = ft.DictDump("dict"); int i = 0; - Assert.Equal("bar",dumResult[i++].ToString()); Assert.Equal("foo",dumResult[i++].ToString()); Assert.Equal("hello world",dumResult[i].ToString()); + Assert.Equal(3L, ft.DictDel("dict", "foo", "bar", "hello world")); Assert.Equal(ft.DictDump("dict").Length, 0); } + [Fact] + public async Task TestDictionaryAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + + Assert.Equal(3L, await ft.DictAddAsync("dict", "bar", "foo", "hello world")); + + var dumResult = await ft.DictDumpAsync("dict"); + int i = 0; + Assert.Equal("bar",dumResult[i++].ToString()); + Assert.Equal("foo",dumResult[i++].ToString()); + Assert.Equal("hello world",dumResult[i].ToString()); + + Assert.Equal(3L, await ft.DictDelAsync("dict", "foo", "bar", "hello world")); + Assert.Equal((await ft.DictDumpAsync("dict")).Length, 0); + } + [Fact] public void TestModulePrefixs() { From eeb870fd465f266c9764b5f492efae0b3b66126b Mon Sep 17 00:00:00 2001 From: shacharPash Date: Tue, 11 Oct 2022 15:37:32 +0300 Subject: [PATCH 24/28] Add FT.DROPINDEX Sync/Async Commands + Tests --- src/NRedisStack/Search/SearchCommands.cs | 28 ++++ tests/NRedisStack.Tests/Search/SearchTests.cs | 120 +++++++++++++++++- 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index 27950793..8031e606 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -468,6 +468,34 @@ public async Task DictDumpAsync(string dict) return (await _db.ExecuteAsync(FT.DICTDUMP, dict)).ToArray(); } + /// + /// Delete an index. + /// + /// The index name + /// If set, deletes the actual document hashes. + /// if executed correctly, error otherwise + /// + public bool DropIndex(string indexName, bool dd = false) + { + return ((dd) ? _db.Execute(FT.DROPINDEX, indexName, "DD") + : _db.Execute(FT.DROPINDEX, indexName)) + .OKtoBoolean(); + } + + /// + /// Delete an index. + /// + /// The index name + /// If set, deletes the actual document hashes. + /// if executed correctly, error otherwise + /// + public async Task DropIndexAsync(string indexName, bool dd = false) + { + return (await ((dd) ? _db.ExecuteAsync(FT.DROPINDEX, indexName, "DD") + : _db.ExecuteAsync(FT.DROPINDEX, indexName))) + .OKtoBoolean(); + } + // TODO: FT.DROPINDEX // TODO: FT.EXPLAIN // TODO: FT.EXPLAINCLI diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 3a219d6a..fa565a46 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -52,7 +52,7 @@ private void AddDocument(IDatabase db, string key, Dictionary ob // hash[i] = new HashEntry(property.Key, property.Value.ToString()); // } // db.HashSet(key, hash); - var nameValue = new List(); + var nameValue = new List() { key }; foreach (var item in objDictionary) { nameValue.Add(item.Key); @@ -724,6 +724,124 @@ public void TestDictionary() Assert.Equal(ft.DictDump("dict").Length, 0); } + [Fact] + public void TestDropIndex() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema().AddTextField("title", 1.0); + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + + Dictionary fields = new Dictionary(); + fields.Add("title", "hello world"); + for (int i = 0; i < 100; i++) + { + AddDocument(db, $"doc{i}", fields); + } + + SearchResult res = ft.Search(index, new Query("hello world")); + Assert.Equal(100, res.TotalResults); + + Assert.True(ft.DropIndex(index)); + + try + { + ft.Search(index, new Query("hello world")); + //fail("Index should not exist."); + } + catch (RedisServerException ex) + { + Assert.True(ex.Message.Contains("no such index")); + } + Assert.Equal("100", db.Execute("DBSIZE").ToString()); + } + + [Fact] + public async Task TestDropIndexAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema().AddTextField("title", 1.0); + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + + Dictionary fields = new Dictionary(); + fields.Add("title", "hello world"); + for (int i = 0; i < 100; i++) + { + AddDocument(db, $"doc{i}", fields); + } + + SearchResult res = ft.Search(index, new Query("hello world")); + Assert.Equal(100, res.TotalResults); + + Assert.True(await ft.DropIndexAsync(index)); + + try + { + ft.Search(index, new Query("hello world")); + //fail("Index should not exist."); + } + catch (RedisServerException ex) + { + Assert.True(ex.Message.Contains("no such index")); + } + Assert.Equal("100", db.Execute("DBSIZE").ToString()); + } + + [Fact] + public void dropIndexDD() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema().AddTextField("title", 1.0); + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + + Dictionary fields = new Dictionary(); + fields.Add("title", "hello world"); + for (int i = 0; i < 100; i++) + { + AddDocument(db, $"doc{i}", fields); + } + + SearchResult res = ft.Search(index, new Query("hello world")); + Assert.Equal(100, res.TotalResults); + + Assert.True(ft.DropIndex(index, true)); + + RedisResult[] keys = (RedisResult[]) db.Execute("KEYS", "*"); + Assert.True(keys.Length == 0); + Assert.Equal("0", db.Execute("DBSIZE").ToString()); + } + + [Fact] + public async Task dropIndexDDAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema().AddTextField("title", 1.0); + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + + Dictionary fields = new Dictionary(); + fields.Add("title", "hello world"); + for (int i = 0; i < 100; i++) + { + AddDocument(db, $"doc{i}", fields); + } + + SearchResult res = ft.Search(index, new Query("hello world")); + Assert.Equal(100, res.TotalResults); + + Assert.True(await ft.DropIndexAsync(index, true)); + + RedisResult[] keys = (RedisResult[]) db.Execute("KEYS", "*"); + Assert.True(keys.Length == 0); + Assert.Equal("0", db.Execute("DBSIZE").ToString()); + } + [Fact] public async Task TestDictionaryAsync() { From d8da602572786c1261747a0156a5c9e9052f42e3 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Tue, 11 Oct 2022 18:13:37 +0300 Subject: [PATCH 25/28] Fix Aggregate & Add more Commands --- src/NRedisStack/Search/SearchCommands.cs | 156 ++++++++- tests/NRedisStack.Tests/Search/SearchTests.cs | 319 +++++++++++++----- 2 files changed, 377 insertions(+), 98 deletions(-) diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index 8031e606..79af17a6 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -47,7 +47,7 @@ public AggregationResult Aggregate(string index, AggregationRequest query) //query.SerializeRedisArgs(args); foreach(var arg in query.GetArgs()) { - args.Add(arg); + args.Add(arg.ToString()); } var result = _db.Execute(FT.AGGREGATE, args); if (query.IsWithCursor()) @@ -294,7 +294,6 @@ public async Task CreateAsync(string indexName, FTCreateParams parameters, return (await _db.ExecuteAsync(FT.CREATE, args)).OKtoBoolean(); } - // TODO: FT.CURSOR DEL test /// /// Delete a cursor from the index. /// @@ -493,12 +492,65 @@ public async Task DropIndexAsync(string indexName, bool dd = false) { return (await ((dd) ? _db.ExecuteAsync(FT.DROPINDEX, indexName, "DD") : _db.ExecuteAsync(FT.DROPINDEX, indexName))) - .OKtoBoolean(); + .OKtoBoolean(); + } + + /// + /// Return the execution plan for a complex query + /// + /// The index name + /// The query to explain + /// String that representing the execution plan + /// + public string Explain(string indexName, Query q) + { + var args = new List { indexName }; + q.SerializeRedisArgs(args); + return _db.Execute(FT.EXPLAIN, args).ToString(); } - // TODO: FT.DROPINDEX - // TODO: FT.EXPLAIN - // TODO: FT.EXPLAINCLI + /// + /// Return the execution plan for a complex query + /// + /// The index name + /// The query to explain + /// String that representing the execution plan + /// + public async Task ExplainAsync(string indexName, Query q) + { + var args = new List { indexName }; + q.SerializeRedisArgs(args); + return (await _db.ExecuteAsync(FT.EXPLAIN, args)).ToString(); + } + + // TODO: FT.EXPLAINCLI - finish this + /// + /// Return the execution plan for a complex query + /// + /// The index name + /// The query to explain + /// An array reply with a string representing the execution plan + /// + public RedisResult[] ExplainCli(string indexName, Query q) + { + var args = new List { indexName }; + q.SerializeRedisArgs(args); + return _db.Execute(FT.EXPLAINCLI, args).ToArray(); + } + + /// + /// Return the execution plan for a complex query + /// + /// The index name + /// The query to explain + /// An array reply with a string representing the execution plan + /// + public async Task ExplainCliAsync(string indexName, Query q) + { + var args = new List { indexName }; + q.SerializeRedisArgs(args); + return (await _db.ExecuteAsync(FT.EXPLAINCLI, args)).ToArray(); + } // /// // /// Return information and statistics on the index. @@ -537,12 +589,10 @@ public async Task InfoAsync(RedisValue index) => /// The index name /// a object with the query string and optional parameters /// a object with the results + /// public SearchResult Search(string indexName, Query q) { var args = new List { indexName }; - // { - // _boxedIndexName - // }; q.SerializeRedisArgs(args); var resp = _db.Execute("FT.SEARCH", args).ToArray(); @@ -555,6 +605,7 @@ public SearchResult Search(string indexName, Query q) /// The index name /// a object with the query string and optional parameters /// a object with the results + /// public async Task SearchAsync(string indexName, Query q) { var args = new List { indexName }; @@ -564,8 +615,91 @@ public async Task SearchAsync(string indexName, Query q) } // TODO: FT.SPELLCHECK - // TODO: FT.SYNDUMP - // TODO: FT.SYNUPDATE + + /// + /// Dump the contents of a synonym group. + /// + /// The index name + /// Pairs of term and an array of synonym groups. + /// + public Dictionary> SynDump(string indexName) + { + var resp = _db.Execute(FT.SYNDUMP, indexName).ToArray(); + var result = new Dictionary>(); + for (int i = 0; i < resp.Length; i += 2) + { + var term = resp[i].ToString(); + var synonyms = (resp[i + 1]).ToArray().Select(x => x.ToString()).ToList(); // TODO: consider leave synonyms as RedisValue[] + result.Add(term, synonyms); + } + return result; + } + + // TODO: FT.SPELLCHECK + + /// + /// Dump the contents of a synonym group. + /// + /// The index name + /// Pairs of term and an array of synonym groups. + /// + public async Task>> SynDumpAsync(string indexName) + { + var resp = (await _db.ExecuteAsync(FT.SYNDUMP, indexName)).ToArray(); + var result = new Dictionary>(); + for (int i = 0; i < resp.Length; i += 2) + { + var term = resp[i].ToString(); + var synonyms = (resp[i + 1]).ToArray().Select(x => x.ToString()).ToList(); // TODO: consider leave synonyms as RedisValue[] + result.Add(term, synonyms); + } + return result; + } + + /// + /// Update a synonym group. + /// + /// The index name + /// Is synonym group to return + /// does not scan and index, and only documents + /// that are indexed after the update are affected + /// The terms + /// Pairs of term and an array of synonym groups. + /// + public bool SynUpdate(string indexName, string synonymGroupId, bool skipInitialScan = false, params string[] terms) + { + if(terms.Length < 1) + { + throw new ArgumentOutOfRangeException("terms must have at least one element"); + } + var args = new List { indexName, synonymGroupId }; + if (skipInitialScan) { args.Add(SearchArgs.SKIPINITIALSCAN); } + args.AddRange(terms); + return _db.Execute(FT.SYNUPDATE, args).OKtoBoolean(); + } + + /// + /// Update a synonym group. + /// + /// The index name + /// Is synonym group to return + /// does not scan and index, and only documents + /// that are indexed after the update are affected + /// The terms + /// Pairs of term and an array of synonym groups. + /// + public async Task SynUpdateAsync(string indexName, string synonymGroupId, bool skipInitialScan = false, params string[] terms) + { + if(terms.Length < 1) + { + throw new ArgumentOutOfRangeException("terms must have at least one element"); + } + var args = new List { indexName, synonymGroupId }; + if (skipInitialScan) { args.Add(SearchArgs.SKIPINITIALSCAN); } + args.AddRange(terms); + return (await _db.ExecuteAsync(FT.SYNUPDATE, args)).OKtoBoolean(); + } + // TODO: FT.TAGVALS } } \ No newline at end of file diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index fa565a46..b0db4241 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -152,49 +152,46 @@ public async Task TestAggregationRequestTimeoutAsync() } // // TODO: underastant why its not working - // [Fact] - // public void TestAggregations() - // { - // IDatabase db = redisFixture.Redis.GetDatabase(); - // db.Execute("FLUSHALL"); - // var ft = db.FT(); - // Schema sc = new Schema(); - // sc.AddTextField("name", 1.0, true); - // sc.AddNumericField("count", true); - // ft.Create(index, FTCreateParams.CreateParams(), sc); - // // client.AddDocument(new Document("data1").Set("name", "abc").Set("count", 10)); - // // client.AddDocument(new Document("data2").Set("name", "def").Set("count", 5)); - // // client.AddDocument(new Document("data3").Set("name", "def").Set("count", 25)); - // AddDocument(db, new Document("data1").Set("name", "abc").Set("count", 10)); - // AddDocument(db, new Document("data2").Set("name", "def").Set("count", 5)); - // AddDocument(db, new Document("data3").Set("name", "def").Set("count", 25)); - - // AggregationRequest r = new AggregationRequest() - // .GroupBy("@name", Reducers.Sum("@count").As ("sum")) - // .SortBy(10, SortedField.Desc("@sum")); - - // // actual search - // // var resBefore = db.Execute("FT.AGGREGATE", index, "*", "GROUPBY", "1", "@name", "REDUCE", "SUM", "1", "@count", "AS", "sum", "SORTBY", "2", "@sum", "DESC", "MAX", "10"); - // //var res = new AggregationResult(resBefore); - - // var res = ft.Aggregate(index, r); - // Assert.Equal(2, res.TotalResults); - - // Row r1 = res.GetRow(0); - // Assert.NotNull(r1); - // Assert.Equal("def", r1.GetString("name")); - // Assert.Equal(30, r1.GetLong("sum")); - // Assert.Equal(30, r1.GetDouble("sum"), 0); - - // Assert.Equal(0L, r1.GetLong("nosuchcol")); - // Assert.Equal(0.0, r1.GetDouble("nosuchcol"), 0); - // Assert.Equal("", r1.GetString("nosuchcol")); - - // Row r2 = res.GetRow(1); - // Assert.NotNull(r2); - // Assert.Equal("abc", r2.GetString("name")); - // Assert.Equal(10, r2.GetLong("sum")); - // } + [Fact] + public void TestAggregations() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema(); + sc.AddTextField("name", 1.0, true); + sc.AddNumericField("count", true); + ft.Create(index, FTCreateParams.CreateParams(), sc); + // client.AddDocument(new Document("data1").Set("name", "abc").Set("count", 10)); + // client.AddDocument(new Document("data2").Set("name", "def").Set("count", 5)); + // client.AddDocument(new Document("data3").Set("name", "def").Set("count", 25)); + AddDocument(db, new Document("data1").Set("name", "abc").Set("count", 10)); + AddDocument(db, new Document("data2").Set("name", "def").Set("count", 5)); + AddDocument(db, new Document("data3").Set("name", "def").Set("count", 25)); + + AggregationRequest r = new AggregationRequest() + .GroupBy("@name", Reducers.Sum("@count").As ("sum")) + .SortBy(10, SortedField.Desc("@sum")); + + // actual search + var res = ft.Aggregate(index, r); + Assert.Equal(2, res.TotalResults); + + Row r1 = res.GetRow(0); + Assert.NotNull(r1); + Assert.Equal("def", r1.GetString("name")); + Assert.Equal(30, r1.GetLong("sum")); + Assert.Equal(30, r1.GetDouble("sum"), 0); + + Assert.Equal(0L, r1.GetLong("nosuchcol")); + Assert.Equal(0.0, r1.GetDouble("nosuchcol"), 0); + Assert.Null(r1.GetString("nosuchcol")); + + Row r2 = res.GetRow(1); + Assert.NotNull(r2); + Assert.Equal("abc", r2.GetString("name")); + Assert.Equal(10, r2.GetLong("sum")); + } [Fact] public void TestAggregationRequestParamsDialect() @@ -259,49 +256,49 @@ public async Task TestAggregationRequestParamsDialectAsync() } // // TODO: underastant why its not working - // [Fact] - // public void TestApplyAndFilterAggregations() - // { - // IDatabase db = redisFixture.Redis.GetDatabase(); - // db.Execute("FLUSHALL"); - // var ft = db.FT(); - // Schema sc = new Schema(); - // sc.AddTextField("name", 1.0, sortable: true); - // sc.AddNumericField("subj1", sortable: true); - // sc.AddNumericField("subj2", sortable: true); - // ft.Create(index, FTCreateParams.CreateParams(), sc); - // // client.AddDocument(db, new Document("data1").Set("name", "abc").Set("subj1", 20).Set("subj2", 70)); - // // client.AddDocument(db, new Document("data2").Set("name", "def").Set("subj1", 60).Set("subj2", 40)); - // // client.AddDocument(db, new Document("data3").Set("name", "ghi").Set("subj1", 50).Set("subj2", 80)); - // // client.AddDocument(db, new Document("data4").Set("name", "abc").Set("subj1", 30).Set("subj2", 20)); - // // client.AddDocument(db, new Document("data5").Set("name", "def").Set("subj1", 65).Set("subj2", 45)); - // // client.AddDocument(db, new Document("data6").Set("name", "ghi").Set("subj1", 70).Set("subj2", 70)); - // AddDocument(db, new Document("data1").Set("name", "abc").Set("subj1", 20).Set("subj2", 70)); - // AddDocument(db, new Document("data2").Set("name", "def").Set("subj1", 60).Set("subj2", 40)); - // AddDocument(db, new Document("data3").Set("name", "ghi").Set("subj1", 50).Set("subj2", 80)); - // AddDocument(db, new Document("data4").Set("name", "abc").Set("subj1", 30).Set("subj2", 20)); - // AddDocument(db, new Document("data5").Set("name", "def").Set("subj1", 65).Set("subj2", 45)); - // AddDocument(db, new Document("data6").Set("name", "ghi").Set("subj1", 70).Set("subj2", 70)); - - // AggregationRequest r = new AggregationRequest().Apply("(@subj1+@subj2)/2", "attemptavg") - // .GroupBy("@name", Reducers.Avg("@attemptavg").As("avgscore")) - // .Filter("@avgscore>=50") - // .SortBy(10, SortedField.Asc("@name")); - - // // actual search - // AggregationResult res = ft.Aggregate(index, r); - // Assert.Equal(3, res.TotalResults); - - // Row r1 = res.GetRow(0); - // Assert.NotNull(r1); - // Assert.Equal("def", r1.GetString("name")); - // Assert.Equal(52.5, r1.GetDouble("avgscore"), 0); - - // Row r2 = res.GetRow(1); - // Assert.NotNull(r2); - // Assert.Equal("ghi", r2.GetString("name")); - // Assert.Equal(67.5, r2.GetDouble("avgscore"), 0); - // } + [Fact] + public void TestApplyAndFilterAggregations() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema(); + sc.AddTextField("name", 1.0, sortable: true); + sc.AddNumericField("subj1", sortable: true); + sc.AddNumericField("subj2", sortable: true); + ft.Create(index, FTCreateParams.CreateParams(), sc); + // client.AddDocument(db, new Document("data1").Set("name", "abc").Set("subj1", 20).Set("subj2", 70)); + // client.AddDocument(db, new Document("data2").Set("name", "def").Set("subj1", 60).Set("subj2", 40)); + // client.AddDocument(db, new Document("data3").Set("name", "ghi").Set("subj1", 50).Set("subj2", 80)); + // client.AddDocument(db, new Document("data4").Set("name", "abc").Set("subj1", 30).Set("subj2", 20)); + // client.AddDocument(db, new Document("data5").Set("name", "def").Set("subj1", 65).Set("subj2", 45)); + // client.AddDocument(db, new Document("data6").Set("name", "ghi").Set("subj1", 70).Set("subj2", 70)); + AddDocument(db, new Document("data1").Set("name", "abc").Set("subj1", 20).Set("subj2", 70)); + AddDocument(db, new Document("data2").Set("name", "def").Set("subj1", 60).Set("subj2", 40)); + AddDocument(db, new Document("data3").Set("name", "ghi").Set("subj1", 50).Set("subj2", 80)); + AddDocument(db, new Document("data4").Set("name", "abc").Set("subj1", 30).Set("subj2", 20)); + AddDocument(db, new Document("data5").Set("name", "def").Set("subj1", 65).Set("subj2", 45)); + AddDocument(db, new Document("data6").Set("name", "ghi").Set("subj1", 70).Set("subj2", 70)); + + AggregationRequest r = new AggregationRequest().Apply("(@subj1+@subj2)/2", "attemptavg") + .GroupBy("@name", Reducers.Avg("@attemptavg").As("avgscore")) + .Filter("@avgscore>=50") + .SortBy(10, SortedField.Asc("@name")); + + // actual search + AggregationResult res = ft.Aggregate(index, r); + Assert.Equal(3, res.TotalResults); + + Row r1 = res.GetRow(0); + Assert.NotNull(r1); + Assert.Equal("def", r1.GetString("name")); + Assert.Equal(52.5, r1.GetDouble("avgscore"), 0); + + Row r2 = res.GetRow(1); + Assert.NotNull(r2); + Assert.Equal("ghi", r2.GetString("name")); + Assert.Equal(67.5, r2.GetDouble("avgscore"), 0); + } [Fact] public void TestCreate() @@ -644,7 +641,7 @@ public async Task TestDialectConfigAsync() } [Fact] - public async Task TestCursor() // TODO: finish this test and understand whats the problem with aggregate (maybe related to the dispose?) + public async Task TestCursor() { IDatabase db = redisFixture.Redis.GetDatabase(); db.Execute("FLUSHALL"); @@ -705,6 +702,68 @@ public async Task TestCursor() // TODO: finish this test and understand whats th catch (RedisException) { } } + [Fact] + public async Task TestCursorAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema(); + sc.AddTextField("name", 1.0, sortable: true); + sc.AddNumericField("count", sortable: true); + ft.Create(index, FTCreateParams.CreateParams(), sc); + AddDocument(db, new Document("data1").Set("name", "abc").Set("count", 10)); + AddDocument(db, new Document("data2").Set("name", "def").Set("count", 5)); + AddDocument(db, new Document("data3").Set("name", "def").Set("count", 25)); + + AggregationRequest r = new AggregationRequest() + .GroupBy("@name", Reducers.Sum("@count").As("sum")) + .SortBy(10, SortedField.Desc("@sum")) + .Cursor(1, 3000); + + // actual search + AggregationResult res = ft.Aggregate(index, r); + Row? row = res.GetRow(0); + Assert.NotNull(row); + Assert.Equal("def", row.Value.GetString("name")); + Assert.Equal(30, row.Value.GetLong("sum")); + Assert.Equal(30.0, row.Value.GetDouble("sum")); + + Assert.Equal(0L, row.Value.GetLong("nosuchcol")); + Assert.Equal(0.0, row.Value.GetDouble("nosuchcol")); + Assert.Null(row.Value.GetString("nosuchcol")); + + res = await ft.CursorReadAsync(index, res.CursorId, 1); + Row? row2 = res.GetRow(0); + + Assert.NotNull(row2); + Assert.Equal("abc", row2.Value.GetString("name")); + Assert.Equal(10, row2.Value.GetLong("sum")); + + Assert.True(await ft.CursorDelAsync(index, res.CursorId)); + + try + { + await ft.CursorReadAsync(index, res.CursorId, 1); + Assert.True(false); + } + catch (RedisException) { } + + _ = new AggregationRequest() + .GroupBy("@name", Reducers.Sum("@count").As("sum")) + .SortBy(10, SortedField.Desc("@sum")) + .Cursor(1, 1000); + + await Task.Delay(1000).ConfigureAwait(false); + + try + { + await ft.CursorReadAsync(index, res.CursorId, 1); + Assert.True(false); + } + catch (RedisException) { } + } + [Fact] public void TestDictionary() { @@ -861,6 +920,92 @@ public async Task TestDictionaryAsync() Assert.Equal((await ft.DictDumpAsync("dict")).Length, 0); } + [Fact] + public void TestExplain() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema() + .AddTextField("f1", 1.0) + .AddTextField("f2", 1.0) + .AddTextField("f3", 1.0); + ft.Create(index, FTCreateParams.CreateParams(), sc); + + String res = ft.Explain(index, new Query("@f3:f3_val @f2:f2_val @f1:f1_val")); + Assert.NotNull(res); + Assert.False(res.Length == 0); + } + + [Fact] + public async Task TestExplainAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema() + .AddTextField("f1", 1.0) + .AddTextField("f2", 1.0) + .AddTextField("f3", 1.0); + ft.Create(index, FTCreateParams.CreateParams(), sc); + + String res = await ft.ExplainAsync(index, new Query("@f3:f3_val @f2:f2_val @f1:f1_val")); + Assert.NotNull(res); + Assert.False(res.Length == 0); + } + + [Fact] + public void TestSynonym() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + var sc = new Schema().AddTextField("name", 1.0).AddTextField("addr", 1.0); + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + + long group1 = 345L; + long group2 = 789L; + string group1_str = group1.ToString(); + string group2_str = group2.ToString(); + Assert.True(ft.SynUpdate(index, group1_str, false, "girl", "baby")); + Assert.True(ft.SynUpdate(index, group1_str, false, "child")); + Assert.True(ft.SynUpdate(index, group2_str, false, "child")); + + Dictionary> dump = ft.SynDump(index); + + Dictionary> expected = new Dictionary>(); + expected.Add("girl", new List() { group1_str }); + expected.Add("baby", new List() { group1_str }); + expected.Add("child", new List() { group1_str, group2_str }); + Assert.Equal(expected, dump); + } + + [Fact] + public async Task TestSynonymAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + var sc = new Schema().AddTextField("name", 1.0).AddTextField("addr", 1.0); + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + + long group1 = 345L; + long group2 = 789L; + string group1_str = group1.ToString(); + string group2_str = group2.ToString(); + Assert.True(await ft.SynUpdateAsync(index, group1_str, false, "girl", "baby")); + Assert.True(await ft.SynUpdateAsync(index, group1_str, false, "child")); + Assert.True(await ft.SynUpdateAsync(index, group2_str, false, "child")); + + Dictionary> dump = await ft.SynDumpAsync(index); + + Dictionary> expected = new Dictionary>(); + expected.Add("girl", new List() { group1_str }); + expected.Add("baby", new List() { group1_str }); + expected.Add("child", new List() { group1_str, group2_str }); + Assert.Equal(expected, dump); + } + [Fact] public void TestModulePrefixs() { From 30330b41be482954df26e536f4ddcf2be08811f1 Mon Sep 17 00:00:00 2001 From: shacharPash Date: Wed, 12 Oct 2022 11:20:41 +0300 Subject: [PATCH 26/28] Add FT.TAGVALS Sync/Async Command + Tests --- src/NRedisStack/Search/SearchCommands.cs | 44 ++++--- tests/NRedisStack.Tests/Search/SearchTests.cs | 120 +++++++++++++++++- 2 files changed, 145 insertions(+), 19 deletions(-) diff --git a/src/NRedisStack/Search/SearchCommands.cs b/src/NRedisStack/Search/SearchCommands.cs index 79af17a6..e994e921 100644 --- a/src/NRedisStack/Search/SearchCommands.cs +++ b/src/NRedisStack/Search/SearchCommands.cs @@ -45,7 +45,7 @@ public AggregationResult Aggregate(string index, AggregationRequest query) { List args = new List { index }; //query.SerializeRedisArgs(args); - foreach(var arg in query.GetArgs()) + foreach (var arg in query.GetArgs()) { args.Add(arg.ToString()); } @@ -73,7 +73,7 @@ public async Task AggregateAsync(string index, AggregationReq { List args = new List { index }; //query.SerializeRedisArgs(args); - foreach(var arg in query.GetArgs()) + foreach (var arg in query.GetArgs()) { args.Add(arg); } @@ -317,7 +317,6 @@ public async Task CursorDelAsync(string indexName, long cursorId) { return (await _db.ExecuteAsync(FT.CURSOR, "DEL", indexName, cursorId)).OKtoBoolean(); } - // TODO: FT.CURSOR READ test /// /// Read next results from an existing cursor. @@ -362,7 +361,7 @@ public async Task CursorReadAsync(string indexName, long curs /// public long DictAdd(string dict, params string[] terms) { - if(terms.Length < 1) + if (terms.Length < 1) { throw new ArgumentOutOfRangeException("At least one term must be provided"); } @@ -385,7 +384,7 @@ public long DictAdd(string dict, params string[] terms) /// public async Task DictAddAsync(string dict, params string[] terms) { - if(terms.Length < 1) + if (terms.Length < 1) { throw new ArgumentOutOfRangeException("At least one term must be provided"); } @@ -408,7 +407,7 @@ public async Task DictAddAsync(string dict, params string[] terms) /// public long DictDel(string dict, params string[] terms) { - if(terms.Length < 1) + if (terms.Length < 1) { throw new ArgumentOutOfRangeException("At least one term must be provided"); } @@ -431,7 +430,7 @@ public long DictDel(string dict, params string[] terms) /// public async Task DictDelAsync(string dict, params string[] terms) { - if(terms.Length < 1) + if (terms.Length < 1) { throw new ArgumentOutOfRangeException("At least one term must be provided"); } @@ -523,7 +522,6 @@ public async Task ExplainAsync(string indexName, Query q) return (await _db.ExecuteAsync(FT.EXPLAIN, args)).ToString(); } - // TODO: FT.EXPLAINCLI - finish this /// /// Return the execution plan for a complex query /// @@ -581,7 +579,7 @@ public InfoResult Info(RedisValue index) => public async Task InfoAsync(RedisValue index) => new InfoResult(await _db.ExecuteAsync("FT.INFO", index)); - // TODO: FT.PROFILE + // TODO: FT.PROFILE (jedis doesn't have it) /// /// Search the index @@ -614,8 +612,6 @@ public async Task SearchAsync(string indexName, Query q) return new SearchResult(resp, !q.NoContent, q.WithScores, q.WithPayloads, q.ExplainScore); } - // TODO: FT.SPELLCHECK - /// /// Dump the contents of a synonym group. /// @@ -635,7 +631,7 @@ public Dictionary> SynDump(string indexName) return result; } - // TODO: FT.SPELLCHECK + // TODO: FT.SPELLCHECK (jedis doesn't have it) /// /// Dump the contents of a synonym group. @@ -668,7 +664,7 @@ public async Task>> SynDumpAsync(string indexNam /// public bool SynUpdate(string indexName, string synonymGroupId, bool skipInitialScan = false, params string[] terms) { - if(terms.Length < 1) + if (terms.Length < 1) { throw new ArgumentOutOfRangeException("terms must have at least one element"); } @@ -690,7 +686,7 @@ public bool SynUpdate(string indexName, string synonymGroupId, bool skipInitialS /// public async Task SynUpdateAsync(string indexName, string synonymGroupId, bool skipInitialScan = false, params string[] terms) { - if(terms.Length < 1) + if (terms.Length < 1) { throw new ArgumentOutOfRangeException("terms must have at least one element"); } @@ -700,6 +696,24 @@ public async Task SynUpdateAsync(string indexName, string synonymGroupId, return (await _db.ExecuteAsync(FT.SYNUPDATE, args)).OKtoBoolean(); } - // TODO: FT.TAGVALS + /// + /// Return a distinct set of values indexed in a Tag field. + /// + /// The index name + /// TAG field name + /// List of TAG field values + /// + public RedisResult[] TagVals(string indexName, string fieldName) => //TODO: consider return Set + _db.Execute(FT.TAGVALS, indexName, fieldName).ToArray(); + + /// + /// Return a distinct set of values indexed in a Tag field. + /// + /// The index name + /// TAG field name + /// List of TAG field values + /// + public async Task TagValsAsync(string indexName, string fieldName) => //TODO: consider return Set + (await _db.ExecuteAsync(FT.TAGVALS, indexName, fieldName)).ToArray(); } } \ No newline at end of file diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index b0db4241..6b958551 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -151,7 +151,6 @@ public async Task TestAggregationRequestTimeoutAsync() Assert.Equal(2, res.TotalResults); } - // // TODO: underastant why its not working [Fact] public void TestAggregations() { @@ -255,7 +254,6 @@ public async Task TestAggregationRequestParamsDialectAsync() Assert.Equal(10, r1.GetLong("sum")); } - // // TODO: underastant why its not working [Fact] public void TestApplyAndFilterAggregations() { @@ -1018,6 +1016,122 @@ public void TestModulePrefixs() Assert.NotEqual(ft1.GetHashCode(), ft2.GetHashCode()); } + [Fact] + public async Task GetTagFieldSyncAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema() + .AddTextField("title", 1.0) + .AddTagField("category"); + + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + Dictionary fields1 = new Dictionary(); + fields1.Add("title", "hello world"); + fields1.Add("category", "red"); + // assertTrue(client.AddDocument(db, "foo", fields1)); + AddDocument(db, "foo", fields1); + Dictionary fields2 = new Dictionary(); + fields2.Add("title", "hello world"); + fields2.Add("category", "blue"); + // assertTrue(client.AddDocument(db, "bar", fields2)); + AddDocument(db, "bar", fields2); + Dictionary fields3 = new Dictionary(); + fields3.Add("title", "hello world"); + fields3.Add("category", "green,yellow"); + // assertTrue(client.AddDocument(db, "baz", fields3)); + AddDocument(db, "baz", fields3); + Dictionary fields4 = new Dictionary(); + fields4.Add("title", "hello world"); + fields4.Add("category", "orange;purple"); + // assertTrue(client.AddDocument(db, "qux", fields4)); + AddDocument(db, "qux", fields4); + + Assert.Equal(1, ft.Search(index, new Query("@category:{red}")).TotalResults); + Assert.Equal(1, ft.Search(index, new Query("@category:{blue}")).TotalResults); + Assert.Equal(1, ft.Search(index, new Query("hello @category:{red}")).TotalResults); + Assert.Equal(1, ft.Search(index, new Query("hello @category:{blue}")).TotalResults); + Assert.Equal(1, ft.Search(index, new Query("@category:{yellow}")).TotalResults); + Assert.Equal(0, ft.Search(index, new Query("@category:{purple}")).TotalResults); + Assert.Equal(1, ft.Search(index, new Query("@category:{orange\\;purple}")).TotalResults); + Assert.Equal(4, ft.Search(index, new Query("hello")).TotalResults); + + var SyncRes = ft.TagVals(index, "category"); + int i = 0; + Assert.Equal(SyncRes[i++].ToString(), "blue"); + Assert.Equal(SyncRes[i++].ToString(), "green"); + Assert.Equal(SyncRes[i++].ToString(), "orange;purple"); + Assert.Equal(SyncRes[i++].ToString(), "red"); + Assert.Equal(SyncRes[i++].ToString(), "yellow"); + + var AsyncRes = await ft.TagValsAsync(index, "category"); + i = 0; + Assert.Equal(SyncRes[i++].ToString(), "blue"); + Assert.Equal(SyncRes[i++].ToString(), "green"); + Assert.Equal(SyncRes[i++].ToString(), "orange;purple"); + Assert.Equal(SyncRes[i++].ToString(), "red"); + Assert.Equal(SyncRes[i++].ToString(), "yellow"); + } + + [Fact] + public async Task TestGetTagFieldWithNonDefaultSeparatorSyncAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema() + .AddTextField("title", 1.0) + .AddTagField("category", separator: ";"); + + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + Dictionary fields1 = new Dictionary(); + fields1.Add("title", "hello world"); + fields1.Add("category", "red"); + // assertTrue(client.AddDocument(db, "foo", fields1)); + AddDocument(db, "foo", fields1); + Dictionary fields2 = new Dictionary(); + fields2.Add("title", "hello world"); + fields2.Add("category", "blue"); + // assertTrue(client.AddDocument(db, "bar", fields2)); + AddDocument(db, "bar", fields2); + Dictionary fields3 = new Dictionary(); + fields3.Add("title", "hello world"); + fields3.Add("category", "green;yellow"); + AddDocument(db, "baz", fields3); + // assertTrue(client.AddDocument(db, "baz", fields3)); + Dictionary fields4 = new Dictionary(); + fields4.Add("title", "hello world"); + fields4.Add("category", "orange,purple"); + // assertTrue(client.AddDocument(db, "qux", fields4)); + AddDocument(db, "qux", fields4); + + Assert.Equal(1, ft.Search(index, new Query("@category:{red}")).TotalResults); + Assert.Equal(1, ft.Search(index, new Query("@category:{blue}")).TotalResults); + Assert.Equal(1, ft.Search(index, new Query("hello @category:{red}")).TotalResults); + Assert.Equal(1, ft.Search(index, new Query("hello @category:{blue}")).TotalResults); + Assert.Equal(1, ft.Search(index, new Query("hello @category:{yellow}")).TotalResults); + Assert.Equal(0, ft.Search(index, new Query("@category:{purple}")).TotalResults); + Assert.Equal(1, ft.Search(index, new Query("@category:{orange\\,purple}")).TotalResults); + Assert.Equal(4, ft.Search(index, new Query("hello")).TotalResults); + + var SyncRes = ft.TagVals(index, "category"); + int i = 0; + Assert.Equal(SyncRes[i++].ToString(), "blue"); + Assert.Equal(SyncRes[i++].ToString(), "green"); + Assert.Equal(SyncRes[i++].ToString(), "orange,purple"); + Assert.Equal(SyncRes[i++].ToString(), "red"); + Assert.Equal(SyncRes[i++].ToString(), "yellow"); + + var AsyncRes = await ft.TagValsAsync(index, "category"); + i = 0; + Assert.Equal(SyncRes[i++].ToString(), "blue"); + Assert.Equal(SyncRes[i++].ToString(), "green"); + Assert.Equal(SyncRes[i++].ToString(), "orange,purple"); + Assert.Equal(SyncRes[i++].ToString(), "red"); + Assert.Equal(SyncRes[i++].ToString(), "yellow"); + } + [Fact] public void TestModulePrefixs1() { @@ -1038,7 +1152,5 @@ public void TestModulePrefixs1() // ... conn.Dispose(); } - } - } \ No newline at end of file From 23601583546b70666a5389965679c1a399415eee Mon Sep 17 00:00:00 2001 From: shacharPash Date: Wed, 12 Oct 2022 12:46:46 +0300 Subject: [PATCH 27/28] Add Sync/Async Tests for FT.ALIAS --- tests/NRedisStack.Tests/Search/SearchTests.cs | 78 ++++++++++++++++--- 1 file changed, 68 insertions(+), 10 deletions(-) diff --git a/tests/NRedisStack.Tests/Search/SearchTests.cs b/tests/NRedisStack.Tests/Search/SearchTests.cs index 6b958551..ddc57920 100644 --- a/tests/NRedisStack.Tests/Search/SearchTests.cs +++ b/tests/NRedisStack.Tests/Search/SearchTests.cs @@ -254,6 +254,64 @@ public async Task TestAggregationRequestParamsDialectAsync() Assert.Equal(10, r1.GetLong("sum")); } + [Fact] + public void TestAlias() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema().AddTextField("field1"); + + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + + Dictionary doc = new Dictionary(); + doc.Add("field1", "value"); + AddDocument(db, "doc1", doc); + + Assert.True(ft.AliasAdd("ALIAS1", index)); + SearchResult res1 = ft.Search("ALIAS1", new Query("*").ReturnFields("field1")); + Assert.Equal(1, res1.TotalResults); + Assert.Equal("value", res1.Documents[0]["field1"]); + + Assert.True(ft.AliasUpdate("ALIAS2", index)); + SearchResult res2 = ft.Search("ALIAS2", new Query("*").ReturnFields("field1")); + Assert.Equal(1, res2.TotalResults); + Assert.Equal("value", res2.Documents[0]["field1"]); + + Assert.Throws(() => ft.AliasDel("ALIAS3")); + Assert.True(ft.AliasDel("ALIAS2")); + Assert.Throws(() => ft.AliasDel("ALIAS2")); + } + + [Fact] + public async Task TestAliasAsync() + { + IDatabase db = redisFixture.Redis.GetDatabase(); + db.Execute("FLUSHALL"); + var ft = db.FT(); + Schema sc = new Schema().AddTextField("field1"); + + Assert.True(ft.Create(index, FTCreateParams.CreateParams(), sc)); + + Dictionary doc = new Dictionary(); + doc.Add("field1", "value"); + AddDocument(db, "doc1", doc); + + Assert.True(await ft.AliasAddAsync("ALIAS1", index)); + SearchResult res1 = ft.Search("ALIAS1", new Query("*").ReturnFields("field1")); + Assert.Equal(1, res1.TotalResults); + Assert.Equal("value", res1.Documents[0]["field1"]); + + Assert.True(await ft.AliasUpdateAsync("ALIAS2", index)); + SearchResult res2 = ft.Search("ALIAS2", new Query("*").ReturnFields("field1")); + Assert.Equal(1, res2.TotalResults); + Assert.Equal("value", res2.Documents[0]["field1"]); + + await Assert.ThrowsAsync(async () => await ft.AliasDelAsync("ALIAS3")); + Assert.True(await ft.AliasDelAsync("ALIAS2")); + await Assert.ThrowsAsync(async () => await ft.AliasDelAsync("ALIAS2")); + } + [Fact] public void TestApplyAndFilterAggregations() { @@ -310,12 +368,12 @@ public void TestCreate() Assert.True(ft.Create(index, parameters, schema)); db.HashSet("profesor:5555", new HashEntry[] { new("first", "Albert"), new("last", "Blue"), new("age", "55") }); - db.HashSet("student:1111", new HashEntry[] { new("first", "Joe"), new("last", "Dod"), new("age", "18") }); - db.HashSet("pupil:2222", new HashEntry[] { new("first", "Jen"), new("last", "Rod"), new("age", "14") }); - db.HashSet("student:3333", new HashEntry[] { new("first", "El"), new("last", "Mark"), new("age", "17") }); - db.HashSet("pupil:4444", new HashEntry[] { new("first", "Pat"), new("last", "Shu"), new("age", "21") }); - db.HashSet("student:5555", new HashEntry[] { new("first", "Joen"), new("last", "Ko"), new("age", "20") }); - db.HashSet("teacher:6666", new HashEntry[] { new("first", "Pat"), new("last", "Rod"), new("age", "20") }); + db.HashSet("student:1111", new HashEntry[] { new("first", "Joe"), new("last", "Dod"), new("age", "18") }); + db.HashSet("pupil:2222", new HashEntry[] { new("first", "Jen"), new("last", "Rod"), new("age", "14") }); + db.HashSet("student:3333", new HashEntry[] { new("first", "El"), new("last", "Mark"), new("age", "17") }); + db.HashSet("pupil:4444", new HashEntry[] { new("first", "Pat"), new("last", "Shu"), new("age", "21") }); + db.HashSet("student:5555", new HashEntry[] { new("first", "Joen"), new("last", "Ko"), new("age", "20") }); + db.HashSet("teacher:6666", new HashEntry[] { new("first", "Pat"), new("last", "Rod"), new("age", "20") }); var noFilters = ft.Search(index, new Query()); Assert.Equal(4, noFilters.TotalResults); @@ -609,8 +667,8 @@ public void TestDialectConfig() try { ft.ConfigSet("DEFAULT_DIALECT", "0"); } catch (RedisServerException) { } try { ft.ConfigSet("DEFAULT_DIALECT", "3"); } catch (RedisServerException) { } - // Assert.Throws(() => ft.ConfigSet("DEFAULT_DIALECT", "0")); - // Assert.Throws(() => ft.ConfigSet("DEFAULT_DIALECT", "3")); + Assert.Throws(() => ft.ConfigSet("DEFAULT_DIALECT", "0")); + Assert.Throws(() => ft.ConfigSet("DEFAULT_DIALECT", "3")); // Restore to default Assert.True(ft.ConfigSet("DEFAULT_DIALECT", "1")); @@ -631,8 +689,8 @@ public async Task TestDialectConfigAsync() try { await ft.ConfigSetAsync("DEFAULT_DIALECT", "0"); } catch (RedisServerException) { } try { await ft.ConfigSetAsync("DEFAULT_DIALECT", "3"); } catch (RedisServerException) { } - // Assert.Throws(() => ft.ConfigSet("DEFAULT_DIALECT", "0")); - // Assert.Throws(() => ft.ConfigSet("DEFAULT_DIALECT", "3")); + Assert.Throws(() => ft.ConfigSet("DEFAULT_DIALECT", "0")); + Assert.Throws(() => ft.ConfigSet("DEFAULT_DIALECT", "3")); // Restore to default Assert.True(ft.ConfigSet("DEFAULT_DIALECT", "1")); From c3ab0efeafdd8565d5d9646941ccbd62463cb6de Mon Sep 17 00:00:00 2001 From: shacharPash Date: Wed, 12 Oct 2022 16:40:58 +0300 Subject: [PATCH 28/28] Fixing Tdigest.Add Command and Tests --- src/NRedisStack/Tdigest/TdigestCommands.cs | 146 +++--- .../NRedisStack.Tests/Tdigest/TdigestTests.cs | 416 ++++++++++-------- 2 files changed, 330 insertions(+), 232 deletions(-) diff --git a/src/NRedisStack/Tdigest/TdigestCommands.cs b/src/NRedisStack/Tdigest/TdigestCommands.cs index 3e373973..ab02de32 100644 --- a/src/NRedisStack/Tdigest/TdigestCommands.cs +++ b/src/NRedisStack/Tdigest/TdigestCommands.cs @@ -16,53 +16,19 @@ public TdigestCommands(IDatabase db) /// Adds one or more observations to a t-digest sketch. /// /// The name of the sketch. - /// The value of the observation. - /// The weight of this observation. + /// The value of the observation. /// if executed correctly, error otherwise /// - public bool Add(RedisKey key, double item, long weight) + public bool Add(RedisKey key, params double[] values) { - if (weight < 0) throw new ArgumentOutOfRangeException(nameof(weight)); - - return _db.Execute(TDIGEST.ADD, key, item, weight).OKtoBoolean(); - } - - /// - /// Adds one or more observations to a t-digest sketch. - /// - /// The name of the sketch. - /// The value of the observation. - /// The weight of this observation. - /// if executed correctly, error otherwise - /// - public async Task AddAsync(RedisKey key, double item, int weight) - { - if (weight < 0) throw new ArgumentOutOfRangeException(nameof(weight)); - - var result = await _db.ExecuteAsync(TDIGEST.ADD, key, item, weight); - return result.OKtoBoolean(); - } - - /// - /// Adds one or more observations to a t-digest sketch. - /// - /// The name of the sketch. - /// Tuple of the value of the observation and The weight of this observation. - /// if executed correctly, error otherwise - /// - public bool Add(RedisKey key, params Tuple[] valueWeight) - { - if (valueWeight.Length < 1) - throw new ArgumentOutOfRangeException(nameof(valueWeight)); - - var args = new List { key }; - - foreach (var pair in valueWeight) + if (values.Length < 0) throw new ArgumentOutOfRangeException(nameof(values)); + var args = new string[values.Length + 1]; + args[0] = key.ToString(); + for (int i = 0; i < values.Length; i++) { - if (pair.Item2 < 0) throw new ArgumentOutOfRangeException(nameof(pair.Item2)); - args.Add(pair.Item1); - args.Add(pair.Item2); + args[i + 1] = values[i].ToString(); } + return _db.Execute(TDIGEST.ADD, args).OKtoBoolean(); } @@ -70,25 +36,99 @@ public bool Add(RedisKey key, params Tuple[] valueWeight) /// Adds one or more observations to a t-digest sketch. /// /// The name of the sketch. - /// Tuple of the value of the observation and The weight of this observation. + /// The value of the observation. /// if executed correctly, error otherwise /// - public async Task AddAsync(RedisKey key, params Tuple[] valueWeight) + public async Task AddAsync(RedisKey key, params double[] values) { - if (valueWeight.Length < 1) - throw new ArgumentOutOfRangeException(nameof(valueWeight)); - - var args = new List { key }; - - foreach (var pair in valueWeight) + if (values.Length < 0) throw new ArgumentOutOfRangeException(nameof(values)); + var args = new string[values.Length + 1]; + args[0] = key; + for (int i = 0; i < values.Length; i++) { - if (pair.Item2 < 0) throw new ArgumentOutOfRangeException(nameof(pair.Item2)); - args.Add(pair.Item1); - args.Add(pair.Item2); + args[i + 1] = values[i].ToString(); } + return (await _db.ExecuteAsync(TDIGEST.ADD, args)).OKtoBoolean(); } + // /// + // /// Adds one or more observations to a t-digest sketch. + // /// + // /// The name of the sketch. + // /// The value of the observation. + // /// The weight of this observation. + // /// if executed correctly, error otherwise + // /// + // public bool Add(RedisKey key, double item, long weight) + // { + // if (weight < 0) throw new ArgumentOutOfRangeException(nameof(weight)); + + // return _db.Execute(TDIGEST.ADD, key, item, weight).OKtoBoolean(); + // } + + // /// + // /// Adds one or more observations to a t-digest sketch. + // /// + // /// The name of the sketch. + // /// The value of the observation. + // /// The weight of this observation. + // /// if executed correctly, error otherwise + // /// + // public async Task AddAsync(RedisKey key, double item, int weight) + // { + // if (weight < 0) throw new ArgumentOutOfRangeException(nameof(weight)); + + // var result = await _db.ExecuteAsync(TDIGEST.ADD, key, item, weight); + // return result.OKtoBoolean(); + // } + + // /// + // /// Adds one or more observations to a t-digest sketch. + // /// + // /// The name of the sketch. + // /// Tuple of the value of the observation and The weight of this observation. + // /// if executed correctly, error otherwise + // /// + // public bool Add(RedisKey key, params Tuple[] valueWeight) + // { + // if (valueWeight.Length < 1) + // throw new ArgumentOutOfRangeException(nameof(valueWeight)); + + // var args = new List { key }; + + // foreach (var pair in valueWeight) + // { + // if (pair.Item2 < 0) throw new ArgumentOutOfRangeException(nameof(pair.Item2)); + // args.Add(pair.Item1); + // args.Add(pair.Item2); + // } + // return _db.Execute(TDIGEST.ADD, args).OKtoBoolean(); + // } + + // /// + // /// Adds one or more observations to a t-digest sketch. + // /// + // /// The name of the sketch. + // /// Tuple of the value of the observation and The weight of this observation. + // /// if executed correctly, error otherwise + // /// + // public async Task AddAsync(RedisKey key, params Tuple[] valueWeight) + // { + // if (valueWeight.Length < 1) + // throw new ArgumentOutOfRangeException(nameof(valueWeight)); + + // var args = new List { key }; + + // foreach (var pair in valueWeight) + // { + // if (pair.Item2 < 0) throw new ArgumentOutOfRangeException(nameof(pair.Item2)); + // args.Add(pair.Item1); + // args.Add(pair.Item2); + // } + // return (await _db.ExecuteAsync(TDIGEST.ADD, args)).OKtoBoolean(); + // } + /// /// Estimate the fraction of all observations added which are <= value. /// diff --git a/tests/NRedisStack.Tests/Tdigest/TdigestTests.cs b/tests/NRedisStack.Tests/Tdigest/TdigestTests.cs index 2e46c406..55e2796c 100644 --- a/tests/NRedisStack.Tests/Tdigest/TdigestTests.cs +++ b/tests/NRedisStack.Tests/Tdigest/TdigestTests.cs @@ -90,163 +90,178 @@ public async Task TestCreateAndInfoAsync() } } - [Fact] - public void TestRank() - { - IDatabase db = redisFixture.Redis.GetDatabase(); - db.Execute("FLUSHALL"); - var tdigest = db.TDIGEST(); - - Assert.True(tdigest.Create("t-digest", 500)); - var tuples = new Tuple[20]; - for (int i = 0; i < 20; i++) - { - tuples[i] = new(i, 1); - } - Assert.True(tdigest.Add("t-digest", tuples)); - Assert.Equal(-1, tdigest.Rank("t-digest", -1)[0]); - Assert.Equal(1, tdigest.Rank("t-digest", 0)[0]); - Assert.Equal(11, tdigest.Rank("t-digest", 10)[0]); - Assert.Equal(new long[3] { -1, 20, 10 }, tdigest.Rank("t-digest", -20, 20, 9)); - } - - [Fact] - public async Task TestRankAsync() - { - IDatabase db = redisFixture.Redis.GetDatabase(); - db.Execute("FLUSHALL"); - var tdigest = db.TDIGEST(); - - Assert.True(tdigest.Create("t-digest", 500)); - var tuples = new Tuple[20]; - for (int i = 0; i < 20; i++) - { - tuples[i] = new(i, 1); - } - Assert.True(tdigest.Add("t-digest", tuples)); - Assert.Equal(-1, (await tdigest.RankAsync("t-digest", -1))[0]); - Assert.Equal(1, (await tdigest.RankAsync("t-digest", 0))[0]); - Assert.Equal(11, (await tdigest.RankAsync("t-digest", 10))[0]); - Assert.Equal(new long[3] { -1, 20, 10 }, await tdigest.RankAsync("t-digest", -20, 20, 9)); - } - - [Fact] - public void TestRevRank() - { - IDatabase db = redisFixture.Redis.GetDatabase(); - db.Execute("FLUSHALL"); - var tdigest = db.TDIGEST(); - - Assert.True(tdigest.Create("t-digest", 500)); - var tuples = new Tuple[20]; - for (int i = 0; i < 20; i++) - { - tuples[i] = new(i, 1); - } - - Assert.True(tdigest.Add("t-digest", tuples)); - Assert.Equal(-1, tdigest.RevRank("t-digest", 20)[0]); - Assert.Equal(20, tdigest.RevRank("t-digest", 0)[0]); - Assert.Equal(new long[3] { -1, 20, 10 }, tdigest.RevRank("t-digest", 21, 0, 10)); - } - - [Fact] - public async Task TestRevRankAsync() - { - IDatabase db = redisFixture.Redis.GetDatabase(); - db.Execute("FLUSHALL"); - var tdigest = db.TDIGEST(); - - Assert.True(tdigest.Create("t-digest", 500)); - var tuples = new Tuple[20]; - for (int i = 0; i < 20; i++) - { - tuples[i] = new(i, 1); - } - - Assert.True(tdigest.Add("t-digest", tuples)); - Assert.Equal(-1, (await tdigest.RevRankAsync("t-digest", 20))[0]); - Assert.Equal(20, (await tdigest.RevRankAsync("t-digest", 0))[0]); - Assert.Equal(new long[3] { -1, 20, 10 }, await tdigest.RevRankAsync("t-digest", 21, 0, 10)); - } + // [Fact] + // public void TestRank() + // { + // IDatabase db = redisFixture.Redis.GetDatabase(); + // db.Execute("FLUSHALL"); + // var tdigest = db.TDIGEST(); + + // Assert.True(tdigest.Create("t-digest", 500)); + // var tuples = new Tuple[20]; + // for (int i = 0; i < 20; i++) + // { + // tuples[i] = new(i, 1); + // } + // Assert.True(tdigest.Add("t-digest", tuples)); + // Assert.Equal(-1, tdigest.Rank("t-digest", -1)[0]); + // Assert.Equal(1, tdigest.Rank("t-digest", 0)[0]); + // Assert.Equal(11, tdigest.Rank("t-digest", 10)[0]); + // Assert.Equal(new long[3] { -1, 20, 10 }, tdigest.Rank("t-digest", -20, 20, 9)); + // } [Fact] - public void TestByRank() + public void TestRankCommands() { + //final String key = "ranks"; IDatabase db = redisFixture.Redis.GetDatabase(); db.Execute("FLUSHALL"); var tdigest = db.TDIGEST(); - - Assert.True(tdigest.Create("t-digest", 500)); - var tuples = new Tuple[10]; - for (int i = 1; i <= 10; i++) - { - tuples[i - 1] = new(i, 1); - } - Assert.True(tdigest.Add("t-digest", tuples)); - Assert.Equal(1, tdigest.ByRank("t-digest", 0)[0]); - Assert.Equal(10, tdigest.ByRank("t-digest", 9)[0]); - Assert.True(double.IsInfinity(tdigest.ByRank("t-digest", 100)[0])); - //Assert.Throws(() => tdigest.ByRank("t-digest", -1)[0]); - } - - [Fact] - public async Task TestByRankAsync() - { - IDatabase db = redisFixture.Redis.GetDatabase(); - db.Execute("FLUSHALL"); - var tdigest = db.TDIGEST(); - - Assert.True(tdigest.Create("t-digest", 500)); - var tuples = new Tuple[10]; - for (int i = 1; i <= 10; i++) - { - tuples[i - 1] = new(i, 1); - } - Assert.True(tdigest.Add("t-digest", tuples)); - Assert.Equal(1, (await tdigest.ByRankAsync("t-digest", 0))[0]); - Assert.Equal(10, (await tdigest.ByRankAsync("t-digest", 9))[0]); - Assert.True(double.IsInfinity((await tdigest.ByRankAsync("t-digest", 100))[0])); - } - - [Fact] - public void TestByRevRank() - { - IDatabase db = redisFixture.Redis.GetDatabase(); - db.Execute("FLUSHALL"); - var tdigest = db.TDIGEST(); - - Assert.True(tdigest.Create("t-digest", 500)); - var tuples = new Tuple[10]; - for (int i = 1; i <= 10; i++) - { - tuples[i - 1] = new(i, 1); - } - Assert.True(tdigest.Add("t-digest", tuples)); - Assert.Equal(10, tdigest.ByRevRank("t-digest", 0)[0]); - Assert.Equal(2, tdigest.ByRevRank("t-digest", 9)[0]); - Assert.True(double.IsInfinity(-tdigest.ByRevRank("t-digest", 100)[0])); - //Assert.Throws(() => tdigest.ByRank("t-digest", -1)[0]); - } - - [Fact] - public async Task TestByRevRankAsync() - { - IDatabase db = redisFixture.Redis.GetDatabase(); - db.Execute("FLUSHALL"); - var tdigest = db.TDIGEST(); - - Assert.True(tdigest.Create("t-digest", 500)); - var tuples = new Tuple[10]; - for (int i = 1; i <= 10; i++) - { - tuples[i - 1] = new(i, 1); - } - Assert.True(tdigest.Add("t-digest", tuples)); - Assert.Equal(10, (await tdigest.ByRevRankAsync("t-digest", 0))[0]); - Assert.Equal(2, (await tdigest.ByRevRankAsync("t-digest", 9))[0]); - Assert.True(double.IsInfinity(-(await tdigest.ByRevRankAsync("t-digest", 100))[0])); - } + tdigest.Create(key); + tdigest.Add(key, 2d, 3d, 5d); + Assert.Equal(new long[] { 1l, 2l }, tdigest.Rank(key, 2, 4)); + Assert.Equal(new long[] { 0, 1 }, tdigest.RevRank(key, 5, 4)); + Assert.Equal(new double[] { 2, 3 }, tdigest.ByRank(key, 0, 1)); + Assert.Equal(new double[] { 5, 3 }, tdigest.ByRevRank(key, 1, 2)); + } + + // [Fact] + // public async Task TestRankAsync() + // { + // IDatabase db = redisFixture.Redis.GetDatabase(); + // db.Execute("FLUSHALL"); + // var tdigest = db.TDIGEST(); + + // Assert.True(tdigest.Create("t-digest", 500)); + // var tuples = new Tuple[20]; + // for (int i = 0; i < 20; i++) + // { + // tuples[i] = new(i, 1); + // } + // Assert.True(tdigest.Add("t-digest", tuples)); + // Assert.Equal(-1, (await tdigest.RankAsync("t-digest", -1))[0]); + // Assert.Equal(1, (await tdigest.RankAsync("t-digest", 0))[0]); + // Assert.Equal(11, (await tdigest.RankAsync("t-digest", 10))[0]); + // Assert.Equal(new long[3] { -1, 20, 10 }, await tdigest.RankAsync("t-digest", -20, 20, 9)); + // } + + // [Fact] + // public void TestRevRank() + // { + // IDatabase db = redisFixture.Redis.GetDatabase(); + // db.Execute("FLUSHALL"); + // var tdigest = db.TDIGEST(); + + // Assert.True(tdigest.Create("t-digest", 500)); + // var tuples = new Tuple[20]; + // for (int i = 0; i < 20; i++) + // { + // tuples[i] = new(i, 1); + // } + + // Assert.True(tdigest.Add("t-digest", tuples)); + // Assert.Equal(-1, tdigest.RevRank("t-digest", 20)[0]); + // Assert.Equal(20, tdigest.RevRank("t-digest", 0)[0]); + // Assert.Equal(new long[3] { -1, 20, 10 }, tdigest.RevRank("t-digest", 21, 0, 10)); + // } + + // [Fact] + // public async Task TestRevRankAsync() + // { + // IDatabase db = redisFixture.Redis.GetDatabase(); + // db.Execute("FLUSHALL"); + // var tdigest = db.TDIGEST(); + + // Assert.True(tdigest.Create("t-digest", 500)); + // var tuples = new Tuple[20]; + // for (int i = 0; i < 20; i++) + // { + // tuples[i] = new(i, 1); + // } + + // Assert.True(tdigest.Add("t-digest", tuples)); + // Assert.Equal(-1, (await tdigest.RevRankAsync("t-digest", 20))[0]); + // Assert.Equal(20, (await tdigest.RevRankAsync("t-digest", 0))[0]); + // Assert.Equal(new long[3] { -1, 20, 10 }, await tdigest.RevRankAsync("t-digest", 21, 0, 10)); + // } + + // [Fact] + // public void TestByRank() + // { + // IDatabase db = redisFixture.Redis.GetDatabase(); + // db.Execute("FLUSHALL"); + // var tdigest = db.TDIGEST(); + + // Assert.True(tdigest.Create("t-digest", 500)); + // var tuples = new Tuple[10]; + // for (int i = 1; i <= 10; i++) + // { + // tuples[i - 1] = new(i, 1); + // } + // Assert.True(tdigest.Add("t-digest", tuples)); + // Assert.Equal(1, tdigest.ByRank("t-digest", 0)[0]); + // Assert.Equal(10, tdigest.ByRank("t-digest", 9)[0]); + // Assert.True(double.IsInfinity(tdigest.ByRank("t-digest", 100)[0])); + // //Assert.Throws(() => tdigest.ByRank("t-digest", -1)[0]); + // } + + // [Fact] + // public async Task TestByRankAsync() + // { + // IDatabase db = redisFixture.Redis.GetDatabase(); + // db.Execute("FLUSHALL"); + // var tdigest = db.TDIGEST(); + + // Assert.True(tdigest.Create("t-digest", 500)); + // var tuples = new Tuple[10]; + // for (int i = 1; i <= 10; i++) + // { + // tuples[i - 1] = new(i, 1); + // } + // Assert.True(tdigest.Add("t-digest", tuples)); + // Assert.Equal(1, (await tdigest.ByRankAsync("t-digest", 0))[0]); + // Assert.Equal(10, (await tdigest.ByRankAsync("t-digest", 9))[0]); + // Assert.True(double.IsInfinity((await tdigest.ByRankAsync("t-digest", 100))[0])); + // } + + // [Fact] + // public void TestByRevRank() + // { + // IDatabase db = redisFixture.Redis.GetDatabase(); + // db.Execute("FLUSHALL"); + // var tdigest = db.TDIGEST(); + + // Assert.True(tdigest.Create("t-digest", 500)); + // var tuples = new Tuple[10]; + // for (int i = 1; i <= 10; i++) + // { + // tuples[i - 1] = new(i, 1); + // } + // Assert.True(tdigest.Add("t-digest", tuples)); + // Assert.Equal(10, tdigest.ByRevRank("t-digest", 0)[0]); + // Assert.Equal(2, tdigest.ByRevRank("t-digest", 9)[0]); + // Assert.True(double.IsInfinity(-tdigest.ByRevRank("t-digest", 100)[0])); + // //Assert.Throws(() => tdigest.ByRank("t-digest", -1)[0]); + // } + + // [Fact] + // public async Task TestByRevRankAsync() + // { + // IDatabase db = redisFixture.Redis.GetDatabase(); + // db.Execute("FLUSHALL"); + // var tdigest = db.TDIGEST(); + + // Assert.True(tdigest.Create("t-digest", 500)); + // var tuples = new Tuple[10]; + // for (int i = 1; i <= 10; i++) + // { + // tuples[i - 1] = new(i, 1); + // } + // Assert.True(tdigest.Add("t-digest", tuples)); + // Assert.Equal(10, (await tdigest.ByRevRankAsync("t-digest", 0))[0]); + // Assert.Equal(2, (await tdigest.ByRevRankAsync("t-digest", 9))[0]); + // Assert.True(double.IsInfinity(-(await tdigest.ByRevRankAsync("t-digest", 100))[0])); + // } [Fact] @@ -263,7 +278,8 @@ public void TestReset() Assert.True(tdigest.Reset("reset")); AssertMergedUnmergedNodes(tdigest, "reset", 0, 0); - tdigest.Add("reset", RandomValueWeight(), RandomValueWeight(), RandomValueWeight()); + // tdigest.Add("reset", RandomValue(), RandomValue(), RandomValue()); + tdigest.Add("reset", RandomValue(), RandomValue(), RandomValue()); AssertMergedUnmergedNodes(tdigest, "reset", 0, 3); Assert.True(tdigest.Reset("reset")); @@ -284,7 +300,9 @@ public async Task TestResetAsync() Assert.True(await tdigest.ResetAsync("reset")); AssertMergedUnmergedNodes(tdigest, "reset", 0, 0); - await tdigest.AddAsync("reset", RandomValueWeight(), RandomValueWeight(), RandomValueWeight()); + //await tdigest.AddAsync("reset", RandomValue(), RandomValue(), RandomValue()); + tdigest.Add("reset", RandomValue(), RandomValue(), RandomValue()); + AssertMergedUnmergedNodes(tdigest, "reset", 0, 3); Assert.True(await tdigest.ResetAsync("reset")); @@ -300,10 +318,10 @@ public void TestAdd() tdigest.Create("tdadd", 100); - Assert.True(tdigest.Add("tdadd", RandomValueWeight())); + Assert.True(tdigest.Add("tdadd", RandomValue())); AssertMergedUnmergedNodes(tdigest, "tdadd", 0, 1); - Assert.True(tdigest.Add("tdadd", RandomValueWeight(), RandomValueWeight(), RandomValueWeight(), RandomValueWeight())); + Assert.True(tdigest.Add("tdadd", RandomValue(), RandomValue(), RandomValue(), RandomValue())); AssertMergedUnmergedNodes(tdigest, "tdadd", 0, 5); } @@ -316,10 +334,10 @@ public async Task TestAddAsync() await tdigest.CreateAsync("tdadd", 100); - Assert.True(await tdigest.AddAsync("tdadd", RandomValueWeight())); + Assert.True(await tdigest.AddAsync("tdadd", RandomValue())); AssertMergedUnmergedNodes(tdigest, "tdadd", 0, 1); - Assert.True(await tdigest.AddAsync("tdadd", RandomValueWeight(), RandomValueWeight(), RandomValueWeight(), RandomValueWeight())); + Assert.True(await tdigest.AddAsync("tdadd", RandomValue(), RandomValue(), RandomValue(), RandomValue())); AssertMergedUnmergedNodes(tdigest, "tdadd", 0, 5); } @@ -336,8 +354,10 @@ public void TestMerge() Assert.True(tdigest.Merge("td2", sourceKeys: "td4m")); AssertMergedUnmergedNodes(tdigest, "td2", 0, 0); - tdigest.Add("td2", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); - tdigest.Add("td4m", DefinedValueWeight(1, 100), DefinedValueWeight(1, 100)); + // tdigest.Add("td2", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); + // tdigest.Add("td4m", DefinedValueWeight(1, 100), DefinedValueWeight(1, 100)); + tdigest.Add("td2", 1, 1, 1); + tdigest.Add("td4m", 1, 1); Assert.True(tdigest.Merge("td2", sourceKeys: "td4m")); AssertMergedUnmergedNodes(tdigest, "td2", 3, 2); @@ -357,8 +377,11 @@ public async Task TestMergeAsync() Assert.True(await tdigest.MergeAsync("td2", sourceKeys: "td4m")); AssertMergedUnmergedNodes(tdigest, "td2", 0, 0); - await tdigest.AddAsync("td2", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); - await tdigest.AddAsync("td4m", DefinedValueWeight(1, 100), DefinedValueWeight(1, 100)); + // await tdigest.AddAsync("td2", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); + // await tdigest.AddAsync("td4m", DefinedValueWeight(1, 100), DefinedValueWeight(1, 100)); + + await tdigest.AddAsync("td2", 1, 1, 1); + await tdigest.AddAsync("td4m", 1, 1); Assert.True(await tdigest.MergeAsync("td2", sourceKeys: "td4m")); AssertMergedUnmergedNodes(tdigest, "td2", 3, 2); @@ -373,8 +396,10 @@ public void MergeMultiAndParams() tdigest.Create("from1", 100); tdigest.Create("from2", 200); - tdigest.Add("from1", 1d, 1); - tdigest.Add("from2", 1d, 10); + // tdigest.Add("from1", 1d, 1); + // tdigest.Add("from2", 1d, 10); + tdigest.Add("from1", 1d); + tdigest.Add("from2", WeightedValue(1d, 10)); Assert.True(tdigest.Merge("to", 2, sourceKeys: new RedisKey[] { "from1", "from2" })); AssertTotalWeight(tdigest, "to", 11d); @@ -392,8 +417,10 @@ public async Task MergeMultiAndParamsAsync() tdigest.Create("from1", 100); tdigest.Create("from2", 200); - tdigest.Add("from1", 1d, 1); - tdigest.Add("from2", 1d, 10); + // tdigest.Add("from1", 1d, 1); + // tdigest.Add("from2", 1d, 10); + tdigest.Add("from1", 1d); + tdigest.Add("from2", WeightedValue(1d, 10)); Assert.True(await tdigest.MergeAsync("to", 2, sourceKeys: new RedisKey[] { "from1", "from2" })); AssertTotalWeight(tdigest, "to", 11d); @@ -415,9 +442,13 @@ public void TestCDF() Assert.Equal(double.NaN, item); } - tdigest.Add("tdcdf", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); - tdigest.Add("tdcdf", DefinedValueWeight(100, 1), DefinedValueWeight(100, 1)); + // tdigest.Add("tdcdf", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); + // tdigest.Add("tdcdf", DefinedValueWeight(100, 1), DefinedValueWeight(100, 1)); + + tdigest.Add("tdcdf", 1, 1, 1); + tdigest.Add("tdcdf", 100, 100); Assert.Equal(new double[] { 0.6 }, tdigest.CDF("tdcdf", 50)); + tdigest.CDF("tdcdf", 25, 50, 75); // TODO: Why needed? } [Fact] @@ -433,9 +464,14 @@ public async Task TestCDFAsync() Assert.Equal(double.NaN, item); } - await tdigest.AddAsync("tdcdf", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); - await tdigest.AddAsync("tdcdf", DefinedValueWeight(100, 1), DefinedValueWeight(100, 1)); + // await tdigest.AddAsync("tdcdf", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); + // await tdigest.AddAsync("tdcdf", DefinedValueWeight(100, 1), DefinedValueWeight(100, 1)); + tdigest.Add("tdcdf", 1, 1, 1); + tdigest.Add("tdcdf", 100, 100); + Assert.Equal(new double[] { 0.6 }, await tdigest.CDFAsync("tdcdf", 50)); + await tdigest.CDFAsync("tdcdf", 25, 50, 75); // TODO: Why needed? + } [Fact] @@ -449,8 +485,10 @@ public void TestQuantile() var resDelete = tdigest.Quantile("tdqnt", 0.5); Assert.Equal(new double[] { double.NaN }, tdigest.Quantile("tdqnt", 0.5)); - tdigest.Add("tdqnt", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); - tdigest.Add("tdqnt", DefinedValueWeight(100, 1), DefinedValueWeight(100, 1)); + // tdigest.Add("tdqnt", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); + // tdigest.Add("tdqnt", DefinedValueWeight(100, 1), DefinedValueWeight(100, 1)); + tdigest.Add("tdqnt", 1, 1, 1); + tdigest.Add("tdqnt", 100, 100); Assert.Equal(new double[] { 1 }, tdigest.Quantile("tdqnt", 0.5)); } @@ -465,8 +503,10 @@ public async Task TestQuantileAsync() var resDelete = await tdigest.QuantileAsync("tdqnt", 0.5); Assert.Equal(new double[] { double.NaN }, await tdigest.QuantileAsync("tdqnt", 0.5)); - await tdigest.AddAsync("tdqnt", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); - await tdigest.AddAsync("tdqnt", DefinedValueWeight(100, 1), DefinedValueWeight(100, 1)); + // await tdigest.AddAsync("tdqnt", DefinedValueWeight(1, 1), DefinedValueWeight(1, 1), DefinedValueWeight(1, 1)); + // await tdigest.AddAsync("tdqnt", DefinedValueWeight(100, 1), DefinedValueWeight(100, 1)); + tdigest.Add("tdqnt", 1, 1, 1); + tdigest.Add("tdqnt", 100, 100); Assert.Equal(new double[] { 1 }, await tdigest.QuantileAsync("tdqnt", 0.5)); } @@ -481,8 +521,10 @@ public void TestMinAndMax() Assert.Equal(double.NaN, tdigest.Min(key)); Assert.Equal(double.NaN, tdigest.Max(key)); - tdigest.Add(key, DefinedValueWeight(2, 1)); - tdigest.Add(key, DefinedValueWeight(5, 1)); + // tdigest.Add(key, DefinedValueWeight(2, 1)); + // tdigest.Add(key, DefinedValueWeight(5, 1)); + tdigest.Add(key, 2); + tdigest.Add(key, 5); Assert.Equal(2d, tdigest.Min(key)); Assert.Equal(5d, tdigest.Max(key)); } @@ -498,8 +540,10 @@ public async Task TestMinAndMaxAsync() Assert.Equal(double.NaN, await tdigest.MinAsync(key)); Assert.Equal(double.NaN, await tdigest.MaxAsync(key)); - await tdigest.AddAsync(key, DefinedValueWeight(2, 1)); - await tdigest.AddAsync(key, DefinedValueWeight(5, 1)); + // await tdigest.AddAsync(key, DefinedValueWeight(2, 1)); + // await tdigest.AddAsync(key, DefinedValueWeight(5, 1)); + tdigest.Add(key, 2); + tdigest.Add(key, 5); Assert.Equal(2d, await tdigest.MinAsync(key)); Assert.Equal(5d, await tdigest.MaxAsync(key)); } @@ -515,7 +559,8 @@ public void TestTrimmedMean() for (int i = 0; i < 20; i++) { - tdigest.Add(key, new Tuple(i, 1)); + //tdigest.Add(key, new Tuple(i, 1)); + tdigest.Add(key, i); } Assert.Equal(9.5, tdigest.TrimmedMean(key, 0.1, 0.9)); @@ -535,7 +580,8 @@ public async Task TestTrimmedMeanAsync() for (int i = 0; i < 20; i++) { - await tdigest.AddAsync(key, new Tuple(i, 1)); + // await tdigest.AddAsync(key, new Tuple(i, 1)); + tdigest.Add(key, i); } Assert.Equal(9.5, await tdigest.TrimmedMeanAsync(key, 0.1, 0.9)); @@ -579,6 +625,11 @@ public void TestModulePrefixs1() } } + private static double RandomValue() + { + Random random = new Random(); + return random.NextDouble() * 10000; + } static Tuple RandomValueWeight() { @@ -601,4 +652,11 @@ static Tuple DefinedValueWeight(double value, long weight) { return new Tuple(value, weight); } + + private static double[] WeightedValue(double value, int weight) + { + double[] values = new double[weight]; + Array.Fill(values, value); + return values; + } }