Skip to content

Fix index creation issue when multiple options are used with fields #405

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 3, 2025
Merged

Fix index creation issue when multiple options are used with fields #405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3ad17c3
- change order of field options in search&query
atakavci Apr 2, 2025
f69d126
fix search tests
atakavci Apr 2, 2025
04e3300
fix format
atakavci Apr 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 46 additions & 9 deletions src/NRedisStack/Search/Schema.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,11 +97,10 @@ internal override void AddFieldTypeArgs(List<object> args)
AddPhonetic(args);
AddWeight(args);
if (WithSuffixTrie) args.Add(SearchArgs.WITHSUFFIXTRIE);
if (Sortable) args.Add(FieldOptions.SORTABLE);
if (Unf) args.Add(SearchArgs.UNF);
if (MissingIndex) args.Add(FieldOptions.INDEXMISSING);
if (EmptyIndex) args.Add(FieldOptions.INDEXEMPTY);

if (Sortable) args.Add(FieldOptions.SORTABLE);
}

private void AddWeight(List<object> args)
Expand Down Expand Up @@ -165,10 +164,10 @@ internal override void AddFieldTypeArgs(List<object> args)
args.Add(Separator);
}
if (CaseSensitive) args.Add(SearchArgs.CASESENSITIVE);
if (Sortable) args.Add(FieldOptions.SORTABLE);
if (Unf) args.Add(SearchArgs.UNF);
if (MissingIndex) args.Add(FieldOptions.INDEXMISSING);
if (EmptyIndex) args.Add(FieldOptions.INDEXEMPTY);
if (Sortable) args.Add(FieldOptions.SORTABLE);
}
}

Expand All @@ -192,10 +191,9 @@ internal GeoField(string name, bool sortable = false, bool noIndex = false, bool
internal override void AddFieldTypeArgs(List<object> args)
{
if (NoIndex) args.Add(SearchArgs.NOINDEX);
if (Sortable) args.Add(FieldOptions.SORTABLE);
if (MissingIndex) args.Add(FieldOptions.INDEXMISSING);
if (Sortable) args.Add(FieldOptions.SORTABLE);
}

}

public class GeoShapeField : Field
Expand Down Expand Up @@ -252,10 +250,9 @@ internal NumericField(string name, bool sortable = false, bool noIndex = false,
internal override void AddFieldTypeArgs(List<object> args)
{
if (NoIndex) args.Add(SearchArgs.NOINDEX);
if (Sortable) args.Add(FieldOptions.SORTABLE);
if (MissingIndex) args.Add(FieldOptions.INDEXMISSING);
if (Sortable) args.Add(FieldOptions.SORTABLE);
}

}

public class VectorField : Field
Expand Down Expand Up @@ -322,6 +319,10 @@ public Schema AddField(Field field)
/// <param name="unf">Set this to true to prevent the indexer from sorting on the normalized form.
/// Normalied form is the field sent to lower case with all diaretics removed</param>
/// <param name="withSuffixTrie">Keeps a suffix trie with all terms which match the suffix.</param>
/// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field.
/// Note the difference between a field with an empty value and a document with a missing value.
/// By default, missing values are not indexed.</param>
/// <param name="emptyIndex"> allows you to index and search for empty strings. By default, empty strings are not indexed.</param>
/// <returns>The <see cref="Schema"/> object.</returns>
public Schema AddTextField(string name, double weight = 1.0, bool sortable = false, bool unf = false, bool noStem = false,
string? phonetic = null, bool noIndex = false, bool withSuffixTrie = false, bool missingIndex = false, bool emptyIndex = false)
Expand All @@ -342,6 +343,10 @@ public Schema AddTextField(string name, double weight = 1.0, bool sortable = fal
/// <param name="unf">Set this to true to prevent the indexer from sorting on the normalized form.
/// Normalied form is the field sent to lower case with all diaretics removed</param>
/// <param name="withSuffixTrie">Keeps a suffix trie with all terms which match the suffix.</param>
/// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field.
/// Note the difference between a field with an empty value and a document with a missing value.
/// By default, missing values are not indexed.</param>
/// <param name="emptyIndex"> allows you to index and search for empty strings. By default, empty strings are not indexed.</param>
/// <returns>The <see cref="Schema"/> object.</returns>
public Schema AddTextField(FieldName name, double weight = 1.0, bool sortable = false, bool unf = false, bool noStem = false,
string? phonetic = null, bool noIndex = false, bool withSuffixTrie = false, bool missingIndex = false, bool emptyIndex = false)
Expand All @@ -355,6 +360,9 @@ public Schema AddTextField(FieldName name, double weight = 1.0, bool sortable =
/// </summary>
/// <param name="name">The field's name.</param>
/// <param name="system">The coordinate system to use.</param>
/// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field.
/// Note the difference between a field with an empty value and a document with a missing value.
/// By default, missing values are not indexed.</param>
/// <returns>The <see cref="Schema"/> object.</returns>
public Schema AddGeoShapeField(string name, CoordinateSystem system, bool missingIndex = false)
{
Expand All @@ -367,6 +375,9 @@ public Schema AddGeoShapeField(string name, CoordinateSystem system, bool missin
/// </summary>
/// <param name="name">The field's name.</param>
/// <param name="system">The coordinate system to use.</param>
/// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field.
/// Note the difference between a field with an empty value and a document with a missing value.
/// By default, missing values are not indexed.</param>
/// <returns>The <see cref="Schema"/> object.</returns>
public Schema AddGeoShapeField(FieldName name, CoordinateSystem system, bool missingIndex = false)
{
Expand All @@ -380,6 +391,9 @@ public Schema AddGeoShapeField(FieldName name, CoordinateSystem system, bool mis
/// <param name="name">The field's name.</param>
/// <param name="sortable">If true, the text field can be sorted.</param>
/// <param name="noIndex">Attributes can have the NOINDEX option, which means they will not be indexed.</param>
/// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field.
/// Note the difference between a field with an empty value and a document with a missing value.
/// By default, missing values are not indexed.</param>
/// <returns>The <see cref="Schema"/> object.</returns>
public Schema AddGeoField(FieldName name, bool sortable = false, bool noIndex = false, bool missingIndex = false)
{
Expand All @@ -393,6 +407,9 @@ public Schema AddGeoField(FieldName name, bool sortable = false, bool noIndex =
/// <param name="name">The field's name.</param>
/// <param name="sortable">If true, the text field can be sorted.</param>
/// <param name="noIndex">Attributes can have the NOINDEX option, which means they will not be indexed.</param>
/// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field.
/// Note the difference between a field with an empty value and a document with a missing value.
/// By default, missing values are not indexed.</param>
/// <returns>The <see cref="Schema"/> object.</returns>
public Schema AddGeoField(string name, bool sortable = false, bool noIndex = false, bool missingIndex = false)
{
Expand All @@ -406,6 +423,9 @@ public Schema AddGeoField(string name, bool sortable = false, bool noIndex = fal
/// <param name="name">The field's name.</param>
/// <param name="sortable">If true, the text field can be sorted.</param>
/// <param name="noIndex">Attributes can have the NOINDEX option, which means they will not be indexed.</param>
/// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field.
/// Note the difference between a field with an empty value and a document with a missing value.
/// By default, missing values are not indexed.</param>
/// <returns>The <see cref="Schema"/> object.</returns>
public Schema AddNumericField(FieldName name, bool sortable = false, bool noIndex = false, bool missingIndex = false)
{
Expand All @@ -419,6 +439,9 @@ public Schema AddNumericField(FieldName name, bool sortable = false, bool noInde
/// <param name="name">The field's name.</param>
/// <param name="sortable">If true, the text field can be sorted.</param>
/// <param name="noIndex">Attributes can have the NOINDEX option, which means they will not be indexed.</param>
/// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field.
/// Note the difference between a field with an empty value and a document with a missing value.
/// By default, missing values are not indexed.</param>
/// <returns>The <see cref="Schema"/> object.</returns>
public Schema AddNumericField(string name, bool sortable = false, bool noIndex = false, bool missingIndex = false)
{
Expand All @@ -437,6 +460,10 @@ public Schema AddNumericField(string name, bool sortable = false, bool noIndex =
/// <param name="caseSensitive">If true, Keeps the original letter cases of the tags.</param>
/// Normalied form is the field sent to lower case with all diaretics removed</param>
/// <param name="withSuffixTrie">Keeps a suffix trie with all terms which match the suffix.</param>
/// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field.
/// Note the difference between a field with an empty value and a document with a missing value.
/// By default, missing values are not indexed.</param>
/// <param name="emptyIndex"> allows you to index and search for empty strings. By default, empty strings are not indexed.</param>
/// <returns>The <see cref="Schema"/> object.</returns>
public Schema AddTagField(FieldName name, bool sortable = false, bool unf = false,
bool noIndex = false, string separator = ",",
Expand All @@ -457,6 +484,10 @@ public Schema AddTagField(FieldName name, bool sortable = false, bool unf = fals
/// <param name="caseSensitive">If true, Keeps the original letter cases of the tags.</param>
/// Normalied form is the field sent to lower case with all diaretics removed</param>
/// <param name="withSuffixTrie">Keeps a suffix trie with all terms which match the suffix.</param>
/// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field.
/// Note the difference between a field with an empty value and a document with a missing value.
/// By default, missing values are not indexed.</param>
/// <param name="emptyIndex"> allows you to index and search for empty strings. By default, empty strings are not indexed.</param>
/// <returns>The <see cref="Schema"/> object.</returns>
public Schema AddTagField(string name, bool sortable = false, bool unf = false,
bool noIndex = false, string separator = ",",
Expand All @@ -471,7 +502,10 @@ public Schema AddTagField(string name, bool sortable = false, bool unf = false,
/// </summary>
/// <param name="name">The field's name.</param>
/// <param name="algorithm">The vector similarity algorithm to use.</param>
/// <param name="attribute">The algorithm attributes for the creation of the vector index.</param>
/// <param name="attributes">The algorithm attributes for the creation of the vector index.</param>
/// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field.
/// Note the difference between a field with an empty value and a document with a missing value.
/// By default, missing values are not indexed.</param>
/// <returns>The <see cref="Schema"/> object.</returns>
public Schema AddVectorField(FieldName name, VectorAlgo algorithm, Dictionary<string, object>? attributes = null, bool missingIndex = false)
{
Expand All @@ -484,7 +518,10 @@ public Schema AddVectorField(FieldName name, VectorAlgo algorithm, Dictionary<st
/// </summary>
/// <param name="name">The field's name.</param>
/// <param name="algorithm">The vector similarity algorithm to use.</param>
/// <param name="attribute">The algorithm attributes for the creation of the vector index.</param>
/// <param name="attributes">The algorithm attributes for the creation of the vector index.</param>
/// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field.
/// Note the difference between a field with an empty value and a document with a missing value.
/// By default, missing values are not indexed.</param>
/// <returns>The <see cref="Schema"/> object.</returns>
public Schema AddVectorField(string name, VectorAlgo algorithm, Dictionary<string, object>? attributes = null, bool missingIndex = false)
{
Expand Down
81 changes: 81 additions & 0 deletions tests/NRedisStack.Tests/Search/IndexCreationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public IndexCreationTests(EndpointsFixture endpointsFixture) : base(endpointsFix

private static readonly string INDEXMISSING = "INDEXMISSING";
private static readonly string INDEXEMPTY = "INDEXEMPTY";
private static readonly string SORTABLE = "SORTABLE";

[Fact]
public void TestMissingEmptyFieldCommandArgs()
Expand Down Expand Up @@ -202,4 +203,84 @@ public void TestCreateInt8VectorField(string endpointId)
res = ft.Search("idx", q.AddParam("vec", vec2));
Assert.Equal(2, res.TotalResults);
}

[Fact]
public void TestMissingSortableFieldCommandArgs()
{
string idx = "MISSING_EMPTY_SORTABLE_INDEX";
Schema sc = new Schema()
.AddTextField("text1", 1.0, missingIndex: true, emptyIndex: true, sortable: true)
.AddTagField("tag1", missingIndex: true, emptyIndex: true, sortable: true)
.AddNumericField("numeric1", missingIndex: true, sortable: true)
.AddGeoField("geo1", missingIndex: true, sortable: true);

var ftCreateParams = FTCreateParams.CreateParams();

var cmd = SearchCommandBuilder.Create(idx, ftCreateParams, sc);
var expectedArgs = new object[] { idx, "SCHEMA",
"text1","TEXT",INDEXMISSING,INDEXEMPTY,SORTABLE,
"tag1","TAG", INDEXMISSING,INDEXEMPTY,SORTABLE,
"numeric1","NUMERIC", INDEXMISSING,SORTABLE,
"geo1","GEO", INDEXMISSING, SORTABLE};
Assert.Equal(expectedArgs, cmd.Args);
}

[SkipIfRedis(Comparison.LessThan, "7.3.240")]
[MemberData(nameof(EndpointsFixture.Env.StandaloneOnly), MemberType = typeof(EndpointsFixture.Env))]
public void TestCombiningMissingEmptySortableFields(string endpointId)
{
string idx = "MISSING_EMPTY_SORTABLE_INDEX";
IDatabase db = GetCleanDatabase(endpointId);
var ft = db.FT(2);
var vectorAttrs = new Dictionary<string, object>()
{
["TYPE"] = "FLOAT32",
["DIM"] = "2",
["DISTANCE_METRIC"] = "L2",
};
Schema sc = new Schema()
.AddTextField("text1", 1.0, missingIndex: true, emptyIndex: true, sortable: true)
.AddTagField("tag1", missingIndex: true, emptyIndex: true, sortable: true)
.AddNumericField("numeric1", missingIndex: true, sortable: true)
.AddGeoField("geo1", missingIndex: true, sortable: true)
.AddGeoShapeField("geoshape1", Schema.GeoShapeField.CoordinateSystem.FLAT, missingIndex: true)
.AddVectorField("vector1", Schema.VectorField.VectorAlgo.FLAT, vectorAttrs, missingIndex: true);

var ftCreateParams = FTCreateParams.CreateParams();
Assert.True(ft.Create(idx, ftCreateParams, sc));

var sampleHash = new HashEntry[] { new("field1", "value1"), new("field2", "value2") };
db.HashSet("hashWithMissingFields", sampleHash);

Polygon polygon = new GeometryFactory().CreatePolygon(new Coordinate[] { new Coordinate(1, 1), new Coordinate(10, 10), new Coordinate(100, 100), new Coordinate(1, 1), });

var hashWithAllFields = new HashEntry[] { new("text1", "value1"), new("tag1", "value2"), new("numeric1", "3.141"), new("geo1", "-0.441,51.458"), new("geoshape1", polygon.ToString()), new("vector1", "aaaaaaaa") };
db.HashSet("hashWithAllFields", hashWithAllFields);

var result = ft.Search(idx, new Query("ismissing(@text1)"));
Assert.Equal(1, result.TotalResults);
Assert.Equal("hashWithMissingFields", result.Documents[0].Id);

result = ft.Search(idx, new Query("ismissing(@tag1)"));
Assert.Equal(1, result.TotalResults);
Assert.Equal("hashWithMissingFields", result.Documents[0].Id);

result = ft.Search(idx, new Query("ismissing(@numeric1)"));
Assert.Equal(1, result.TotalResults);
Assert.Equal("hashWithMissingFields", result.Documents[0].Id);

result = ft.Search(idx, new Query("ismissing(@geo1)"));
Assert.Equal(1, result.TotalResults);
Assert.Equal("hashWithMissingFields", result.Documents[0].Id);

result = ft.Search(idx, new Query("ismissing(@geoshape1)"));
Assert.Equal(1, result.TotalResults);
Assert.Equal("hashWithMissingFields", result.Documents[0].Id);

result = ft.Search(idx, new Query("ismissing(@vector1)"));
Assert.Equal(1, result.TotalResults);
Assert.Equal("hashWithMissingFields", result.Documents[0].Id);
}


}
4 changes: 2 additions & 2 deletions tests/NRedisStack.Tests/Search/SearchTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2140,8 +2140,8 @@ public void TestFieldsCommandBuilder()
"PHONETIC",
"dm:en",
"WITHSUFFIXTRIE",
"SORTABLE",
"UNF",
"SORTABLE",
"num",
"NUMERIC",
"NOINDEX",
Expand All @@ -2157,8 +2157,8 @@ public void TestFieldsCommandBuilder()
"SEPARATOR",
";",
"CASESENSITIVE",
"SORTABLE",
"UNF",
"SORTABLE",
"vec",
"VECTOR",
"FLAT",
Expand Down
Loading