Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Very early prototype of ExplainOptions for Datastore #14081

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Very early prototype of ExplainOptions for Datastore #14081

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2caa892
Very early prototype of ExplainOptions for Datastore
jskeet Jan 10, 2025
a5bab39
Fix infinite loop in tests
jskeet Jan 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions apis/Google.Cloud.Datastore.V1/Google.Cloud.Datastore.V1.IntegrationTests/DatastoreDbTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,58 @@ public void SyncQueries()
});
}

[Fact]
public void Explain()
{
var db = _fixture.CreateDatastoreDb();
var keyFactory = db.CreateKeyFactory("syncqueries_explain");
using (var transaction = db.BeginTransaction())
{
var entities = Enumerable.Range(0, 5)
.Select(x => new Entity { Key = keyFactory.CreateIncompleteKey(), ["x"] = x })
.ToList();
transaction.Insert(entities);
transaction.Commit();
}

var query = new Query("syncqueries") { Filter = Filter.LessThan("x", 3) };
var gql = new GqlQuery { QueryString = "SELECT * FROM syncqueries WHERE x < 3", AllowLiterals = true };

_fixture.RetryQuery(() =>
{
var results = db.RunQuery(new AdvancedQuery { GqlQuery = gql, ExplainOptions = new ExplainOptions { Analyze = false } });
Assert.NotNull(results.PlanSummary);
Assert.Null(results.ExecutionStats);
Assert.Empty(results.Entities);
});
}

[Fact]
public void ExplainAnalyze()
{
var db = _fixture.CreateDatastoreDb();
var keyFactory = db.CreateKeyFactory("syncqueries_explain");
using (var transaction = db.BeginTransaction())
{
var entities = Enumerable.Range(0, 5)
.Select(x => new Entity { Key = keyFactory.CreateIncompleteKey(), ["x"] = x })
.ToList();
transaction.Insert(entities);
transaction.Commit();
}

var query = new Query("syncqueries") { Filter = Filter.LessThan("x", 3) };
var gql = new GqlQuery { QueryString = "SELECT * FROM syncqueries WHERE x < 3", AllowLiterals = true };

_fixture.RetryQuery(() =>
{
var results = db.RunQuery(new AdvancedQuery { GqlQuery = gql, ExplainOptions = new ExplainOptions { Analyze = true } });
Assert.NotNull(results.PlanSummary);
Assert.NotNull(results.ExecutionStats);
Assert.NotEmpty(results.Entities);
});
}

[Fact]
public async Task AsyncQueries()
{
Expand Down
78 changes: 42 additions & 36 deletions apis/Google.Cloud.Datastore.V1/Google.Cloud.Datastore.V1.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,47 +3,53 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.2.32516.85
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Google.Cloud.Datastore.V1", "Google.Cloud.Datastore.V1\Google.Cloud.Datastore.V1.csproj", "{8D442A59-2660-609D-33A9-844FA37132AA}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Google.Cloud.Datastore.V1", "Google.Cloud.Datastore.V1\Google.Cloud.Datastore.V1.csproj", "{8D442A59-2660-609D-33A9-844FA37132AA}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Google.Cloud.Datastore.V1.Snippets", "Google.Cloud.Datastore.V1.Snippets\Google.Cloud.Datastore.V1.Snippets.csproj", "{B05695A0-022D-652A-8FBE-BD5976085F58}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Google.Cloud.Datastore.V1.Snippets", "Google.Cloud.Datastore.V1.Snippets\Google.Cloud.Datastore.V1.Snippets.csproj", "{B05695A0-022D-652A-8FBE-BD5976085F58}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Google.Cloud.Datastore.V1.GeneratedSnippets", "Google.Cloud.Datastore.V1.GeneratedSnippets\Google.Cloud.Datastore.V1.GeneratedSnippets.csproj", "{D71882DB-B962-6046-E4D2-76BFADC6C0C4}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Google.Cloud.Datastore.V1.GeneratedSnippets", "Google.Cloud.Datastore.V1.GeneratedSnippets\Google.Cloud.Datastore.V1.GeneratedSnippets.csproj", "{D71882DB-B962-6046-E4D2-76BFADC6C0C4}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Google.Cloud.Datastore.V1.IntegrationTests", "Google.Cloud.Datastore.V1.IntegrationTests\Google.Cloud.Datastore.V1.IntegrationTests.csproj", "{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Google.Cloud.Datastore.V1.IntegrationTests", "Google.Cloud.Datastore.V1.IntegrationTests\Google.Cloud.Datastore.V1.IntegrationTests.csproj", "{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Google.Cloud.Datastore.V1.Tests", "Google.Cloud.Datastore.V1.Tests\Google.Cloud.Datastore.V1.Tests.csproj", "{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Google.Cloud.Datastore.V1.Tests", "Google.Cloud.Datastore.V1.Tests\Google.Cloud.Datastore.V1.Tests.csproj", "{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Google.Cloud.ClientTesting", "..\..\tools\Google.Cloud.ClientTesting\Google.Cloud.ClientTesting.csproj", "{29974B0C-A7B0-8CA8-AE32-99F622C89044}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Google.Cloud.ClientTesting", "..\..\tools\Google.Cloud.ClientTesting\Google.Cloud.ClientTesting.csproj", "{29974B0C-A7B0-8CA8-AE32-99F622C89044}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{8D442A59-2660-609D-33A9-844FA37132AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8D442A59-2660-609D-33A9-844FA37132AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8D442A59-2660-609D-33A9-844FA37132AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8D442A59-2660-609D-33A9-844FA37132AA}.Release|Any CPU.Build.0 = Release|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Release|Any CPU.Build.0 = Release|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Release|Any CPU.Build.0 = Release|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Release|Any CPU.Build.0 = Release|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Release|Any CPU.Build.0 = Release|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Debug|Any CPU.Build.0 = Debug|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Release|Any CPU.ActiveCfg = Release|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{8D442A59-2660-609D-33A9-844FA37132AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8D442A59-2660-609D-33A9-844FA37132AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8D442A59-2660-609D-33A9-844FA37132AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8D442A59-2660-609D-33A9-844FA37132AA}.Release|Any CPU.Build.0 = Release|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B05695A0-022D-652A-8FBE-BD5976085F58}.Release|Any CPU.Build.0 = Release|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D71882DB-B962-6046-E4D2-76BFADC6C0C4}.Release|Any CPU.Build.0 = Release|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6A1CB7FE-3F24-EBBB-28A1-0F64ED124D81}.Release|Any CPU.Build.0 = Release|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0E4BEA11-EB67-2BF9-1A53-69D3CE17FC14}.Release|Any CPU.Build.0 = Release|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Debug|Any CPU.Build.0 = Debug|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Release|Any CPU.ActiveCfg = Release|Any CPU
{29974B0C-A7B0-8CA8-AE32-99F622C89044}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {69E8BF28-102E-4CFC-BB48-6543F4E335CC}
EndGlobalSection
EndGlobal
72 changes: 72 additions & 0 deletions apis/Google.Cloud.Datastore.V1/Google.Cloud.Datastore.V1/AdvancedQuery.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License"):
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

using System;
using System.Collections;
using System.Collections.Generic;
using System.Text;
using static Google.Cloud.Datastore.V1.ReadOptions.Types;

namespace Google.Cloud.Datastore.V1;

/// <summary>
/// Prototype only!
/// </summary>
public class AdvancedQuery
{
/// <summary>
///
/// </summary>
public GqlQuery GqlQuery { get; set; }

/// <summary>
///
/// </summary>
public Query Query { get; set; }

/// <summary>
///
/// </summary>
public ExplainOptions ExplainOptions { get; set; }

/// <summary>
///
/// </summary>
public ReadConsistency? ReadConsistency { get; set; }

internal RunQueryRequest ToRequest(string projectId, string databaseId, PartitionId partitionId)
{
var request = new RunQueryRequest
{
ProjectId = projectId,
DatabaseId = databaseId,
PartitionId = partitionId,
ReadOptions = GetReadOptions(ReadConsistency),
ExplainOptions = ExplainOptions
};
// TODO: Validation of exactly one being set, or make the properties behave like a oneof.
if (GqlQuery is not null)
{
request.GqlQuery = GqlQuery;
}
if (Query is not null)
{
request.Query = Query;
}
return request;
}

private static ReadOptions GetReadOptions(ReadConsistency? readConsistency) =>
readConsistency == null ? null : new ReadOptions { ReadConsistency = readConsistency.Value };
}
8 changes: 5 additions & 3 deletions apis/Google.Cloud.Datastore.V1/Google.Cloud.Datastore.V1/AsyncLazyDatastoreQuery.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2016 Google Inc. All Rights Reserved.
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -18,6 +18,7 @@
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using static Google.Cloud.Datastore.V1.QueryResultBatch.Types;

namespace Google.Cloud.Datastore.V1
{
Expand Down Expand Up @@ -63,9 +64,10 @@ public async Task<DatastoreQueryResults> GetAllResultsAsync()
// cursors etc, but that's likely to be insignificant compared with the
// entity data, and is immediately eligible for garbage collection.
var responses = await _responses.ToListAsync().ConfigureAwait(false);
var entities = responses.SelectMany(r => r.Batch.EntityResults.Select(er => er.Entity)).ToList().AsReadOnly();
var entities = responses.SelectMany(r => r.Batch?.EntityResults.Select(er => er.Entity) ?? Enumerable.Empty<Entity>()).ToList().AsReadOnly();
var lastBatch = responses.Last().Batch;
return new DatastoreQueryResults(entities, lastBatch.MoreResults, lastBatch.EndCursor);
var metrics = responses.Last().ExplainMetrics;
return new DatastoreQueryResults(entities, lastBatch?.MoreResults ?? MoreResultsType.Unspecified, lastBatch?.EndCursor, metrics);
}

/// <inheritdoc />
Expand Down
56 changes: 56 additions & 0 deletions apis/Google.Cloud.Datastore.V1/Google.Cloud.Datastore.V1/DatastoreDb.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,18 @@ public virtual KeyFactory CreateKeyFactory(string kind)
throw new NotImplementedException();
}

/// <summary>
/// Runs the given query eagerly, retrieving all results in memory and indicating whether more
/// results may be available beyond the query's limit. Use this method when your query has a limited
/// number of results, for example to build a web application which fetches results in pages.
/// </summary>
/// <param name="query">The query to execute. Must not be null.</param>
/// <param name="callSettings">If not null, applies overrides to RPC calls.</param>
/// <returns>The complete query results.</returns>
public virtual DatastoreQueryResults RunQuery(
AdvancedQuery query, CallSettings callSettings = null) =>
RunQueryLazily(query, callSettings).GetAllResults();

/// <summary>
/// Runs the given query eagerly, retrieving all results in memory and indicating whether more
/// results may be available beyond the query's limit. Use this method when your query has a limited
Expand All @@ -115,6 +127,17 @@ public virtual DatastoreQueryResults RunQuery(
Query query, ReadConsistency? readConsistency = null, CallSettings callSettings = null) =>
RunQueryLazily(query, readConsistency, callSettings).GetAllResults();

/// <summary>
/// Runs the given query eagerly and asynchronously, retrieving all results in memory and indicating whether more
/// results may be available beyond the query's limit. Use this method when your query has a limited
/// number of results, for example to build a web application which fetches results in pages.
/// </summary>
/// <param name="query">The query to execute. Must not be null.</param>
/// <param name="callSettings">If not null, applies overrides to RPC calls.</param>
/// <returns>A task representing the asynchronous operation. The result of the task is the complete set of query results.</returns>
public virtual Task<DatastoreQueryResults> RunQueryAsync(AdvancedQuery query, CallSettings callSettings = null) =>
RunQueryLazilyAsync(query, callSettings).GetAllResultsAsync();

/// <summary>
/// Runs the given query eagerly and asynchronously, retrieving all results in memory and indicating whether more
/// results may be available beyond the query's limit. Use this method when your query has a limited
Expand Down Expand Up @@ -176,6 +199,23 @@ public virtual Task<AggregationQueryResults> RunAggregationQueryAsync(GqlQuery q
throw new NotImplementedException();
}

/// <summary>
/// Lazily executes the given structured query.
/// </summary>
/// <remarks>
/// The results are requested lazily: no API calls will be made until the application starts
/// iterating over the results. Iterating over the same <see cref="LazyDatastoreQuery"/> object
/// multiple times will execute the query again, potentially returning different results.
/// </remarks>
/// <param name="query">The query to execute. Must not be null.</param>
/// <param name="callSettings">If not null, applies overrides to RPC calls.</param>
/// <returns>A <see cref="LazyDatastoreQuery"/> representing the lazy query results.</returns>
public virtual LazyDatastoreQuery RunQueryLazily(
AdvancedQuery query, CallSettings callSettings = null)
{
throw new NotImplementedException();
}

/// <summary>
/// Lazily executes the given structured query.
/// </summary>
Expand All @@ -194,6 +234,22 @@ public virtual LazyDatastoreQuery RunQueryLazily(
throw new NotImplementedException();
}

/// <summary>
/// Lazily executes the given structured query for asynchronous consumption.
/// </summary>
/// <remarks>
/// The results are requested lazily: no API calls will be made until the application starts
/// iterating over the results. Iterating over the same <see cref="LazyDatastoreQuery"/> object
/// multiple times will execute the query again, potentially returning different results.
/// </remarks>
/// <param name="query">The query to execute. Must not be null.</param>
/// <param name="callSettings">If not null, applies overrides to RPC calls.</param>
/// <returns>An <see cref="AsyncLazyDatastoreQuery"/> representing the lazy query results.</returns>
public virtual AsyncLazyDatastoreQuery RunQueryLazilyAsync(AdvancedQuery query, CallSettings callSettings = null)
{
throw new NotImplementedException();
}

/// <summary>
/// Lazily executes the given structured query for asynchronous consumption.
/// </summary>
Expand Down
20 changes: 20 additions & 0 deletions apis/Google.Cloud.Datastore.V1/Google.Cloud.Datastore.V1/DatastoreDbImpl.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
using Google.Api.Gax;
using Google.Api.Gax.Grpc;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using static Google.Cloud.Datastore.V1.CommitRequest.Types;
Expand Down Expand Up @@ -215,6 +217,24 @@ public override AsyncLazyDatastoreQuery RunQueryLazilyAsync(
return new AsyncLazyDatastoreQuery(streamer.Async());
}

/// <inheritdoc/>
public override LazyDatastoreQuery RunQueryLazily(AdvancedQuery query, CallSettings callSettings = null)
{
GaxPreconditions.CheckNotNull(query, nameof(query));
var request = query.ToRequest(ProjectId, DatabaseId, _partitionId);
var streamer = new QueryStreamer(request, Client.RunQueryApiCall, callSettings);
return new LazyDatastoreQuery(streamer.Sync());
}

/// <inheritdoc/>
public override AsyncLazyDatastoreQuery RunQueryLazilyAsync(AdvancedQuery query, CallSettings callSettings = null)
{
GaxPreconditions.CheckNotNull(query, nameof(query));
var request = query.ToRequest(ProjectId, DatabaseId, _partitionId);
var streamer = new QueryStreamer(request, Client.RunQueryApiCall, callSettings);
return new AsyncLazyDatastoreQuery(streamer.Async());
}

/// <inheritdoc/>
public override DatastoreTransaction BeginTransaction(CallSettings callSettings = null)
{
Expand Down
Loading
Loading