Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Optimizing Sitemap Creation with Batched Content Items #16643

Merged
merged 2 commits into from
Aug 31, 2024
Merged

Optimizing Sitemap Creation with Batched Content Items #16643

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ed8d018
Optimizing Sitemap Creation with Batched Content Items (#16636)
MikeAlhayek Aug 30, 2024
f8e0ba9
Optimizing Sitemap Creation with Batched Content Items
MikeAlhayek Aug 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/OrchardCore.Build/Dependencies.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
<PackageManagement Include="AWSSDK.SecurityToken" Version="3.7.101.60" />
<PackageManagement Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.3.0" />
<PackageManagement Include="Azure.Extensions.AspNetCore.DataProtection.Blobs" Version="1.3.2" />
<PackageManagement Include="Azure.Identity" Version="1.11.2" />
<PackageManagement Include="Azure.Identity" Version="1.12.0" />
<PackageManagement Include="Azure.Storage.Blobs" Version="12.19.1" />
<PackageManagement Include="BenchmarkDotNet" Version="0.13.11" />
<PackageManagement Include="Castle.Core" Version="5.1.1" />
Expand All @@ -39,7 +39,7 @@
<PackageManagement Include="Microsoft.Identity.Web" Version="2.16.0" />
<PackageManagement Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
<PackageManagement Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
<PackageManagement Include="MimeKit" Version="4.3.0" />
<PackageManagement Include="MimeKit" Version="4.7.1" />
<PackageManagement Include="MiniProfiler.AspNetCore.Mvc" Version="4.3.8" />
<PackageManagement Include="Moq" Version="4.20.70" />
<PackageManagement Include="ncrontab" Version="3.3.3" />
Expand Down
186 changes: 95 additions & 91 deletions ...re.Modules/OrchardCore.ContentLocalization/Sitemaps/LocalizedContentItemsQueryProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using OrchardCore.ContentLocalization.Models;
Expand All @@ -11,113 +13,115 @@
using YesSql;
using YesSql.Services;

namespace OrchardCore.ContentLocalization.Sitemaps
namespace OrchardCore.ContentLocalization.Sitemaps;

public class LocalizedContentItemsQueryProvider : IContentItemsQueryProvider
{
public class LocalizedContentItemsQueryProvider : IContentItemsQueryProvider
private readonly ISession _session;
private readonly IRouteableContentTypeCoordinator _routeableContentTypeCoordinator;
private readonly ILocalizationService _localizationService;

public LocalizedContentItemsQueryProvider(
ISession session,
IRouteableContentTypeCoordinator routeableContentTypeCoordinator,
ILocalizationService localizationService
)
{
private readonly ISession _session;
private readonly IRouteableContentTypeCoordinator _routeableContentTypeCoordinator;
private readonly ILocalizationService _localizationService;

public LocalizedContentItemsQueryProvider(
ISession session,
IRouteableContentTypeCoordinator routeableContentTypeCoordinator,
ILocalizationService localizationService
)
{
_session = session;
_routeableContentTypeCoordinator = routeableContentTypeCoordinator;
_localizationService = localizationService;
}
_session = session;
_routeableContentTypeCoordinator = routeableContentTypeCoordinator;
_localizationService = localizationService;
}

public async Task GetContentItemsAsync(ContentTypesSitemapSource source, ContentItemsQueryContext queryContext)
{
var routeableContentTypeDefinitions = await _routeableContentTypeCoordinator.ListRoutableTypeDefinitionsAsync();
public async Task GetContentItemsAsync(ContentTypesSitemapSource source, ContentItemsQueryContext context, int? skip = null, int? take = null)
{
var routeableContentTypeDefinitions = await _routeableContentTypeCoordinator.ListRoutableTypeDefinitionsAsync();

if (source.IndexAll)
{
// Assumption here is that at least one content type will be localized.
var ctdNames = routeableContentTypeDefinitions.Select(ctd => ctd.Name);
IEnumerable<ContentItem> contentItems = null;

var queryResults = await _session.Query<ContentItem>()
.With<ContentItemIndex>(x => x.Published && x.ContentType.IsIn(ctdNames))
.OrderBy(x => x.CreatedUtc)
.ListAsync();
if (source.IndexAll)
{
// Assumption here is that at least one content type will be localized.
var ctdNames = routeableContentTypeDefinitions.Select(ctd => ctd.Name);

queryContext.ContentItems = queryResults;
contentItems = await _session.Query<ContentItem>()
.With<ContentItemIndex>(x => x.Published && x.ContentType.IsIn(ctdNames))
.OrderBy(x => x.CreatedUtc)
.ThenBy(x => x.Id)
.Skip(skip ?? 0)
.Take(take ?? 0)
.ListAsync();

// Provide all content items with localization as reference content items.
queryContext.ReferenceContentItems = queryResults
.Where(ci => ci.Has<LocalizationPart>());
}
else if (source.LimitItems)
{
// Test that content type is still valid to include in sitemap.
var contentType = routeableContentTypeDefinitions
.FirstOrDefault(ctd => string.Equals(source.LimitedContentType.ContentTypeName, ctd.Name, StringComparison.Ordinal));

if (contentType == null)
{
return;
}
else if (source.LimitItems)

if (contentType.Parts.Any(ctd => string.Equals(ctd.Name, nameof(LocalizationPart), StringComparison.Ordinal)))
{
// Test that content type is still valid to include in sitemap.
var contentType = routeableContentTypeDefinitions
.FirstOrDefault(ctd => string.Equals(source.LimitedContentType.ContentTypeName, ctd.Name));

if (contentType == null)
{
return;
}

if (contentType.Parts.Any(ctd => string.Equals(ctd.Name, nameof(LocalizationPart))))
{
// Get all content items here for reference. Then reduce by default culture.
// We know that the content item should be localized.
// If it doesn't have a localization part, the content item should have been saved.
var queryResults = await _session.Query<ContentItem>()
.With<ContentItemIndex>(ci => ci.ContentType == source.LimitedContentType.ContentTypeName && ci.Published)
.OrderBy(ci => ci.CreatedUtc)
.With<LocalizedContentItemIndex>()
.ListAsync();

// When limiting items Content item is valid if it is for the default culture.
var defaultCulture = await _localizationService.GetDefaultCultureAsync();

// Reduce by default culture.
var items = queryResults
.Where(ci => string.Equals(ci.As<LocalizationPart>().Culture, defaultCulture))
.Skip(source.LimitedContentType.Skip)
.Take(source.LimitedContentType.Take);

queryContext.ContentItems = items;

// Provide all content items with localization as reference content items.
queryContext.ReferenceContentItems = queryResults
.Where(ci => ci.Has<LocalizationPart>());
}
else
{
// Content type is not localized. Produce standard results.
var queryResults = await _session.Query<ContentItem>()
.With<ContentItemIndex>(x => x.ContentType == source.LimitedContentType.ContentTypeName && x.Published)
.OrderBy(x => x.CreatedUtc)
.Skip(source.LimitedContentType.Skip)
.Take(source.LimitedContentType.Take)
.ListAsync();

queryContext.ContentItems = queryResults;
}
// When limiting items Content item is valid if it is for the default culture.
var defaultCulture = await _localizationService.GetDefaultCultureAsync();

// Get all content items here for reference. Then reduce by default culture.
// We know that the content item should be localized.
// If it doesn't have a localization part, the content item should have been saved.
contentItems = await _session.Query<ContentItem>()
.With<ContentItemIndex>(ci => ci.ContentType == source.LimitedContentType.ContentTypeName && ci.Published)
.OrderBy(ci => ci.CreatedUtc)
.ThenBy(ci => ci.Id)
.With<LocalizedContentItemIndex>(x => x.Culture == defaultCulture)
.Take(take ?? 0)
.Skip(skip ?? 0)
.ListAsync();
}
else
{
// Test that content types are still valid to include in sitemap.
var typesToIndex = routeableContentTypeDefinitions
.Where(ctd => source.ContentTypes.Any(s => string.Equals(ctd.Name, s.ContentTypeName)))
.Select(x => x.Name);

// No advantage here in reducing with localized index.
var queryResults = await _session.Query<ContentItem>()
.With<ContentItemIndex>(x => x.ContentType.IsIn(typesToIndex) && x.Published)
// Content type is not localized. Produce standard results.
contentItems = await _session.Query<ContentItem>()
.With<ContentItemIndex>(x => x.ContentType == source.LimitedContentType.ContentTypeName && x.Published)
.OrderBy(x => x.CreatedUtc)
.Skip(skip ?? 0)
.Take(take ?? 0)
.ListAsync();

queryContext.ContentItems = queryResults;
}
}
else
{
// Test that content types are still valid to include in sitemap.
var typesToIndex = routeableContentTypeDefinitions
.Where(ctd => source.ContentTypes.Any(s => string.Equals(ctd.Name, s.ContentTypeName, StringComparison.Ordinal)))
.Select(x => x.Name);

// No advantage here in reducing with localized index.

contentItems = await _session.Query<ContentItem>()
.With<ContentItemIndex>(x => x.ContentType.IsIn(typesToIndex) && x.Published)
.OrderBy(x => x.CreatedUtc)
.ThenBy(x => x.Id)
.Skip(skip ?? 0)
.Take(take ?? 0)
.ListAsync();

// Provide all content items with localization as reference content items.
queryContext.ReferenceContentItems = queryResults
.Where(ci => ci.Has<LocalizationPart>());
}

if (contentItems != null)
{
context.ContentItems = contentItems;

// Provide all content items with localization as reference content items.
context.ReferenceContentItems = contentItems.Where(ci => ci.Has<LocalizationPart>());

// Free up memeory
foreach (var contentItem in contentItems)
{
_session.Detach(contentItem);
}
}
}
Expand Down
85 changes: 43 additions & 42 deletions ...es/OrchardCore.ContentLocalization/Sitemaps/SitemapUrlHrefLangExtendedMetadataProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,62 +7,63 @@
using OrchardCore.Sitemaps.Builders;
using OrchardCore.Sitemaps.Services;

namespace OrchardCore.ContentLocalization.Sitemaps
namespace OrchardCore.ContentLocalization.Sitemaps;

public class SitemapUrlHrefLangExtendedMetadataProvider : ISitemapContentItemExtendedMetadataProvider
{
public class SitemapUrlHrefLangExtendedMetadataProvider : ISitemapContentItemExtendedMetadataProvider
private static readonly XNamespace _extendedNamespace = "http://www.w3.org/1999/xhtml";
private static readonly XAttribute _extendedAttribute = new(XNamespace.Xmlns + "xhtml", _extendedNamespace);

private readonly IContentManager _contentManager;
private readonly IRouteableContentTypeCoordinator _routeableContentTypeCoordinator;

public SitemapUrlHrefLangExtendedMetadataProvider(
IContentManager contentManager,
IRouteableContentTypeCoordinator routeableContentTypeCoordinator
)
{
private static readonly XNamespace _extendedNamespace = "http://www.w3.org/1999/xhtml";
private static readonly XAttribute _extendedAttribute = new(XNamespace.Xmlns + "xhtml", _extendedNamespace);
_contentManager = contentManager;
_routeableContentTypeCoordinator = routeableContentTypeCoordinator;
}

private readonly IContentManager _contentManager;
private readonly IRouteableContentTypeCoordinator _routeableContentTypeCoordinator;
public XAttribute GetExtendedAttribute => _extendedAttribute;

public SitemapUrlHrefLangExtendedMetadataProvider(
IContentManager contentManager,
IRouteableContentTypeCoordinator routeableContentTypeCoordinator
)
public async Task<bool> ApplyExtendedMetadataAsync(
SitemapBuilderContext context,
ContentItemsQueryContext queryContext,
ContentItem contentItem,
XElement url)
{
var part = contentItem.As<LocalizationPart>();
if (part == null ||
queryContext.ReferenceContentItems == null ||
!queryContext.ReferenceContentItems.Any())
{
_contentManager = contentManager;
_routeableContentTypeCoordinator = routeableContentTypeCoordinator;
return true;
}

public XAttribute GetExtendedAttribute => _extendedAttribute;
var localizedContentParts = queryContext.ReferenceContentItems
.Select(ci => ci.As<LocalizationPart>())
.Where(cp => cp.LocalizationSet == part.LocalizationSet);

public async Task<bool> ApplyExtendedMetadataAsync(
SitemapBuilderContext context,
ContentItemsQueryContext queryContext,
ContentItem contentItem,
XElement url)
foreach (var localizedPart in localizedContentParts)
{
var part = contentItem.As<LocalizationPart>();
if (part == null)
var sitemapMetadataAspect = await _contentManager.PopulateAspectAsync<SitemapMetadataAspect>(localizedPart.ContentItem);
if (sitemapMetadataAspect.Exclude)
{
return true;
continue;
}

var localizedContentParts = queryContext.ReferenceContentItems
.Select(ci => ci.As<LocalizationPart>())
.Where(cp => cp.LocalizationSet == part.LocalizationSet);
var hrefValue = await _routeableContentTypeCoordinator.GetRouteAsync(context, localizedPart.ContentItem);

foreach (var localizedPart in localizedContentParts)
{
var sitemapMetadataAspect = await _contentManager.PopulateAspectAsync<SitemapMetadataAspect>(localizedPart.ContentItem);
if (sitemapMetadataAspect.Exclude)
{
continue;
}
var linkNode = new XElement(_extendedNamespace + "link",
new XAttribute("rel", "alternate"),
new XAttribute("hreflang", localizedPart.Culture),
new XAttribute("href", hrefValue));

var hrefValue = await _routeableContentTypeCoordinator.GetRouteAsync(context, localizedPart.ContentItem);

var linkNode = new XElement(_extendedNamespace + "link",
new XAttribute("rel", "alternate"),
new XAttribute("hreflang", localizedPart.Culture),
new XAttribute("href", hrefValue));

url.Add(linkNode);
}

return true;
url.Add(linkNode);
}

return true;
}
}
Loading
Loading