.NET 6 WebApplicationFactory test throws System.InvalidOperationException : The entry point exited without ever building an IHost.

Description When I have Serilog configured on my Program.cs and if I run more than one test that uses WebApplicationFactory I get an exception thrown System.InvalidOperationException : The entry point exited without ever building an IHost. accessing the application factory services.

Reproduction Program.cs

using System.Text.Json.Serialization;
using Serilog;

Log.Logger = new LoggerConfiguration()
    .WriteTo.Console()
    .CreateBootstrapLogger();

Log.Information("Starting up...");

try
{
    var builder = WebApplication.CreateBuilder(args);
    builder.Host.UseSerilog((ctx, lc) => lc.ReadFrom.Configuration(ctx.Configuration));

    var configuration = builder.Configuration;
    var services = builder.Services;
    // Add services to the container.

    services.AddControllers()
         .AddJsonOptions(options =>
         {
             options.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter());
         })
         .AddControllersAsServices();
    services.AddEndpointsApiExplorer();

    services.AddSwaggerGen();

    var app = builder.Build();

    app.UseSerilogRequestLogging();

    app.UseCors(builder =>
        builder
            .WithOrigins(configuration.GetSection("AllowedOrigins")
                .AsEnumerable()
                .Select(kvp => kvp.Value)
                .Where(origin => !string.IsNullOrEmpty(origin))
                .ToArray()
            )
            .SetIsOriginAllowedToAllowWildcardSubdomains()
            .AllowAnyMethod()
            .AllowAnyHeader()
            .AllowCredentials());

    // Configure the HTTP request pipeline.
    if (app.Environment.IsDevelopment())
    {
        app.UseSwagger();
        app.UseSwaggerUI();
    }

    app.UseHttpsRedirection();

    app.UseAuthentication();
    app.UseAuthorization();

    app.MapControllers();

    app.Run();
    return 0;
}
catch (Exception ex)
{
    Log.Fatal(ex, "Host terminated unexpectedly.");
    return 1;
}
finally
{
    Log.CloseAndFlush();
}

#pragma warning disable CA1050 // Declare types in namespaces
public partial class Program { } // so you can reference it in tests
#pragma warning restore CA1050 // Declare types in namespaces

appsettings.json:

{
  "AllowedHosts": "*",
  "Serilog": {
    "AllowedHosts": "*",
    "MinimumLevel": {
      "Default": "Information",
      "Override": {
        "Microsoft": "Warning",
        "Microsoft.Hosting.Lifetime": "Information"
      }
    },
    "Enrich": [
      "FromLogContext"
    ],
    "Filter": [
      {
        "Name": "ByExcluding",
        "Args": {
          "expression": "@mt = 'An unhandled exception has occurred while executing the request.'"
        }
      }
    ],
    "WriteTo": [
      {
        "Name": "Console",
        "Args": {
          "outputTemplate": "[{Timestamp:HH:mm:ss} [{Level}]: {Message:l}{NewLine}{Exception}"
        }
      }
    ]
  }
}

TestApplication.cs:

using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.Extensions.Hosting;

namespace Api.Tests;

internal class TestApplication : WebApplicationFactory<Program>
{
    private readonly string environment;

    public TestApplication(string environment = "Development")
    {
        this.environment = environment;
    }

    protected override IHost CreateHost(IHostBuilder builder)
    {
        builder.UseEnvironment(environment);
        return base.CreateHost(builder);
    }
}

SwaggerTest.cs:

using System.Linq;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Extensions.DependencyInjection;
using Swashbuckle.AspNetCore.Swagger;
using Xunit;

namespace Api.Tests.Swagger;

public class SwaggerTest
{
    [Fact]
    public async Task OperationIds_Should_Be_Unique()
    {
        await using var application = new TestApplication();
        var swaggerProvider = application.Services.GetRequiredService<ISwaggerProvider>();
        var swagger = swaggerProvider.GetSwagger("v1");
        var operationIds = swagger.Paths.Values.SelectMany(path => path.Operations.Values.Select(operation => operation.OperationId)).ToList();

        operationIds.Should().OnlyHaveUniqueItems();
    }

    [Fact]
    public async Task OperationIds_Should_Be_Unique2()
    {
        await using var application = new TestApplication();
        var swaggerProvider = application.Services.GetRequiredService<ISwaggerProvider>();
        var swagger = swaggerProvider.GetSwagger("v1");
        var operationIds = swagger.Paths.Values.SelectMany(path => path.Operations.Values.Select(operation => operation.OperationId)).ToList();

        operationIds.Should().OnlyHaveUniqueItems();
    }
}

Expected behavior Tests pass, and does not throw calling application.Services

Relevant package, tooling and runtime versions

<PackageReference Include="Serilog.AspNetCore" Version="4.1.0" />
<PackageReference Include="Serilog.Expressions" Version="3.2.1" />

dotnet --version:

6.0.102

Additional context The tests pass if I comment out the line builder.Host.UseSerilog((ctx, lc) => lc.ReadFrom.Configuration(ctx.Configuration)); from Program.cs.