#15 Add GitLab buckets indexing

src/ScoopSearch.Indexer.Console/LoggingExtensions.cs

@@ -31,6 +31,7 @@ public static IHostBuilder ConfigureSerilog(this IHostBuilder @this, string logF
                     var tokens = new[]
                         {
                             provider.GetRequiredService<IOptions<GitHubOptions>>().Value.Token,
+                            provider.GetRequiredService<IOptions<GitLabOptions>>().Value.Token,
                             provider.GetRequiredService<IOptions<AzureSearchOptions>>().Value.AdminApiKey
                         }
                         .Where(token => token != null)

src/ScoopSearch.Indexer.Tests/Buckets/Providers/GitHubBucketsProviderTests.cs

@@ -28,6 +28,7 @@ public GitHubBucketsProviderTests()
     [InlineData("https://www.github.com", true)]
     [InlineData("http://www.GitHub.com", true)]
     [InlineData("https://www.GitHub.com", true)]
+    [InlineData("https://www.github.com/foo/bar", true)]
     public void IsCompatible_Succeeds(string input, bool expectedResult)
     {
         // Arrange

src/ScoopSearch.Indexer.Tests/Buckets/Providers/GitLabBucketsProviderTests.cs

@@ -0,0 +1,76 @@
+using FluentAssertions;
+using Moq;
+using ScoopSearch.Indexer.Buckets.Providers;
+using ScoopSearch.Indexer.GitLab;
+using ScoopSearch.Indexer.Tests.Helpers;
+
+namespace ScoopSearch.Indexer.Tests.Buckets.Providers;
+
+public class GitLabBucketsProviderTests
+{
+    private readonly Mock<IGitLabClient> _gitLabClientMock;
+    private readonly GitLabBucketsProvider _sut;
+
+    public GitLabBucketsProviderTests()
+    {
+        _gitLabClientMock = new Mock<IGitLabClient>();
+        _sut = new GitLabBucketsProvider(_gitLabClientMock.Object);
+    }
+
+    [Theory]
+    [InlineData("http://foo/bar", false)]
+    [InlineData("https://foo/bar", false)]
+    [InlineData("http://www.google.fr/foo", false)]
+    [InlineData("https://www.google.fr/foo", false)]
+    [InlineData("http://gitlab.com", true)]
+    [InlineData("https://gitlab.com", true)]
+    [InlineData("http://www.gitlab.com", true)]
+    [InlineData("https://www.gitlab.com", true)]
+    [InlineData("http://www.GitLab.com", true)]
+    [InlineData("https://www.GitLab.com", true)]
+    [InlineData("https://www.gitlab.com/foo/bar", true)]
+    public void IsCompatible_Succeeds(string input, bool expectedResult)
+    {
+        // Arrange
+        var uri = new Uri(input);
+
+        // Act
+        var result = _sut.IsCompatible(uri);
+
+        // Arrange
+        result.Should().Be(expectedResult);
+    }
+
+    [Fact]
+    public async void GetBucketAsync_ValidRepo_ReturnsBucket()
+    {
+        // Arrange
+        var cancellationToken = new CancellationToken();
+        var uri = Faker.CreateUri();
+        var gitLabRepo = Faker.CreateGitLabRepo().Generate();
+        _gitLabClientMock.Setup(x => x.GetRepositoryAsync(uri, cancellationToken)).ReturnsAsync(gitLabRepo);
+
+        // Act
+        var result = await _sut.GetBucketAsync(uri, cancellationToken);
+
+        // Assert
+        result.Should().NotBeNull();
+        result!.Uri.Should().Be(gitLabRepo.WebUrl);
+        result.Stars.Should().Be(gitLabRepo.Stars);
+    }
+
+    [Fact]
+    public async void GetBucketAsync_InvalidRepo_ReturnsNull()
+    {
+        // Arrange
+        var cancellationToken = new CancellationToken();
+        var uri = Faker.CreateUri();
+        _gitLabClientMock.Setup(x => x.GetRepositoryAsync(uri, cancellationToken)).ReturnsAsync((GitLabRepo?)null);
+
+        // Act
+        var result = await _sut.GetBucketAsync(uri, cancellationToken);
+
+        // Assert
+        result.Should().BeNull();
+    }
+}

src/ScoopSearch.Indexer.Tests/Buckets/Sources/GitLabBucketsSourceTests.cs

@@ -0,0 +1,67 @@
+using FluentAssertions;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Moq;
+using ScoopSearch.Indexer.Buckets;
+using ScoopSearch.Indexer.Buckets.Sources;
+using ScoopSearch.Indexer.Configuration;
+using ScoopSearch.Indexer.GitLab;
+using ScoopSearch.Indexer.Tests.Helpers;
+using Xunit.Abstractions;
+
+namespace ScoopSearch.Indexer.Tests.Buckets.Sources;
+
+public class GitLabBucketsSourceTests
+{
+    private readonly Mock<IGitLabClient> _gitLabClientMock;
+    private readonly GitLabOptions _gitLabOptions;
+    private readonly XUnitLogger<GitLabBucketsSource> _logger;
+    private readonly GitLabBucketsSource _sut;
+
+    public GitLabBucketsSourceTests(ITestOutputHelper testOutputHelper)
+    {
+        _gitLabClientMock = new Mock<IGitLabClient>();
+        _gitLabOptions = new GitLabOptions();
+        _logger = new XUnitLogger<GitLabBucketsSource>(testOutputHelper);
+        _sut = new GitLabBucketsSource(_gitLabClientMock.Object, new OptionsWrapper<GitLabOptions>(_gitLabOptions), _logger);
+    }
+
+    [Fact]
+    public async void GetBucketsAsync_InvalidQueries_ReturnsEmpty()
+    {
+        // Arrange
+        var cancellationToken = new CancellationToken();
+        _gitLabOptions.BucketsSearchQueries = null;
+
+        // Act
+        var result = await _sut.GetBucketsAsync(cancellationToken).ToArrayAsync(cancellationToken);
+
+        // Arrange
+        result.Should().BeEmpty();
+        _logger.Should().Log(LogLevel.Warning, "No buckets search queries found in configuration");
+    }
+
+    [Fact]
+    public async void GetBucketsAsync_Succeeds()
+    {
+        // Arrange
+        var cancellationToken = new CancellationToken();
+        var input = new (string query, GitLabRepo[] repos)[]
+        {
+            ("foo", new[] { Faker.CreateGitLabRepo().Generate() }),
+            ("bar", new[] { Faker.CreateGitLabRepo().Generate() }),
+        };
+        _gitLabOptions.BucketsSearchQueries = input.Select(x => x.query).ToArray();
+        _gitLabClientMock.Setup(x => x.SearchRepositoriesAsync(input[0].query, cancellationToken)).Returns(input[0].repos.ToAsyncEnumerable());
+        _gitLabClientMock.Setup(x => x.SearchRepositoriesAsync(input[1].query, cancellationToken)).Returns(input[1].repos.ToAsyncEnumerable());
+
+        // Act
+        var result = await _sut.GetBucketsAsync(cancellationToken).ToArrayAsync(cancellationToken);
+
+        // Arrange
+        result.Should().BeEquivalentTo(
+            input.SelectMany(x => x.repos),
+            options => options
+                .WithMapping<Bucket>(x => x.WebUrl, y => y.Uri));
+    }
+}

src/ScoopSearch.Indexer.Tests/GitHub/GitHubClientTests.cs

@@ -93,7 +93,7 @@ public async void GetRepositoryAsync_ValidRepo_ReturnsGitHubRepo(string input, i
         // Assert
         result.Should().NotBeNull();
         result!.HtmlUri.Should().Be(uri);
-        result.Stars.Should().BeGreaterThan(expectedMinimumStars, "because official repo should have a large amount of stars");
+        result.Stars.Should().BeGreaterOrEqualTo(expectedMinimumStars, "because official repo should have a large amount of stars");
     }
 
     [Theory]
@@ -102,30 +102,25 @@ public async void GetRepositoryAsync_ValidRepo_ReturnsGitHubRepo(string input, i
     [InlineData(new object[] { new[] { "&&==" } })]
     public async void SearchRepositoriesAsync_InvalidQueryUrl_Throws(string[] input)
     {
-        // Arrange + Act
+        // Arrange
         var cancellationToken = new CancellationToken();
-        try
-        {
-            await _sut.SearchRepositoriesAsync(input, cancellationToken).ToArrayAsync(cancellationToken);
-            Assert.Fail("Should have thrown");
-        }
-        catch (AggregateException ex)
-        {
-            // Assert
-            ex.InnerException.Should().BeOfType<HttpRequestException>();
-            return;
-        }
-
-        Assert.Fail("Should have thrown an AggregateException");
+
+        // Act
+        var result = await _sut.SearchRepositoriesAsync(input, cancellationToken).ToArrayAsync(cancellationToken);
+
+        // Assert
+        result.Should().BeEmpty();
     }
 
     [Theory]
     [InlineData(new object[] { new[] { "scoop-bucket", "created:>2023-01-01" } })]
     [InlineData(new object[] { new[] { "scoop+bucket", "created:>2023-01-01" } })]
     public async void SearchRepositoriesAsync_ValidQuery_ReturnsSearchResults(string[] input)
     {
-        // Arrange + Act
+        // Arrange
         var cancellationToken = new CancellationToken();
+
+        // Act
         var result = await _sut.SearchRepositoriesAsync(input, cancellationToken).ToArrayAsync(cancellationToken);
 
         // Assert

src/ScoopSearch.Indexer.Tests/GitLab/GitLabClientTests.cs

@@ -0,0 +1,100 @@
+using FluentAssertions;
+using Microsoft.Extensions.DependencyInjection;
+using ScoopSearch.Indexer.GitLab;
+using ScoopSearch.Indexer.Tests.Helpers;
+using Xunit.Abstractions;
+
+namespace ScoopSearch.Indexer.Tests.GitLab;
+
+public class GitLabClientTests : IClassFixture<HostFixture>
+{
+    private readonly GitLabClient _sut;
+
+    public GitLabClientTests(HostFixture hostFixture, ITestOutputHelper testOutputHelper)
+    {
+        hostFixture.Configure(testOutputHelper);
+        var logger = new XUnitLogger<GitLabClient>(testOutputHelper);
+
+        _sut = new GitLabClient(hostFixture.Instance.Services.GetRequiredService<IHttpClientFactory>(), logger);
+    }
+
+    [Theory]
+    [InlineData("http://example.com/foo/bar")]
+    public async void GetRepositoryAsync_InvalidRepo_ReturnsNull(string input)
+    {
+        // Arrange
+        var uri = new Uri(input);
+        var cancellationToken = new CancellationToken();
+
+        // Act
+        var result = () => _sut.GetRepositoryAsync(uri, cancellationToken);
+
+        // Assert
+        var taskResult = await result.Should().NotThrowAsync();
+        taskResult.Subject.Should().BeNull();
+    }
+
+    [Fact]
+    public async void GetRepositoryAsync_NonExistentRepo_ReturnsNull()
+    {
+        // Arrange
+        var uri = new Uri(Constants.NonExistentTestRepositoryUri);
+        var cancellationToken = new CancellationToken();
+
+        // Act
+        var result = await _sut.GetRepositoryAsync(uri, cancellationToken);
+
+        // Assert
+        result.Should().BeNull();
+    }
+
+    [Theory]
+    [InlineData("https://gitlab.com/aknackd/scoop-nightly-neovim", 0)]
+    [InlineData("https://gitlab.com/jbmorice/scoop_bucket", 1)]
+    public async void GetRepositoryAsync_ValidRepo_ReturnsGitLabRepo(string input, int expectedMinimumStars)
+    {
+        // Arrange
+        var uri = new Uri(input);
+        var cancellationToken = new CancellationToken();
+
+        // Act
+        var result = await _sut.GetRepositoryAsync(uri, cancellationToken);
+
+        // Assert
+        result.Should().NotBeNull();
+        result!.WebUrl.Should().Be(uri);
+        result.Stars.Should().BeGreaterOrEqualTo(expectedMinimumStars, "because official repo should have a large amount of stars");
+    }
+
+    [Theory]
+    [InlineData("&&==")]
+    public async void SearchRepositoriesAsync_InvalidQueryUrl_Throws(string input)
+    {
+        // Arrange
+        var cancellationToken = new CancellationToken();
+
+        // Act
+        var result = await _sut.SearchRepositoriesAsync(input, cancellationToken).ToArrayAsync(cancellationToken);
+
+        // Assert
+        result.Should().BeEmpty();
+    }
+
+    [Theory]
+    [InlineData("scoop-bucket")]
+    [InlineData("scoop")]
+    public async void SearchRepositoriesAsync_ValidQuery_ReturnsSearchResults(string input)
+    {
+        // Arrange
+        var cancellationToken = new CancellationToken();
+
+        // Act
+        var result = await _sut.SearchRepositoriesAsync(input, cancellationToken).ToArrayAsync(cancellationToken);
+
+        // Assert
+        result.Should().NotBeNull();
+        result.Length.Should()
+            .BeGreaterThan(0, "because there should be at least 1 result")
+            .And.BeLessThan(90, "because there should be less than 90 results. If it returns more than 90, the pagination should be implemented (max items per page is 100)");
+    }
+}

src/ScoopSearch.Indexer.Tests/Helpers/Faker.cs

@@ -1,6 +1,7 @@
 using Bogus;
 using ScoopSearch.Indexer.Data;
 using ScoopSearch.Indexer.GitHub;
+using ScoopSearch.Indexer.GitLab;
 
 namespace ScoopSearch.Indexer.Tests.Helpers;
 
@@ -54,6 +55,16 @@ public static Faker<GitHubRepo> CreateGitHubRepo()
         return faker;
     }
 
+    public static Faker<GitLabRepo> CreateGitLabRepo()
+    {
+        var faker = new Faker<GitLabRepo>()
+            .StrictMode(true)
+            .RuleFor(_ => _.WebUrl, f => new Uri(f.Internet.Url()))
+            .RuleFor(_ => _.Stars, f => f.Random.Int(0, 1000));
+
+        return faker;
+    }
+
     public static string CreateUrl()
     {
         return new Bogus.Faker().Internet.UrlWithPath();

src/ScoopSearch.Indexer/Buckets/Providers/GitLabBucketsProvider.cs

@@ -0,0 +1,29 @@
+using ScoopSearch.Indexer.GitLab;
+
+namespace ScoopSearch.Indexer.Buckets.Providers;
+
+internal class GitLabBucketsProvider : IBucketsProvider
+{
+    private const string GitLabDomain = "gitlab.com";
+
+    private readonly IGitLabClient _gitLabClient;
+
+    public GitLabBucketsProvider(IGitLabClient gitLabClient)
+    {
+        _gitLabClient = gitLabClient;
+    }
+
+    public async Task<Bucket?> GetBucketAsync(Uri uri, CancellationToken cancellationToken)
+    {
+        var result = await _gitLabClient.GetRepositoryAsync(uri, cancellationToken);
+        if (result is not null)
+        {
+            return new Bucket(result.WebUrl, result.Stars);
+        }
+
+        return null;
+    }
+
+    public bool IsCompatible(Uri uri) => uri.Host.EndsWith(GitLabDomain, StringComparison.Ordinal);
+
+}