diff --git a/src/ScoopSearch.Indexer.Console/LoggingExtensions.cs b/src/ScoopSearch.Indexer.Console/LoggingExtensions.cs index 7e57122..ca04f69 100644 --- a/src/ScoopSearch.Indexer.Console/LoggingExtensions.cs +++ b/src/ScoopSearch.Indexer.Console/LoggingExtensions.cs @@ -31,6 +31,7 @@ public static IHostBuilder ConfigureSerilog(this IHostBuilder @this, string logF var tokens = new[] { provider.GetRequiredService>().Value.Token, + provider.GetRequiredService>().Value.Token, provider.GetRequiredService>().Value.AdminApiKey } .Where(token => token != null) diff --git a/src/ScoopSearch.Indexer.Tests/Buckets/Providers/GitHubBucketsProviderTests.cs b/src/ScoopSearch.Indexer.Tests/Buckets/Providers/GitHubBucketsProviderTests.cs index 3e789d4..af68eaf 100644 --- a/src/ScoopSearch.Indexer.Tests/Buckets/Providers/GitHubBucketsProviderTests.cs +++ b/src/ScoopSearch.Indexer.Tests/Buckets/Providers/GitHubBucketsProviderTests.cs @@ -28,6 +28,7 @@ public GitHubBucketsProviderTests() [InlineData("https://www.github.com", true)] [InlineData("http://www.GitHub.com", true)] [InlineData("https://www.GitHub.com", true)] + [InlineData("https://www.github.com/foo/bar", true)] public void IsCompatible_Succeeds(string input, bool expectedResult) { // Arrange diff --git a/src/ScoopSearch.Indexer.Tests/Buckets/Providers/GitLabBucketsProviderTests.cs b/src/ScoopSearch.Indexer.Tests/Buckets/Providers/GitLabBucketsProviderTests.cs new file mode 100644 index 0000000..709dce7 --- /dev/null +++ b/src/ScoopSearch.Indexer.Tests/Buckets/Providers/GitLabBucketsProviderTests.cs @@ -0,0 +1,76 @@ +using FluentAssertions; +using Moq; +using ScoopSearch.Indexer.Buckets.Providers; +using ScoopSearch.Indexer.GitLab; +using ScoopSearch.Indexer.Tests.Helpers; + +namespace ScoopSearch.Indexer.Tests.Buckets.Providers; + +public class GitLabBucketsProviderTests +{ + private readonly Mock _gitLabClientMock; + private readonly GitLabBucketsProvider _sut; + + public GitLabBucketsProviderTests() + { + _gitLabClientMock = new Mock(); + _sut = new GitLabBucketsProvider(_gitLabClientMock.Object); + } + + [Theory] + [InlineData("http://foo/bar", false)] + [InlineData("https://foo/bar", false)] + [InlineData("http://www.google.fr/foo", false)] + [InlineData("https://www.google.fr/foo", false)] + [InlineData("http://gitlab.com", true)] + [InlineData("https://gitlab.com", true)] + [InlineData("http://www.gitlab.com", true)] + [InlineData("https://www.gitlab.com", true)] + [InlineData("http://www.GitLab.com", true)] + [InlineData("https://www.GitLab.com", true)] + [InlineData("https://www.gitlab.com/foo/bar", true)] + public void IsCompatible_Succeeds(string input, bool expectedResult) + { + // Arrange + var uri = new Uri(input); + + // Act + var result = _sut.IsCompatible(uri); + + // Arrange + result.Should().Be(expectedResult); + } + + [Fact] + public async void GetBucketAsync_ValidRepo_ReturnsBucket() + { + // Arrange + var cancellationToken = new CancellationToken(); + var uri = Faker.CreateUri(); + var gitLabRepo = Faker.CreateGitLabRepo().Generate(); + _gitLabClientMock.Setup(x => x.GetRepositoryAsync(uri, cancellationToken)).ReturnsAsync(gitLabRepo); + + // Act + var result = await _sut.GetBucketAsync(uri, cancellationToken); + + // Assert + result.Should().NotBeNull(); + result!.Uri.Should().Be(gitLabRepo.WebUrl); + result.Stars.Should().Be(gitLabRepo.Stars); + } + + [Fact] + public async void GetBucketAsync_InvalidRepo_ReturnsNull() + { + // Arrange + var cancellationToken = new CancellationToken(); + var uri = Faker.CreateUri(); + _gitLabClientMock.Setup(x => x.GetRepositoryAsync(uri, cancellationToken)).ReturnsAsync((GitLabRepo?)null); + + // Act + var result = await _sut.GetBucketAsync(uri, cancellationToken); + + // Assert + result.Should().BeNull(); + } +} diff --git a/src/ScoopSearch.Indexer.Tests/Buckets/Sources/GitLabBucketsSourceTests.cs b/src/ScoopSearch.Indexer.Tests/Buckets/Sources/GitLabBucketsSourceTests.cs new file mode 100644 index 0000000..e06c3b1 --- /dev/null +++ b/src/ScoopSearch.Indexer.Tests/Buckets/Sources/GitLabBucketsSourceTests.cs @@ -0,0 +1,67 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using ScoopSearch.Indexer.Buckets; +using ScoopSearch.Indexer.Buckets.Sources; +using ScoopSearch.Indexer.Configuration; +using ScoopSearch.Indexer.GitLab; +using ScoopSearch.Indexer.Tests.Helpers; +using Xunit.Abstractions; + +namespace ScoopSearch.Indexer.Tests.Buckets.Sources; + +public class GitLabBucketsSourceTests +{ + private readonly Mock _gitLabClientMock; + private readonly GitLabOptions _gitLabOptions; + private readonly XUnitLogger _logger; + private readonly GitLabBucketsSource _sut; + + public GitLabBucketsSourceTests(ITestOutputHelper testOutputHelper) + { + _gitLabClientMock = new Mock(); + _gitLabOptions = new GitLabOptions(); + _logger = new XUnitLogger(testOutputHelper); + _sut = new GitLabBucketsSource(_gitLabClientMock.Object, new OptionsWrapper(_gitLabOptions), _logger); + } + + [Fact] + public async void GetBucketsAsync_InvalidQueries_ReturnsEmpty() + { + // Arrange + var cancellationToken = new CancellationToken(); + _gitLabOptions.BucketsSearchQueries = null; + + // Act + var result = await _sut.GetBucketsAsync(cancellationToken).ToArrayAsync(cancellationToken); + + // Arrange + result.Should().BeEmpty(); + _logger.Should().Log(LogLevel.Warning, "No buckets search queries found in configuration"); + } + + [Fact] + public async void GetBucketsAsync_Succeeds() + { + // Arrange + var cancellationToken = new CancellationToken(); + var input = new (string query, GitLabRepo[] repos)[] + { + ("foo", new[] { Faker.CreateGitLabRepo().Generate() }), + ("bar", new[] { Faker.CreateGitLabRepo().Generate() }), + }; + _gitLabOptions.BucketsSearchQueries = input.Select(x => x.query).ToArray(); + _gitLabClientMock.Setup(x => x.SearchRepositoriesAsync(input[0].query, cancellationToken)).Returns(input[0].repos.ToAsyncEnumerable()); + _gitLabClientMock.Setup(x => x.SearchRepositoriesAsync(input[1].query, cancellationToken)).Returns(input[1].repos.ToAsyncEnumerable()); + + // Act + var result = await _sut.GetBucketsAsync(cancellationToken).ToArrayAsync(cancellationToken); + + // Arrange + result.Should().BeEquivalentTo( + input.SelectMany(x => x.repos), + options => options + .WithMapping(x => x.WebUrl, y => y.Uri)); + } +} diff --git a/src/ScoopSearch.Indexer.Tests/GitHub/GitHubClientTests.cs b/src/ScoopSearch.Indexer.Tests/GitHub/GitHubClientTests.cs index c8faa0e..2f0adf2 100644 --- a/src/ScoopSearch.Indexer.Tests/GitHub/GitHubClientTests.cs +++ b/src/ScoopSearch.Indexer.Tests/GitHub/GitHubClientTests.cs @@ -93,7 +93,7 @@ public async void GetRepositoryAsync_ValidRepo_ReturnsGitHubRepo(string input, i // Assert result.Should().NotBeNull(); result!.HtmlUri.Should().Be(uri); - result.Stars.Should().BeGreaterThan(expectedMinimumStars, "because official repo should have a large amount of stars"); + result.Stars.Should().BeGreaterOrEqualTo(expectedMinimumStars, "because official repo should have a large amount of stars"); } [Theory] @@ -102,21 +102,14 @@ public async void GetRepositoryAsync_ValidRepo_ReturnsGitHubRepo(string input, i [InlineData(new object[] { new[] { "&&==" } })] public async void SearchRepositoriesAsync_InvalidQueryUrl_Throws(string[] input) { - // Arrange + Act + // Arrange var cancellationToken = new CancellationToken(); - try - { - await _sut.SearchRepositoriesAsync(input, cancellationToken).ToArrayAsync(cancellationToken); - Assert.Fail("Should have thrown"); - } - catch (AggregateException ex) - { - // Assert - ex.InnerException.Should().BeOfType(); - return; - } - - Assert.Fail("Should have thrown an AggregateException"); + + // Act + var result = await _sut.SearchRepositoriesAsync(input, cancellationToken).ToArrayAsync(cancellationToken); + + // Assert + result.Should().BeEmpty(); } [Theory] @@ -124,8 +117,10 @@ public async void SearchRepositoriesAsync_InvalidQueryUrl_Throws(string[] input) [InlineData(new object[] { new[] { "scoop+bucket", "created:>2023-01-01" } })] public async void SearchRepositoriesAsync_ValidQuery_ReturnsSearchResults(string[] input) { - // Arrange + Act + // Arrange var cancellationToken = new CancellationToken(); + + // Act var result = await _sut.SearchRepositoriesAsync(input, cancellationToken).ToArrayAsync(cancellationToken); // Assert diff --git a/src/ScoopSearch.Indexer.Tests/GitLab/GitLabClientTests.cs b/src/ScoopSearch.Indexer.Tests/GitLab/GitLabClientTests.cs new file mode 100644 index 0000000..5412e89 --- /dev/null +++ b/src/ScoopSearch.Indexer.Tests/GitLab/GitLabClientTests.cs @@ -0,0 +1,100 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using ScoopSearch.Indexer.GitLab; +using ScoopSearch.Indexer.Tests.Helpers; +using Xunit.Abstractions; + +namespace ScoopSearch.Indexer.Tests.GitLab; + +public class GitLabClientTests : IClassFixture +{ + private readonly GitLabClient _sut; + + public GitLabClientTests(HostFixture hostFixture, ITestOutputHelper testOutputHelper) + { + hostFixture.Configure(testOutputHelper); + var logger = new XUnitLogger(testOutputHelper); + + _sut = new GitLabClient(hostFixture.Instance.Services.GetRequiredService(), logger); + } + + [Theory] + [InlineData("http://example.com/foo/bar")] + public async void GetRepositoryAsync_InvalidRepo_ReturnsNull(string input) + { + // Arrange + var uri = new Uri(input); + var cancellationToken = new CancellationToken(); + + // Act + var result = () => _sut.GetRepositoryAsync(uri, cancellationToken); + + // Assert + var taskResult = await result.Should().NotThrowAsync(); + taskResult.Subject.Should().BeNull(); + } + + [Fact] + public async void GetRepositoryAsync_NonExistentRepo_ReturnsNull() + { + // Arrange + var uri = new Uri(Constants.NonExistentTestRepositoryUri); + var cancellationToken = new CancellationToken(); + + // Act + var result = await _sut.GetRepositoryAsync(uri, cancellationToken); + + // Assert + result.Should().BeNull(); + } + + [Theory] + [InlineData("https://gitlab.com/aknackd/scoop-nightly-neovim", 0)] + [InlineData("https://gitlab.com/jbmorice/scoop_bucket", 1)] + public async void GetRepositoryAsync_ValidRepo_ReturnsGitLabRepo(string input, int expectedMinimumStars) + { + // Arrange + var uri = new Uri(input); + var cancellationToken = new CancellationToken(); + + // Act + var result = await _sut.GetRepositoryAsync(uri, cancellationToken); + + // Assert + result.Should().NotBeNull(); + result!.WebUrl.Should().Be(uri); + result.Stars.Should().BeGreaterOrEqualTo(expectedMinimumStars, "because official repo should have a large amount of stars"); + } + + [Theory] + [InlineData("&&==")] + public async void SearchRepositoriesAsync_InvalidQueryUrl_Throws(string input) + { + // Arrange + var cancellationToken = new CancellationToken(); + + // Act + var result = await _sut.SearchRepositoriesAsync(input, cancellationToken).ToArrayAsync(cancellationToken); + + // Assert + result.Should().BeEmpty(); + } + + [Theory] + [InlineData("scoop-bucket")] + [InlineData("scoop")] + public async void SearchRepositoriesAsync_ValidQuery_ReturnsSearchResults(string input) + { + // Arrange + var cancellationToken = new CancellationToken(); + + // Act + var result = await _sut.SearchRepositoriesAsync(input, cancellationToken).ToArrayAsync(cancellationToken); + + // Assert + result.Should().NotBeNull(); + result.Length.Should() + .BeGreaterThan(0, "because there should be at least 1 result") + .And.BeLessThan(90, "because there should be less than 90 results. If it returns more than 90, the pagination should be implemented (max items per page is 100)"); + } +} diff --git a/src/ScoopSearch.Indexer.Tests/Helpers/Faker.cs b/src/ScoopSearch.Indexer.Tests/Helpers/Faker.cs index f52e146..4d3dd48 100644 --- a/src/ScoopSearch.Indexer.Tests/Helpers/Faker.cs +++ b/src/ScoopSearch.Indexer.Tests/Helpers/Faker.cs @@ -1,6 +1,7 @@ using Bogus; using ScoopSearch.Indexer.Data; using ScoopSearch.Indexer.GitHub; +using ScoopSearch.Indexer.GitLab; namespace ScoopSearch.Indexer.Tests.Helpers; @@ -54,6 +55,16 @@ public static Faker CreateGitHubRepo() return faker; } + public static Faker CreateGitLabRepo() + { + var faker = new Faker() + .StrictMode(true) + .RuleFor(_ => _.WebUrl, f => new Uri(f.Internet.Url())) + .RuleFor(_ => _.Stars, f => f.Random.Int(0, 1000)); + + return faker; + } + public static string CreateUrl() { return new Bogus.Faker().Internet.UrlWithPath(); diff --git a/src/ScoopSearch.Indexer/Buckets/Providers/GitLabBucketsProvider.cs b/src/ScoopSearch.Indexer/Buckets/Providers/GitLabBucketsProvider.cs new file mode 100644 index 0000000..91c67a5 --- /dev/null +++ b/src/ScoopSearch.Indexer/Buckets/Providers/GitLabBucketsProvider.cs @@ -0,0 +1,29 @@ +using ScoopSearch.Indexer.GitLab; + +namespace ScoopSearch.Indexer.Buckets.Providers; + +internal class GitLabBucketsProvider : IBucketsProvider +{ + private const string GitLabDomain = "gitlab.com"; + + private readonly IGitLabClient _gitLabClient; + + public GitLabBucketsProvider(IGitLabClient gitLabClient) + { + _gitLabClient = gitLabClient; + } + + public async Task GetBucketAsync(Uri uri, CancellationToken cancellationToken) + { + var result = await _gitLabClient.GetRepositoryAsync(uri, cancellationToken); + if (result is not null) + { + return new Bucket(result.WebUrl, result.Stars); + } + + return null; + } + + public bool IsCompatible(Uri uri) => uri.Host.EndsWith(GitLabDomain, StringComparison.Ordinal); + +} diff --git a/src/ScoopSearch.Indexer/Buckets/Sources/GitLabBucketsSource.cs b/src/ScoopSearch.Indexer/Buckets/Sources/GitLabBucketsSource.cs new file mode 100644 index 0000000..003817d --- /dev/null +++ b/src/ScoopSearch.Indexer/Buckets/Sources/GitLabBucketsSource.cs @@ -0,0 +1,41 @@ +using System.Runtime.CompilerServices; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ScoopSearch.Indexer.Configuration; +using ScoopSearch.Indexer.GitLab; + +namespace ScoopSearch.Indexer.Buckets.Sources; + +internal class GitLabBucketsSource : IBucketsSource +{ + private readonly IGitLabClient _gitLabClient; + private readonly GitLabOptions _gitLabOptions; + private readonly ILogger _logger; + + public GitLabBucketsSource( + IGitLabClient gitLabClient, + IOptions gitLabOptions, + ILogger logger) + { + _gitLabClient = gitLabClient; + _logger = logger; + _gitLabOptions = gitLabOptions.Value; + } + + public async IAsyncEnumerable GetBucketsAsync([EnumeratorCancellation] CancellationToken cancellationToken) + { + if (_gitLabOptions.BucketsSearchQueries is null || _gitLabOptions.BucketsSearchQueries.Length == 0) + { + _logger.LogWarning("No buckets search queries found in configuration"); + yield break; + } + + foreach (var query in _gitLabOptions.BucketsSearchQueries) + { + await foreach(var repo in _gitLabClient.SearchRepositoriesAsync(query, cancellationToken)) + { + yield return new Bucket(repo.WebUrl, repo.Stars); + } + } + } +} diff --git a/src/ScoopSearch.Indexer/Configuration/GitLabOptions.cs b/src/ScoopSearch.Indexer/Configuration/GitLabOptions.cs new file mode 100644 index 0000000..2179b88 --- /dev/null +++ b/src/ScoopSearch.Indexer/Configuration/GitLabOptions.cs @@ -0,0 +1,10 @@ +namespace ScoopSearch.Indexer.Configuration; + +public class GitLabOptions +{ + public const string Key = "GitLab"; + + public string? Token { get; set; } + + public string[]? BucketsSearchQueries { get; set; } +} diff --git a/src/ScoopSearch.Indexer/Extensions/Extensions.cs b/src/ScoopSearch.Indexer/Extensions/Extensions.cs index 7ac6a79..96d128c 100644 --- a/src/ScoopSearch.Indexer/Extensions/Extensions.cs +++ b/src/ScoopSearch.Indexer/Extensions/Extensions.cs @@ -1,4 +1,5 @@ using System.Security.Cryptography; +using System.Text.Json; namespace ScoopSearch.Indexer.Extensions; @@ -18,4 +19,14 @@ public static string Sha1Sum(this string @this) var hash = sha1.ComputeHash(System.Text.Encoding.UTF8.GetBytes(@this)); return string.Concat(hash.Select(b => b.ToString("x2"))); } + + public static T? Deserialize(this Task @this) + { + if (@this.IsCompletedSuccessfully) + { + return JsonSerializer.Deserialize(@this.Result); + } + + return default; + } } diff --git a/src/ScoopSearch.Indexer/Extensions/HttpClientExtensions.cs b/src/ScoopSearch.Indexer/Extensions/HttpClientExtensions.cs index b27e69d..7d18343 100644 --- a/src/ScoopSearch.Indexer/Extensions/HttpClientExtensions.cs +++ b/src/ScoopSearch.Indexer/Extensions/HttpClientExtensions.cs @@ -13,12 +13,15 @@ internal static class HttpClientExtensions { private const string DefaultHttpClient = "Default"; private const string GitHubHttpClient = "GitHub"; + private const string GitLabHttpClient = "GitLab"; public static void AddHttpClients(this IServiceCollection services) { + var retryPolicy = CreateRetryPolicy(); + services .AddHttpClient(DefaultHttpClient) - .AddTransientHttpErrorPolicy(policyBuilder => policyBuilder.WaitAndRetryAsync(5, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)))); + .AddPolicyHandler(retryPolicy); services .AddHttpClient(GitHubHttpClient, (serviceProvider, client) => @@ -38,6 +41,19 @@ public static void AddHttpClients(this IServiceCollection services) client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", gitHubOptions.Value.Token); }) .AddPolicyHandler((provider, _) => CreateGitHubRetryPolicy(provider)); + + services + .AddHttpClient(GitLabHttpClient, (serviceProvider, client) => + { + // Authentication + var gitLabOptions = serviceProvider.GetRequiredService>(); + if (gitLabOptions.Value.Token == null) + { + serviceProvider.GetRequiredService>().LogWarning("GitLab Token is not defined in configuration."); + } + client.DefaultRequestHeaders.Add("PRIVATE-TOKEN", gitLabOptions.Value.Token); + }) + .AddPolicyHandler(retryPolicy); } public static HttpClient CreateDefaultClient(this IHttpClientFactory @this) @@ -50,6 +66,18 @@ public static HttpClient CreateGitHubClient(this IHttpClientFactory @this) return @this.CreateClient(GitHubHttpClient); } + public static HttpClient CreateGitLabClient(this IHttpClientFactory @this) + { + return @this.CreateClient(GitLabHttpClient); + } + + private static IAsyncPolicy CreateRetryPolicy() + { + return HttpPolicyExtensions + .HandleTransientHttpError() + .WaitAndRetryAsync(5, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))); + } + private static IAsyncPolicy CreateGitHubRetryPolicy(IServiceProvider provider) { return Policy diff --git a/src/ScoopSearch.Indexer/GitHub/GitHubClient.cs b/src/ScoopSearch.Indexer/GitHub/GitHubClient.cs index 0c44632..ec8d4ba 100644 --- a/src/ScoopSearch.Indexer/GitHub/GitHubClient.cs +++ b/src/ScoopSearch.Indexer/GitHub/GitHubClient.cs @@ -1,5 +1,5 @@ using System.Runtime.CompilerServices; -using System.Text.Json; +using System.Text.Json.Serialization; using Microsoft.Extensions.Logging; using ScoopSearch.Indexer.Extensions; @@ -35,17 +35,7 @@ public GitHubClient(IHttpClientFactory httpClientFactory, ILogger var getRepoUri = BuildUri("repos" + targetUri.PathAndQuery); return await _httpClientFactory.CreateGitHubClient().GetStringAsync(getRepoUri, cancellationToken) - .ContinueWith(task => - { - if (task.IsCompletedSuccessfully) - { - return JsonSerializer.Deserialize(task.Result); - } - else - { - return null; - } - }, cancellationToken); + .ContinueWith(task => task.Deserialize(), cancellationToken); } private async Task GetTargetRepositoryAsync(Uri uri, CancellationToken cancellationToken) @@ -84,7 +74,7 @@ public async IAsyncEnumerable SearchRepositoriesAsync(string[] query var results = await GetSearchResultsAsync(searchReposUri, cancellationToken); if (results == null) { - break; + yield break; } _logger.LogDebug("Found {Count} repositories for query {Query}", results.Items.Length, searchReposUri); @@ -100,7 +90,7 @@ public async IAsyncEnumerable SearchRepositoriesAsync(string[] query private async Task GetSearchResultsAsync(Uri searchUri, CancellationToken cancellationToken) { return await _httpClientFactory.CreateGitHubClient().GetStringAsync(searchUri, cancellationToken) - .ContinueWith(task => JsonSerializer.Deserialize(task.Result), cancellationToken); + .ContinueWith(task => task.Deserialize(), cancellationToken); } private static Uri BuildUri(string path, Dictionary? queryString = null) @@ -113,4 +103,13 @@ private static Uri BuildUri(string path, Dictionary? queryString return uriBuilder.Uri; } + + private class GitHubSearchResults + { + [JsonInclude, JsonPropertyName("total_count")] + public int TotalCount { get; private set; } + + [JsonInclude, JsonPropertyName("items")] + public GitHubRepo[] Items { get; private set; } = null!; + } } diff --git a/src/ScoopSearch.Indexer/GitHub/GitHubSearchResults.cs b/src/ScoopSearch.Indexer/GitHub/GitHubSearchResults.cs deleted file mode 100644 index 8525e28..0000000 --- a/src/ScoopSearch.Indexer/GitHub/GitHubSearchResults.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.Text.Json.Serialization; - -namespace ScoopSearch.Indexer.GitHub; - -public class GitHubSearchResults -{ - [JsonInclude, JsonPropertyName("total_count")] - public int TotalCount { get; private set; } - - [JsonInclude, JsonPropertyName("items")] - public GitHubRepo[] Items { get; private set; } = null!; -} diff --git a/src/ScoopSearch.Indexer/GitLab/GitLabClient.cs b/src/ScoopSearch.Indexer/GitLab/GitLabClient.cs new file mode 100644 index 0000000..4e4c39c --- /dev/null +++ b/src/ScoopSearch.Indexer/GitLab/GitLabClient.cs @@ -0,0 +1,69 @@ +using System.Net; +using System.Runtime.CompilerServices; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using ScoopSearch.Indexer.Extensions; + +namespace ScoopSearch.Indexer.GitLab; + +internal class GitLabClient : IGitLabClient +{ + private const string GitLabApiBaseUrl = "https://gitlab.com/api/v4"; + private const int ResultsPerPage = 100; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + + public GitLabClient(IHttpClientFactory httpClientFactory, ILogger logger) + { + _httpClientFactory = httpClientFactory; + _logger = logger; + } + + public async Task GetRepositoryAsync(Uri uri, CancellationToken cancellationToken) + { + var absolutePath = uri.AbsolutePath[1..]; + if (absolutePath.Count(c => c == '/') != 1) + { + _logger.LogWarning("{Uri} doesn't appear to be a valid GitLab project", uri); + return null; + } + + var apiUri = $"{GitLabApiBaseUrl}/projects/{WebUtility.UrlEncode(absolutePath)}"; + return await _httpClientFactory.CreateDefaultClient().GetStringAsync(apiUri, cancellationToken) + .ContinueWith(task => task.Deserialize(), cancellationToken); + } + + public async IAsyncEnumerable SearchRepositoriesAsync(string query, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var gitLabTopics = await _httpClientFactory.CreateGitLabClient().GetStringAsync(GitLabApiBaseUrl + $"/topics?search={WebUtility.UrlEncode(query)}&per_page={ResultsPerPage}", cancellationToken) + .ContinueWith(task => task.Deserialize(), cancellationToken); + + if (gitLabTopics != null) + { + foreach (var gitLabTopic in gitLabTopics) + { + var gitLabRepos = await _httpClientFactory.CreateGitLabClient().GetStringAsync(GitLabApiBaseUrl + $"/projects?topic_id={gitLabTopic.Id}&per_page={ResultsPerPage}", cancellationToken) + .ContinueWith(task => task.Deserialize(), cancellationToken); + + if (gitLabRepos != null) + { + foreach (var gitLabRepo in gitLabRepos) + { + yield return gitLabRepo; + } + } + } + } + + } + + private class GitLabTopic + { + [JsonInclude, JsonPropertyName("id")] + public int Id { get; private set; } + + [JsonInclude, JsonPropertyName("total_projects_count")] + public int Projects { get; private set; } + } +} diff --git a/src/ScoopSearch.Indexer/GitLab/GitLabRepo.cs b/src/ScoopSearch.Indexer/GitLab/GitLabRepo.cs new file mode 100644 index 0000000..a50d1b6 --- /dev/null +++ b/src/ScoopSearch.Indexer/GitLab/GitLabRepo.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace ScoopSearch.Indexer.GitLab; + +public class GitLabRepo +{ + [JsonInclude, JsonPropertyName("web_url")] + public Uri WebUrl { get; private set; } = null!; + + [JsonInclude, JsonPropertyName("star_count")] + public int Stars { get; private set; } +} diff --git a/src/ScoopSearch.Indexer/GitLab/IGitLabClient.cs b/src/ScoopSearch.Indexer/GitLab/IGitLabClient.cs new file mode 100644 index 0000000..09d77e4 --- /dev/null +++ b/src/ScoopSearch.Indexer/GitLab/IGitLabClient.cs @@ -0,0 +1,8 @@ +namespace ScoopSearch.Indexer.GitLab; + +public interface IGitLabClient +{ + Task GetRepositoryAsync(Uri uri, CancellationToken cancellationToken); + + IAsyncEnumerable SearchRepositoriesAsync(string query, CancellationToken cancellationToken); +} diff --git a/src/ScoopSearch.Indexer/ServicesExtensions.cs b/src/ScoopSearch.Indexer/ServicesExtensions.cs index b7fa9f6..c7a446a 100644 --- a/src/ScoopSearch.Indexer/ServicesExtensions.cs +++ b/src/ScoopSearch.Indexer/ServicesExtensions.cs @@ -6,6 +6,7 @@ using ScoopSearch.Indexer.Extensions; using ScoopSearch.Indexer.Git; using ScoopSearch.Indexer.GitHub; +using ScoopSearch.Indexer.GitLab; using ScoopSearch.Indexer.Indexer; using ScoopSearch.Indexer.Manifest; using ScoopSearch.Indexer.Processor; @@ -26,23 +27,27 @@ public static void RegisterScoopSearchIndexer(this IServiceCollection @this) @this .AddOptions() .Configure((options, configuration) => configuration.GetRequiredSection(GitHubOptions.Key).Bind(options)); + @this + .AddOptions() + .Configure((options, configuration) => configuration.GetRequiredSection(GitLabOptions.Key).Bind(options)); // Services @this.AddHttpClients(); @this.AddSingleton(); @this.AddSingleton(); + @this.AddSingleton(); @this.AddSingleton(); @this.AddSingleton(); @this.AddSingleton(); @this.AddSingleton(); @this.AddSingleton(); - // TODO Add other providers (GitLab...) + @this.AddSingleton(); @this.AddSingleton(); @this.AddSingleton(); @this.AddSingleton(); - // TODO Add other sources (GitLab...) + @this.AddSingleton(); @this.AddSingleton(); @this.AddSingleton(); diff --git a/src/ScoopSearch.Indexer/appsettings.json b/src/ScoopSearch.Indexer/appsettings.json index d517bc5..6a91af5 100644 --- a/src/ScoopSearch.Indexer/appsettings.json +++ b/src/ScoopSearch.Indexer/appsettings.json @@ -22,6 +22,16 @@ ] }, + "GitLab": { + // GitLab API token with read_api scope + "Token": "", + + "BucketsSearchQueries": [ + "scoop-bucket", + "scoop" + ] + }, + "Buckets": { "OfficialBucketsListUrl": "https://raw.githubusercontent.com/ScoopInstaller/Scoop/master/buckets.json", @@ -31,7 +41,8 @@ // No manifests inside "https://github.com/lukesampson/scoop", "https://github.com/frostming/scoop-action", - "https://github.com/rasa/scoop-directory" + "https://github.com/rasa/scoop-directory", + "https://gitlab.com/megabyte-labs/cloud/scoops" ], "ManualBuckets": [