Skip to content

Introduce HMCR throttling #694

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ionwyn wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Introduce HMCR throttling #694

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3e7d2c6
Introduce HMCR throttling
ionwyn Jun 11, 2025
9f4c6b0
Fix post with retry
ionwyn Jun 11, 2025
76ba92f
Increase throttle time
ionwyn Jun 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 29 additions & 22 deletions api/Hmcr.Chris/Api.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,38 +73,45 @@ public async Task<HttpResponseMessage> GetWithRetry(HttpClient client, string pa

public async Task<HttpResponseMessage> PostWithRetry(HttpClient client, string path, string body)
{
var response
= await client.PostAsync(path, new StringContent(body, Encoding.UTF8));
using var content = new StringContent(body, Encoding.UTF8, "application/xml");

if (!response.IsSuccessStatusCode)
{
for (var attempt = 2; attempt <= maxAttempt; attempt++)
{
await Task.Delay(100 * attempt);
HttpResponseMessage response = null;
Exception lastException = null;

response = await client.PostAsync(path, new StringContent(body, Encoding.UTF8));
for (int attempt = 1; attempt <= maxAttempt; attempt++)
{
try
{
response = await client.PostAsync(path, content);

if (response.IsSuccessStatusCode)
{
break;
return response;
}
else if (attempt == maxAttempt)
{
string message = "";
}
catch (Exception ex)
{
lastException = ex;
}

if (response.Content != null)
{
var bytes = await response.Content.ReadAsByteArrayAsync();
message = Encoding.UTF8.GetString(bytes);
}
if (attempt < maxAttempt)
{
await Task.Delay(100 * attempt);
}
}

throw new Exception($"Status Code: {response.StatusCode}" + Environment.NewLine + message);
}
}
// Final failure
string message = "";

if (response?.Content != null)
{
var bytes = await response.Content.ReadAsByteArrayAsync();
message = Encoding.UTF8.GetString(bytes);
}

return response;
throw new Exception($"Failed POST to {path}. " +
(response != null ? $"Status Code: {response.StatusCode}" : "") +
Environment.NewLine + message, lastException);
}

}
}
70 changes: 60 additions & 10 deletions api/Hmcr.Chris/OasApi.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using System.Xml.Linq;

namespace Hmcr.Chris
{
Expand Down Expand Up @@ -74,29 +78,75 @@ public OasApi(HttpClient client, IApi api, IConfiguration config, ILogger<OasApi
_path = config["CHRIS:OASPath"];
_logger = logger;
}

public async Task<bool> IsPointOnRfiSegmentAsync(int tolerance, Point point, string rfiSegment)
{
var body = "";
var content = "";
string Clean(string input) => input?.Replace("\0", "") ?? "";

try
string rfiSegmentClean = Clean(rfiSegment);
string lon = Clean(point.Longitude.ToString(CultureInfo.InvariantCulture));
string lat = Clean(point.Latitude.ToString(CultureInfo.InvariantCulture));

string body = string.Format(_queries.PointOnRfiSegQuery, tolerance, lon, lat, rfiSegmentClean);
string content = "";
Exception lastException = null;

if (body.Contains('\0', StringComparison.Ordinal))
{
body = string.Format(_queries.PointOnRfiSegQuery, tolerance, point.Longitude, point.Latitude, rfiSegment);
_logger.LogError("XML body contains NULL character prior to transmission.");

content = await (await _api.PostWithRetry(_client, _path, body)).Content.ReadAsStringAsync();
var bytes = Encoding.UTF8.GetBytes(body);
var dumpPath = Path.Combine("/tmp", $"corrupted_body_{DateTime.UtcNow:yyyyMMdd_HHmmss}.bin");
await File.WriteAllBytesAsync(dumpPath, bytes);

var features = JsonSerializer.Deserialize<FeatureCollection<decimal[]>>(content);
_logger.LogError($"Corrupted XML body written to {dumpPath} for inspection.");
throw new Exception("Aborting request: XML contains illegal null character.");
}

return features.numberMatched > 0;
try
{
_ = XDocument.Parse(body);
}
catch (Exception ex)
{
_logger.LogError($"Exception - IsPointOnRfiSegmentAsync: {body} - {content}");
throw ex;
_logger.LogError(ex, "XML body failed validation before transmission.");
throw;
}

_logger.LogDebug($"IsPointOnRfiSegmentAsync - body: {body}");
_logger.LogInformation($"IsPointOnRfiSegmentAsync - rfiSegment: {rfiSegmentClean}, point: ({lon}, {lat}), tolerance: {tolerance}");

for (int i = 0; i < 5; i++)
{
try
{
var response = await _api.PostWithRetry(_client, _path, body);

if (response.Content.Headers.ContentType?.MediaType != "application/json")
{
content = await response.Content.ReadAsStringAsync();
_logger.LogWarning($"Non-JSON response (attempt {i + 1}): {content}");
throw new Exception("Invalid content type");
}

content = await response.Content.ReadAsStringAsync();
var features = JsonSerializer.Deserialize<FeatureCollection<decimal[]>>(content);

return features?.numberMatched > 0;
}
catch (Exception ex)
{
lastException = ex;
_logger.LogWarning(ex, $"Retry {i + 1} failed for IsPointOnRfiSegmentAsync with body: {body}");
await Task.Delay((int)Math.Pow(2, i) * 500); // 500ms, 1000ms, 2000ms, 4000ms, 8000ms
}
}

_logger.LogError($"All retries failed for IsPointOnRfiSegmentAsync: {body} - {content}");
throw new Exception("Failed to determine if point is on RFI segment after multiple attempts.", lastException);
}



public async Task<List<Line>> GetLineFromOffsetMeasureOnRfiSegmentAsync(string rfiSegment, decimal start, decimal end)
{
var query = "";
Expand Down
19 changes: 18 additions & 1 deletion api/Hmcr.Domain/Services/SpatialService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using Hmcr.Chris.Models;
using Hmcr.Model;
using Hmcr.Model.Utils;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Linq;
Expand Down Expand Up @@ -36,6 +37,7 @@ public class SpatialService : ISpatialService

private IFieldValidatorService _validator;
private ILookupCodeService _lookupService;
private ILogger<OasApi> _logger;

private IEnumerable<string> _nonSpHighwayUniques = null;
private IEnumerable<string> NonSpHighwayUniques => _nonSpHighwayUniques ??= _validator.CodeLookup.Where(x => x.CodeSet == CodeSet.NonSpHighwayUnique).Select(x => x.CodeValue).ToArray().ToLowercase();
Expand All @@ -59,7 +61,22 @@ public SpatialService(IOasApi oasApi, IFieldValidatorService validator, ILookupC

var threshold = _lookupService.GetThresholdLevel(thresholdLevel);

var isOnRfi = await _oasApi.IsPointOnRfiSegmentAsync(threshold.Error, point, rfiSegment);
bool isOnRfi;

try
{
isOnRfi = await _oasApi.IsPointOnRfiSegmentAsync(threshold.Error, point, rfiSegment);
}
catch (Exception ex)
{
// Log detailed error
_logger.LogError(ex, $"Exception during WFS check for RFI segment '{rfiSegment}' at point [{point.Latitude}, {point.Longitude}]");

// Surface error to validation errors
errors.AddItem("WFS Error", $"Failed to validate GPS point against RFI segment [{rfiSegment}]. Details: {ex.Message}");

return (SpValidationResult.Fail, null, rfiResult.segment);
}

if (!isOnRfi)
{
Expand Down
Loading