feat(rate-limiting): Implement core rate limiting functionality with configuration, decision-making, metrics, middleware, and service registration
- Add RateLimitConfig for configuration management with YAML binding support. - Introduce RateLimitDecision to encapsulate the result of rate limit checks. - Implement RateLimitMetrics for OpenTelemetry metrics tracking. - Create RateLimitMiddleware for enforcing rate limits on incoming requests. - Develop RateLimitService to orchestrate instance and environment rate limit checks. - Add RateLimitServiceCollectionExtensions for dependency injection registration.
This commit is contained in:
master
@@ -0,0 +1,282 @@
|
||||
using System.Globalization;
|
||||
using System.IO.Compression;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text.RegularExpressions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Concelier.Epss.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Epss.Parsing;
|
||||
|
||||
/// <summary>
|
||||
/// Parses EPSS CSV stream from FIRST.org into structured <see cref="EpssScoreRow"/> records.
|
||||
/// Handles GZip compression, leading comment line extraction, and row validation.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// EPSS CSV format (FIRST.org):
|
||||
/// - Leading comment line (optional): <c># model: v2025.03.14, published: 2025-03-14</c>
|
||||
/// - Header line: <c>cve,epss,percentile</c>
|
||||
/// - Data rows: <c>CVE-2024-12345,0.42357,0.88234</c>
|
||||
///
|
||||
/// Reference: https://www.first.org/epss/data_stats
|
||||
/// </remarks>
|
||||
public sealed class EpssCsvStreamParser : IDisposable
|
||||
{
|
||||
private readonly Stream _sourceStream;
|
||||
private readonly DateOnly _modelDate;
|
||||
private readonly ILogger<EpssCsvStreamParser> _logger;
|
||||
private readonly bool _isCompressed;
|
||||
|
||||
// Regex for comment line: # model: v2025.03.14, published: 2025-03-14
|
||||
private static readonly Regex CommentLineRegex = new(
|
||||
@"^#\s*model:\s*(?<version>v?[\d.]+)\s*,\s*published:\s*(?<date>\d{4}-\d{2}-\d{2})",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Metadata extracted from CSV comment line (if present).
|
||||
/// </summary>
|
||||
public EpssModelMetadata? ModelMetadata { get; private set; }
|
||||
|
||||
public EpssCsvStreamParser(
|
||||
Stream sourceStream,
|
||||
DateOnly modelDate,
|
||||
bool isCompressed = true,
|
||||
ILogger<EpssCsvStreamParser>? logger = null)
|
||||
{
|
||||
_sourceStream = sourceStream ?? throw new ArgumentNullException(nameof(sourceStream));
|
||||
_modelDate = modelDate;
|
||||
_isCompressed = isCompressed;
|
||||
_logger = logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger<EpssCsvStreamParser>.Instance;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses EPSS CSV stream into an async enumerable of validated rows.
|
||||
/// Yields rows incrementally for memory-efficient streaming.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token</param>
|
||||
/// <returns>Async enumerable of parsed and validated EPSS score rows</returns>
|
||||
public async IAsyncEnumerable<EpssScoreRow> ParseAsync(
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
var stream = _isCompressed
|
||||
? new GZipStream(_sourceStream, CompressionMode.Decompress, leaveOpen: false)
|
||||
: _sourceStream;
|
||||
|
||||
using var reader = new StreamReader(stream);
|
||||
|
||||
var lineNumber = 0;
|
||||
var rowsYielded = 0;
|
||||
var rowsSkipped = 0;
|
||||
|
||||
// Read first line - may be comment, may be header
|
||||
lineNumber++;
|
||||
var firstLine = await reader.ReadLineAsync(cancellationToken);
|
||||
if (string.IsNullOrWhiteSpace(firstLine))
|
||||
{
|
||||
_logger.LogWarning("EPSS CSV is empty (model_date: {ModelDate})", _modelDate);
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Try to extract model metadata from comment line
|
||||
if (firstLine.StartsWith('#'))
|
||||
{
|
||||
ModelMetadata = TryParseCommentLine(firstLine);
|
||||
if (ModelMetadata is not null)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"EPSS CSV metadata: model_version={ModelVersion}, published_date={PublishedDate}",
|
||||
ModelMetadata.ModelVersion,
|
||||
ModelMetadata.PublishedDate);
|
||||
}
|
||||
|
||||
// Read header line
|
||||
lineNumber++;
|
||||
var headerLine = await reader.ReadLineAsync(cancellationToken);
|
||||
if (!IsValidHeader(headerLine))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"EPSS CSV has invalid header (expected: cve,epss,percentile, got: {Header})",
|
||||
headerLine);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// First line is header (no comment)
|
||||
if (!IsValidHeader(firstLine))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"EPSS CSV has invalid header (expected: cve,epss,percentile, got: {Header})",
|
||||
firstLine);
|
||||
}
|
||||
}
|
||||
|
||||
// Parse data rows
|
||||
await foreach (var line in ReadLinesAsync(reader, cancellationToken))
|
||||
{
|
||||
lineNumber++;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(line) || line.StartsWith('#'))
|
||||
{
|
||||
continue; // Skip blank lines and additional comments
|
||||
}
|
||||
|
||||
var row = TryParseRow(line, lineNumber);
|
||||
if (row is null)
|
||||
{
|
||||
rowsSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
rowsYielded++;
|
||||
yield return row;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"EPSS CSV parsed: model_date={ModelDate}, rows_yielded={RowsYielded}, rows_skipped={RowsSkipped}",
|
||||
_modelDate,
|
||||
rowsYielded,
|
||||
rowsSkipped);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to extract model metadata from CSV comment line.
|
||||
/// Example: "# model: v2025.03.14, published: 2025-03-14"
|
||||
/// </summary>
|
||||
private EpssModelMetadata? TryParseCommentLine(string commentLine)
|
||||
{
|
||||
var match = CommentLineRegex.Match(commentLine);
|
||||
if (!match.Success)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var versionStr = match.Groups["version"].Value;
|
||||
var dateStr = match.Groups["date"].Value;
|
||||
|
||||
if (DateOnly.TryParseExact(dateStr, "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var publishedDate))
|
||||
{
|
||||
return new EpssModelMetadata
|
||||
{
|
||||
ModelVersion = versionStr,
|
||||
PublishedDate = publishedDate
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates CSV header line.
|
||||
/// Expected: "cve,epss,percentile" (case-insensitive)
|
||||
/// </summary>
|
||||
private bool IsValidHeader(string? headerLine)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(headerLine))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var normalized = headerLine.Replace(" ", "").ToLowerInvariant();
|
||||
return normalized == "cve,epss,percentile";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses a single CSV row into <see cref="EpssScoreRow"/>.
|
||||
/// Returns null if row is malformed or invalid.
|
||||
/// </summary>
|
||||
private EpssScoreRow? TryParseRow(string line, int lineNumber)
|
||||
{
|
||||
var parts = line.Split(',');
|
||||
if (parts.Length < 3)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"EPSS CSV line {LineNumber}: insufficient columns (expected 3, got {Count}): {Line}",
|
||||
lineNumber,
|
||||
parts.Length,
|
||||
line.Length > 100 ? line[..100] : line);
|
||||
return null;
|
||||
}
|
||||
|
||||
var cveId = parts[0].Trim();
|
||||
var epssScoreStr = parts[1].Trim();
|
||||
var percentileStr = parts[2].Trim();
|
||||
|
||||
// Parse score
|
||||
if (!double.TryParse(epssScoreStr, NumberStyles.Float, CultureInfo.InvariantCulture, out var epssScore))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"EPSS CSV line {LineNumber}: invalid epss_score '{EpssScore}' for CVE {CveId}",
|
||||
lineNumber,
|
||||
epssScoreStr,
|
||||
cveId);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Parse percentile
|
||||
if (!double.TryParse(percentileStr, NumberStyles.Float, CultureInfo.InvariantCulture, out var percentile))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"EPSS CSV line {LineNumber}: invalid percentile '{Percentile}' for CVE {CveId}",
|
||||
lineNumber,
|
||||
percentileStr,
|
||||
cveId);
|
||||
return null;
|
||||
}
|
||||
|
||||
var row = new EpssScoreRow
|
||||
{
|
||||
CveId = cveId,
|
||||
EpssScore = epssScore,
|
||||
Percentile = percentile,
|
||||
ModelDate = _modelDate,
|
||||
LineNumber = lineNumber
|
||||
};
|
||||
|
||||
// Validate bounds
|
||||
if (!row.IsValid(out var validationError))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"EPSS CSV line {LineNumber}: validation failed for CVE {CveId}: {Error}",
|
||||
lineNumber,
|
||||
cveId,
|
||||
validationError);
|
||||
return null;
|
||||
}
|
||||
|
||||
return row;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads lines from StreamReader as async enumerable.
|
||||
/// </summary>
|
||||
private static async IAsyncEnumerable<string> ReadLinesAsync(
|
||||
StreamReader reader,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
while (!reader.EndOfStream)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var line = await reader.ReadLineAsync(cancellationToken);
|
||||
if (line is not null)
|
||||
{
|
||||
yield return line;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_sourceStream.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Metadata extracted from EPSS CSV comment line.
|
||||
/// </summary>
|
||||
public sealed record EpssModelMetadata
|
||||
{
|
||||
/// <summary>EPSS model version (e.g., "v2025.03.14" or "2025.03.14")</summary>
|
||||
public required string ModelVersion { get; init; }
|
||||
|
||||
/// <summary>Date the model was published by FIRST.org</summary>
|
||||
public required DateOnly PublishedDate { get; init; }
|
||||
}
|
||||
Reference in New Issue
Block a user