git.stella-ops.org/src/Concelier/__Libraries/StellaOps.Concelier.Epss/Parsing/EpssCsvStreamParser.cs

using System.Globalization;
using System.IO.Compression;
using System.Runtime.CompilerServices;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Epss.Models;

namespace StellaOps.Concelier.Epss.Parsing;

/// <summary>
/// Parses EPSS CSV stream from FIRST.org into structured <see cref="EpssScoreRow"/> records.
/// Handles GZip compression, leading comment line extraction, and row validation.
/// </summary>
/// <remarks>
/// EPSS CSV format (FIRST.org):
/// - Leading comment line (optional): <c># model: v2025.03.14, published: 2025-03-14</c>
/// - Header line: <c>cve,epss,percentile</c>
/// - Data rows: <c>CVE-2024-12345,0.42357,0.88234</c>
///
/// Reference: https://www.first.org/epss/data_stats
/// </remarks>
public sealed class EpssCsvStreamParser : IDisposable
{
    private readonly Stream _sourceStream;
    private readonly DateOnly _modelDate;
    private readonly ILogger<EpssCsvStreamParser> _logger;
    private readonly bool _isCompressed;

    // Regex for comment line: # model: v2025.03.14, published: 2025-03-14
    private static readonly Regex CommentLineRegex = new(
        @"^#\s*model:\s*(?<version>v?[\d.]+)\s*,\s*published:\s*(?<date>\d{4}-\d{2}-\d{2})",
        RegexOptions.Compiled | RegexOptions.IgnoreCase);

    /// <summary>
    /// Metadata extracted from CSV comment line (if present).
    /// </summary>
    public EpssModelMetadata? ModelMetadata { get; private set; }

    public EpssCsvStreamParser(
        Stream sourceStream,
        DateOnly modelDate,
        bool isCompressed = true,
        ILogger<EpssCsvStreamParser>? logger = null)
    {
        _sourceStream = sourceStream ?? throw new ArgumentNullException(nameof(sourceStream));
        _modelDate = modelDate;
        _isCompressed = isCompressed;
        _logger = logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger<EpssCsvStreamParser>.Instance;
    }

    /// <summary>
    /// Parses EPSS CSV stream into an async enumerable of validated rows.
    /// Yields rows incrementally for memory-efficient streaming.
    /// </summary>
    /// <param name="cancellationToken">Cancellation token</param>
    /// <returns>Async enumerable of parsed and validated EPSS score rows</returns>
    public async IAsyncEnumerable<EpssScoreRow> ParseAsync(
        [EnumeratorCancellation] CancellationToken cancellationToken = default)
    {
        var stream = _isCompressed
            ? new GZipStream(_sourceStream, CompressionMode.Decompress, leaveOpen: false)
            : _sourceStream;

        using var reader = new StreamReader(stream);

        var lineNumber = 0;
        var rowsYielded = 0;
        var rowsSkipped = 0;

        // Read first line - may be comment, may be header
        lineNumber++;
        var firstLine = await reader.ReadLineAsync(cancellationToken);
        if (string.IsNullOrWhiteSpace(firstLine))
        {
            _logger.LogWarning("EPSS CSV is empty (model_date: {ModelDate})", _modelDate);
            yield break;
        }

        // Try to extract model metadata from comment line
        if (firstLine.StartsWith('#'))
        {
            ModelMetadata = TryParseCommentLine(firstLine);
            if (ModelMetadata is not null)
            {
                _logger.LogInformation(
                    "EPSS CSV metadata: model_version={ModelVersion}, published_date={PublishedDate}",
                    ModelMetadata.ModelVersion,
                    ModelMetadata.PublishedDate);
            }

            // Read header line
            lineNumber++;
            var headerLine = await reader.ReadLineAsync(cancellationToken);
            if (!IsValidHeader(headerLine))
            {
                _logger.LogWarning(
                    "EPSS CSV has invalid header (expected: cve,epss,percentile, got: {Header})",
                    headerLine);
            }
        }
        else
        {
            // First line is header (no comment)
            if (!IsValidHeader(firstLine))
            {
                _logger.LogWarning(
                    "EPSS CSV has invalid header (expected: cve,epss,percentile, got: {Header})",
                    firstLine);
            }
        }

        // Parse data rows
        await foreach (var line in ReadLinesAsync(reader, cancellationToken))
        {
            lineNumber++;

            if (string.IsNullOrWhiteSpace(line) || line.StartsWith('#'))
            {
                continue;  // Skip blank lines and additional comments
            }

            var row = TryParseRow(line, lineNumber);
            if (row is null)
            {
                rowsSkipped++;
                continue;
            }

            rowsYielded++;
            yield return row;
        }

        _logger.LogInformation(
            "EPSS CSV parsed: model_date={ModelDate}, rows_yielded={RowsYielded}, rows_skipped={RowsSkipped}",
            _modelDate,
            rowsYielded,
            rowsSkipped);
    }

    /// <summary>
    /// Attempts to extract model metadata from CSV comment line.
    /// Example: "# model: v2025.03.14, published: 2025-03-14"
    /// </summary>
    private EpssModelMetadata? TryParseCommentLine(string commentLine)
    {
        var match = CommentLineRegex.Match(commentLine);
        if (!match.Success)
        {
            return null;
        }

        var versionStr = match.Groups["version"].Value;
        var dateStr = match.Groups["date"].Value;

        if (DateOnly.TryParseExact(dateStr, "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var publishedDate))
        {
            return new EpssModelMetadata
            {
                ModelVersion = versionStr,
                PublishedDate = publishedDate
            };
        }

        return null;
    }

    /// <summary>
    /// Validates CSV header line.
    /// Expected: "cve,epss,percentile" (case-insensitive)
    /// </summary>
    private bool IsValidHeader(string? headerLine)
    {
        if (string.IsNullOrWhiteSpace(headerLine))
        {
            return false;
        }

        var normalized = headerLine.Replace(" ", "").ToLowerInvariant();
        return normalized == "cve,epss,percentile";
    }

    /// <summary>
    /// Parses a single CSV row into <see cref="EpssScoreRow"/>.
    /// Returns null if row is malformed or invalid.
    /// </summary>
    private EpssScoreRow? TryParseRow(string line, int lineNumber)
    {
        var parts = line.Split(',');
        if (parts.Length < 3)
        {
            _logger.LogWarning(
                "EPSS CSV line {LineNumber}: insufficient columns (expected 3, got {Count}): {Line}",
                lineNumber,
                parts.Length,
                line.Length > 100 ? line[..100] : line);
            return null;
        }

        var cveId = parts[0].Trim();
        var epssScoreStr = parts[1].Trim();
        var percentileStr = parts[2].Trim();

        // Parse score
        if (!double.TryParse(epssScoreStr, NumberStyles.Float, CultureInfo.InvariantCulture, out var epssScore))
        {
            _logger.LogWarning(
                "EPSS CSV line {LineNumber}: invalid epss_score '{EpssScore}' for CVE {CveId}",
                lineNumber,
                epssScoreStr,
                cveId);
            return null;
        }

        // Parse percentile
        if (!double.TryParse(percentileStr, NumberStyles.Float, CultureInfo.InvariantCulture, out var percentile))
        {
            _logger.LogWarning(
                "EPSS CSV line {LineNumber}: invalid percentile '{Percentile}' for CVE {CveId}",
                lineNumber,
                percentileStr,
                cveId);
            return null;
        }

        var row = new EpssScoreRow
        {
            CveId = cveId,
            EpssScore = epssScore,
            Percentile = percentile,
            ModelDate = _modelDate,
            LineNumber = lineNumber
        };

        // Validate bounds
        if (!row.IsValid(out var validationError))
        {
            _logger.LogWarning(
                "EPSS CSV line {LineNumber}: validation failed for CVE {CveId}: {Error}",
                lineNumber,
                cveId,
                validationError);
            return null;
        }

        return row;
    }

    /// <summary>
    /// Reads lines from StreamReader as async enumerable.
    /// </summary>
    private static async IAsyncEnumerable<string> ReadLinesAsync(
        StreamReader reader,
        [EnumeratorCancellation] CancellationToken cancellationToken)
    {
        while (!reader.EndOfStream)
        {
            cancellationToken.ThrowIfCancellationRequested();
            var line = await reader.ReadLineAsync(cancellationToken);
            if (line is not null)
            {
                yield return line;
            }
        }
    }

    public void Dispose()
    {
        _sourceStream.Dispose();
    }
}

/// <summary>
/// Metadata extracted from EPSS CSV comment line.
/// </summary>
public sealed record EpssModelMetadata
{
    /// <summary>EPSS model version (e.g., "v2025.03.14" or "2025.03.14")</summary>
    public required string ModelVersion { get; init; }

    /// <summary>Date the model was published by FIRST.org</summary>
    public required DateOnly PublishedDate { get; init; }
}