up

src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaDiagnostics.cs

@@ -16,6 +16,12 @@ public sealed class GhsaDiagnostics : IDisposable
     private readonly Counter<long> _parseFailures;
     private readonly Counter<long> _parseQuarantine;
     private readonly Counter<long> _mapSuccess;
+    private readonly Histogram<long> _rateLimitRemaining;
+    private readonly Histogram<long> _rateLimitLimit;
+    private readonly Histogram<double> _rateLimitResetSeconds;
+    private readonly Counter<long> _rateLimitExhausted;
+    private readonly object _rateLimitLock = new();
+    private GhsaRateLimitSnapshot? _lastRateLimitSnapshot;
 
     public GhsaDiagnostics()
     {
@@ -28,6 +34,10 @@ public sealed class GhsaDiagnostics : IDisposable
         _parseFailures = _meter.CreateCounter<long>("ghsa.parse.failures", unit: "documents");
         _parseQuarantine = _meter.CreateCounter<long>("ghsa.parse.quarantine", unit: "documents");
         _mapSuccess = _meter.CreateCounter<long>("ghsa.map.success", unit: "advisories");
+        _rateLimitRemaining = _meter.CreateHistogram<long>("ghsa.ratelimit.remaining", unit: "requests");
+        _rateLimitLimit = _meter.CreateHistogram<long>("ghsa.ratelimit.limit", unit: "requests");
+        _rateLimitResetSeconds = _meter.CreateHistogram<double>("ghsa.ratelimit.reset_seconds", unit: "s");
+        _rateLimitExhausted = _meter.CreateCounter<long>("ghsa.ratelimit.exhausted", unit: "events");
     }
 
     public void FetchAttempt() => _fetchAttempts.Add(1);
@@ -46,5 +56,45 @@ public sealed class GhsaDiagnostics : IDisposable
 
     public void MapSuccess(long count) => _mapSuccess.Add(count);
 
+    internal void RecordRateLimit(GhsaRateLimitSnapshot snapshot)
+    {
+        var tags = new KeyValuePair<string, object?>[]
+        {
+            new("phase", snapshot.Phase),
+            new("resource", snapshot.Resource ?? "unknown")
+        };
+
+        if (snapshot.Limit.HasValue)
+        {
+            _rateLimitLimit.Record(snapshot.Limit.Value, tags);
+        }
+
+        if (snapshot.Remaining.HasValue)
+        {
+            _rateLimitRemaining.Record(snapshot.Remaining.Value, tags);
+        }
+
+        if (snapshot.ResetAfter.HasValue)
+        {
+            _rateLimitResetSeconds.Record(snapshot.ResetAfter.Value.TotalSeconds, tags);
+        }
+
+        lock (_rateLimitLock)
+        {
+            _lastRateLimitSnapshot = snapshot;
+        }
+    }
+
+    internal void RateLimitExhausted(string phase)
+        => _rateLimitExhausted.Add(1, new KeyValuePair<string, object?>("phase", phase));
+
+    internal GhsaRateLimitSnapshot? GetLastRateLimitSnapshot()
+    {
+        lock (_rateLimitLock)
+        {
+            return _lastRateLimitSnapshot;
+        }
+    }
+
     public void Dispose() => _meter.Dispose();
 }

src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaMapper.cs

@@ -1,6 +1,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using StellaOps.Feedser.Models;
+using StellaOps.Feedser.Normalization.SemVer;
 using StellaOps.Feedser.Storage.Mongo.Documents;
 
 namespace StellaOps.Feedser.Source.Ghsa.Internal;
@@ -120,29 +121,9 @@ internal static class GhsaMapper
             var rangeKind = SemVerEcosystems.Contains(ecosystem) ? "semver" : "vendor";
             var packageType = SemVerEcosystems.Contains(ecosystem) ? AffectedPackageTypes.SemVer : AffectedPackageTypes.Vendor;
 
-            var versionRanges = new List<AffectedVersionRange>();
-            if (!string.IsNullOrWhiteSpace(affected.VulnerableRange) || !string.IsNullOrWhiteSpace(affected.PatchedVersion))
-            {
-                var primitives = new RangePrimitives(null, null, null, new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
-                {
-                    ["ecosystem"] = ecosystem,
-                    ["package"] = packageName,
-                });
-
-                versionRanges.Add(new AffectedVersionRange(
-                    rangeKind,
-                    introducedVersion: null,
-                    fixedVersion: Validation.TrimToNull(affected.PatchedVersion),
-                    lastAffectedVersion: null,
-                    rangeExpression: Validation.TrimToNull(affected.VulnerableRange),
-                    provenance: new AdvisoryProvenance(
-                        GhsaConnectorPlugin.SourceName,
-                        "affected-range",
-                        identifier,
-                        recordedAt,
-                        new[] { ProvenanceFieldMasks.VersionRanges }),
-                    primitives: primitives));
-            }
+            var (ranges, normalizedVersions) = SemVerEcosystems.Contains(ecosystem)
+                ? CreateSemVerVersionArtifacts(affected, identifier, ecosystem, packageName, recordedAt)
+                : CreateVendorVersionArtifacts(affected, rangeKind, identifier, ecosystem, packageName, recordedAt);
 
             var statuses = new[]
             {
@@ -160,9 +141,10 @@ internal static class GhsaMapper
                 packageType,
                 identifier,
                 platform: null,
-                versionRanges: versionRanges,
+                versionRanges: ranges,
                 statuses: statuses,
-                provenance: provenance));
+                provenance: provenance,
+                normalizedVersions: normalizedVersions));
         }
 
         return packages;
@@ -206,4 +188,142 @@ internal static class GhsaMapper
 
         return results.Count == 0 ? Array.Empty<AdvisoryCredit>() : results;
     }
+
+    private static (IReadOnlyList<AffectedVersionRange> Ranges, IReadOnlyList<NormalizedVersionRule> Normalized) CreateSemVerVersionArtifacts(
+        GhsaAffectedDto affected,
+        string identifier,
+        string ecosystem,
+        string packageName,
+        DateTimeOffset recordedAt)
+    {
+        var note = BuildNormalizedNote(identifier);
+        var results = SemVerRangeRuleBuilder.Build(affected.VulnerableRange, affected.PatchedVersion, note);
+
+        if (results.Count > 0)
+        {
+            var ranges = new List<AffectedVersionRange>(results.Count);
+            var normalized = new List<NormalizedVersionRule>(results.Count);
+
+            foreach (var result in results)
+            {
+                var primitive = result.Primitive;
+                var rangeExpression = ResolveRangeExpression(result.Expression, primitive.ConstraintExpression, affected.VulnerableRange);
+
+                ranges.Add(new AffectedVersionRange(
+                    rangeKind: "semver",
+                    introducedVersion: Validation.TrimToNull(primitive.Introduced),
+                    fixedVersion: Validation.TrimToNull(primitive.Fixed),
+                    lastAffectedVersion: Validation.TrimToNull(primitive.LastAffected),
+                    rangeExpression: rangeExpression,
+                    provenance: CreateRangeProvenance(identifier, recordedAt),
+                    primitives: new RangePrimitives(
+                        SemVer: primitive,
+                        Nevra: null,
+                        Evr: null,
+                        VendorExtensions: CreateVendorExtensions(ecosystem, packageName))));
+
+                normalized.Add(result.NormalizedRule);
+            }
+
+            return (ranges.ToArray(), normalized.ToArray());
+        }
+
+        var fallbackRange = CreateFallbackRange("semver", affected, identifier, ecosystem, packageName, recordedAt);
+        if (fallbackRange is null)
+        {
+            return (Array.Empty<AffectedVersionRange>(), Array.Empty<NormalizedVersionRule>());
+        }
+
+        var fallbackRule = fallbackRange.ToNormalizedVersionRule(note);
+        var normalizedFallback = fallbackRule is null
+            ? Array.Empty<NormalizedVersionRule>()
+            : new[] { fallbackRule };
+
+        return (new[] { fallbackRange }, normalizedFallback);
+    }
+
+    private static (IReadOnlyList<AffectedVersionRange> Ranges, IReadOnlyList<NormalizedVersionRule> Normalized) CreateVendorVersionArtifacts(
+        GhsaAffectedDto affected,
+        string rangeKind,
+        string identifier,
+        string ecosystem,
+        string packageName,
+        DateTimeOffset recordedAt)
+    {
+        var range = CreateFallbackRange(rangeKind, affected, identifier, ecosystem, packageName, recordedAt);
+        if (range is null)
+        {
+            return (Array.Empty<AffectedVersionRange>(), Array.Empty<NormalizedVersionRule>());
+        }
+
+        return (new[] { range }, Array.Empty<NormalizedVersionRule>());
+    }
+
+    private static AffectedVersionRange? CreateFallbackRange(
+        string rangeKind,
+        GhsaAffectedDto affected,
+        string identifier,
+        string ecosystem,
+        string packageName,
+        DateTimeOffset recordedAt)
+    {
+        var fixedVersion = Validation.TrimToNull(affected.PatchedVersion);
+        var rangeExpression = Validation.TrimToNull(affected.VulnerableRange);
+
+        if (fixedVersion is null && rangeExpression is null)
+        {
+            return null;
+        }
+
+        return new AffectedVersionRange(
+            rangeKind,
+            introducedVersion: null,
+            fixedVersion: fixedVersion,
+            lastAffectedVersion: null,
+            rangeExpression: rangeExpression,
+            provenance: CreateRangeProvenance(identifier, recordedAt),
+            primitives: new RangePrimitives(
+                SemVer: null,
+                Nevra: null,
+                Evr: null,
+                VendorExtensions: CreateVendorExtensions(ecosystem, packageName)));
+    }
+
+    private static AdvisoryProvenance CreateRangeProvenance(string identifier, DateTimeOffset recordedAt)
+        => new(
+            GhsaConnectorPlugin.SourceName,
+            "affected-range",
+            identifier,
+            recordedAt,
+            new[] { ProvenanceFieldMasks.VersionRanges });
+
+    private static IReadOnlyDictionary<string, string> CreateVendorExtensions(string ecosystem, string packageName)
+        => new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
+        {
+            ["ecosystem"] = ecosystem,
+            ["package"] = packageName,
+        };
+
+    private static string? BuildNormalizedNote(string identifier)
+    {
+        var trimmed = Validation.TrimToNull(identifier);
+        return trimmed is null ? null : $"ghsa:{trimmed}";
+    }
+
+    private static string? ResolveRangeExpression(string? parsedExpression, string? constraintExpression, string? fallbackExpression)
+    {
+        var parsed = Validation.TrimToNull(parsedExpression);
+        if (parsed is not null)
+        {
+            return parsed;
+        }
+
+        var constraint = Validation.TrimToNull(constraintExpression);
+        if (constraint is not null)
+        {
+            return constraint;
+        }
+
+        return Validation.TrimToNull(fallbackExpression);
+    }
 }

src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRateLimitParser.cs

@@ -0,0 +1,111 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+
+namespace StellaOps.Feedser.Source.Ghsa.Internal;
+
+internal static class GhsaRateLimitParser
+{
+    public static GhsaRateLimitSnapshot? TryParse(IReadOnlyDictionary<string, string>? headers, DateTimeOffset now, string phase)
+    {
+        if (headers is null || headers.Count == 0)
+        {
+            return null;
+        }
+
+        string? resource = null;
+        long? limit = null;
+        long? remaining = null;
+        long? used = null;
+        DateTimeOffset? resetAt = null;
+        TimeSpan? resetAfter = null;
+        TimeSpan? retryAfter = null;
+        var hasData = false;
+
+        if (TryGet(headers, "X-RateLimit-Resource", out var resourceValue) && !string.IsNullOrWhiteSpace(resourceValue))
+        {
+            resource = resourceValue;
+            hasData = true;
+        }
+
+        if (TryParseLong(headers, "X-RateLimit-Limit", out var limitValue))
+        {
+            limit = limitValue;
+            hasData = true;
+        }
+
+        if (TryParseLong(headers, "X-RateLimit-Remaining", out var remainingValue))
+        {
+            remaining = remainingValue;
+            hasData = true;
+        }
+
+        if (TryParseLong(headers, "X-RateLimit-Used", out var usedValue))
+        {
+            used = usedValue;
+            hasData = true;
+        }
+
+        if (TryParseLong(headers, "X-RateLimit-Reset", out var resetValue))
+        {
+            resetAt = DateTimeOffset.FromUnixTimeSeconds(resetValue);
+            var delta = resetAt.Value - now;
+            if (delta > TimeSpan.Zero)
+            {
+                resetAfter = delta;
+            }
+            hasData = true;
+        }
+
+        if (TryGet(headers, "Retry-After", out var retryAfterValue) && !string.IsNullOrWhiteSpace(retryAfterValue))
+        {
+            if (double.TryParse(retryAfterValue, NumberStyles.Float, CultureInfo.InvariantCulture, out var seconds) && seconds > 0)
+            {
+                retryAfter = TimeSpan.FromSeconds(seconds);
+            }
+            else if (DateTimeOffset.TryParse(retryAfterValue, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var retryAfterDate))
+            {
+                var delta = retryAfterDate - now;
+                if (delta > TimeSpan.Zero)
+                {
+                    retryAfter = delta;
+                }
+            }
+            hasData = true;
+        }
+
+        if (!hasData)
+        {
+            return null;
+        }
+
+        return new GhsaRateLimitSnapshot(phase, resource, limit, remaining, used, resetAt, resetAfter, retryAfter);
+    }
+
+    private static bool TryGet(IReadOnlyDictionary<string, string> headers, string key, out string value)
+    {
+        foreach (var pair in headers)
+        {
+            if (pair.Key.Equals(key, StringComparison.OrdinalIgnoreCase))
+            {
+                value = pair.Value;
+                return true;
+            }
+        }
+
+        value = string.Empty;
+        return false;
+    }
+
+    private static bool TryParseLong(IReadOnlyDictionary<string, string> headers, string key, out long result)
+    {
+        result = 0;
+        if (TryGet(headers, key, out var value) && long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
+        {
+            result = parsed;
+            return true;
+        }
+
+        return false;
+    }
+}

src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRateLimitSnapshot.cs

@@ -0,0 +1,23 @@
+using System;
+
+namespace StellaOps.Feedser.Source.Ghsa.Internal;
+
+internal readonly record struct GhsaRateLimitSnapshot(
+    string Phase,
+    string? Resource,
+    long? Limit,
+    long? Remaining,
+    long? Used,
+    DateTimeOffset? ResetAt,
+    TimeSpan? ResetAfter,
+    TimeSpan? RetryAfter)
+{
+    public bool HasData =>
+        Limit.HasValue ||
+        Remaining.HasValue ||
+        Used.HasValue ||
+        ResetAt.HasValue ||
+        ResetAfter.HasValue ||
+        RetryAfter.HasValue ||
+        !string.IsNullOrEmpty(Resource);
+}