feat: Add Bun language analyzer and related functionality
- Implemented BunPackageNormalizer to deduplicate packages by name and version. - Created BunProjectDiscoverer to identify Bun project roots in the filesystem. - Added project files for the Bun analyzer including manifest and project configuration. - Developed comprehensive tests for Bun language analyzer covering various scenarios. - Included fixture files for testing standard installs, isolated linker installs, lockfile-only scenarios, and workspaces. - Established stubs for authentication sessions to facilitate testing in the web application.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,30 @@
|
||||
namespace StellaOps.Concelier.Core.Linksets;
|
||||
|
||||
/// <summary>
|
||||
/// Abstraction for linkset cache telemetry.
|
||||
/// Per CONCELIER-AIAI-31-002.
|
||||
/// </summary>
|
||||
public interface ILinksetCacheTelemetry
|
||||
{
|
||||
/// <summary>
|
||||
/// Records a cache hit.
|
||||
/// </summary>
|
||||
/// <param name="tenant">Tenant identifier.</param>
|
||||
/// <param name="source">Source vendor (e.g., "ghsa", "nvd").</param>
|
||||
void RecordHit(string? tenant, string source);
|
||||
|
||||
/// <summary>
|
||||
/// Records a cache write.
|
||||
/// </summary>
|
||||
/// <param name="tenant">Tenant identifier.</param>
|
||||
/// <param name="source">Source vendor.</param>
|
||||
void RecordWrite(string? tenant, string source);
|
||||
|
||||
/// <summary>
|
||||
/// Records a synchronous rebuild latency.
|
||||
/// </summary>
|
||||
/// <param name="tenant">Tenant identifier.</param>
|
||||
/// <param name="source">Source vendor.</param>
|
||||
/// <param name="elapsedMs">Elapsed time in milliseconds.</param>
|
||||
void RecordRebuild(string? tenant, string source, double elapsedMs);
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using StellaOps.Concelier.Core.Observations;
|
||||
using StellaOps.Concelier.Models.Observations;
|
||||
|
||||
namespace StellaOps.Concelier.Core.Linksets;
|
||||
|
||||
/// <summary>
|
||||
/// Provides read-through caching for LNM linksets.
|
||||
/// Per CONCELIER-AIAI-31-002.
|
||||
///
|
||||
/// Read-through behavior:
|
||||
/// 1. First queries the configured cache (Postgres via IAdvisoryLinksetLookup)
|
||||
/// 2. On cache miss, rebuilds from MongoDB observations
|
||||
/// 3. Stores rebuilt linksets in cache
|
||||
/// 4. Returns results
|
||||
/// </summary>
|
||||
public sealed class ReadThroughLinksetCacheService : IAdvisoryLinksetLookup
|
||||
{
|
||||
private readonly IAdvisoryLinksetLookup? _cacheLookup;
|
||||
private readonly IAdvisoryLinksetSink? _cacheSink;
|
||||
private readonly IAdvisoryObservationLookup _observations;
|
||||
private readonly ILinksetCacheTelemetry _telemetry;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public ReadThroughLinksetCacheService(
|
||||
IAdvisoryObservationLookup observations,
|
||||
ILinksetCacheTelemetry telemetry,
|
||||
TimeProvider timeProvider,
|
||||
IAdvisoryLinksetLookup? cacheLookup = null,
|
||||
IAdvisoryLinksetSink? cacheSink = null)
|
||||
{
|
||||
_observations = observations ?? throw new ArgumentNullException(nameof(observations));
|
||||
_telemetry = telemetry ?? throw new ArgumentNullException(nameof(telemetry));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_cacheLookup = cacheLookup;
|
||||
_cacheSink = cacheSink;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<AdvisoryLinkset>> FindByTenantAsync(
|
||||
string tenantId,
|
||||
IEnumerable<string>? advisoryIds,
|
||||
IEnumerable<string>? sources,
|
||||
AdvisoryLinksetCursor? cursor,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
if (limit <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(limit), "Limit must be positive.");
|
||||
}
|
||||
|
||||
var normalizedTenant = tenantId.Trim().ToLowerInvariant();
|
||||
var advisoryIdSet = advisoryIds?.Select(a => a.Trim()).Where(a => !string.IsNullOrWhiteSpace(a)).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var sourceSet = sources?.Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s)).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Step 1: Try cache first if available
|
||||
if (_cacheLookup is not null)
|
||||
{
|
||||
var cached = await _cacheLookup
|
||||
.FindByTenantAsync(normalizedTenant, advisoryIdSet, sourceSet, cursor, limit, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (cached.Count > 0)
|
||||
{
|
||||
// Cache hit
|
||||
foreach (var linkset in cached)
|
||||
{
|
||||
_telemetry.RecordHit(normalizedTenant, linkset.Source);
|
||||
}
|
||||
return cached;
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: Cache miss - rebuild from observations
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var linksets = await RebuildFromObservationsAsync(
|
||||
normalizedTenant,
|
||||
advisoryIdSet,
|
||||
sourceSet,
|
||||
cursor,
|
||||
limit,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
stopwatch.Stop();
|
||||
|
||||
if (linksets.Count == 0)
|
||||
{
|
||||
return linksets;
|
||||
}
|
||||
|
||||
// Step 3: Store in cache if sink is available
|
||||
if (_cacheSink is not null)
|
||||
{
|
||||
foreach (var linkset in linksets)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _cacheSink.UpsertAsync(linkset, cancellationToken).ConfigureAwait(false);
|
||||
_telemetry.RecordWrite(normalizedTenant, linkset.Source);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Cache write failure should not fail the request
|
||||
// Log would be handled by the sink implementation
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Record rebuild metrics
|
||||
foreach (var linkset in linksets)
|
||||
{
|
||||
_telemetry.RecordRebuild(normalizedTenant, linkset.Source, stopwatch.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
|
||||
return linksets;
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<AdvisoryLinkset>> RebuildFromObservationsAsync(
|
||||
string tenant,
|
||||
IReadOnlySet<string>? advisoryIds,
|
||||
IReadOnlySet<string>? sources,
|
||||
AdvisoryLinksetCursor? cursor,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Query observations for the tenant
|
||||
// Note: For specific advisoryIds, we'd ideally have a more targeted query
|
||||
// but the current interface returns all tenant observations
|
||||
var observations = await _observations
|
||||
.ListByTenantAsync(tenant, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (observations.Count == 0)
|
||||
{
|
||||
return Array.Empty<AdvisoryLinkset>();
|
||||
}
|
||||
|
||||
// Filter by advisoryId and source if specified
|
||||
var filtered = observations.AsEnumerable();
|
||||
|
||||
if (advisoryIds is { Count: > 0 })
|
||||
{
|
||||
filtered = filtered.Where(o => advisoryIds.Contains(o.Upstream.UpstreamId));
|
||||
}
|
||||
|
||||
if (sources is { Count: > 0 })
|
||||
{
|
||||
filtered = filtered.Where(o => sources.Contains(o.Source.Vendor));
|
||||
}
|
||||
|
||||
// Group by (source, advisoryId) to build linksets
|
||||
var groups = filtered
|
||||
.GroupBy(
|
||||
o => (o.Source.Vendor, o.Upstream.UpstreamId),
|
||||
new VendorUpstreamComparer())
|
||||
.ToList();
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var linksets = new List<AdvisoryLinkset>(groups.Count);
|
||||
|
||||
foreach (var group in groups)
|
||||
{
|
||||
var observationIds = group
|
||||
.Select(o => o.ObservationId)
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
var createdAt = group.Max(o => o.CreatedAt);
|
||||
var normalized = BuildNormalized(group);
|
||||
var provenance = BuildProvenance(group, now);
|
||||
|
||||
var linkset = new AdvisoryLinkset(
|
||||
tenant,
|
||||
group.Key.Vendor,
|
||||
group.Key.UpstreamId,
|
||||
observationIds,
|
||||
normalized,
|
||||
provenance,
|
||||
ComputeConfidence(group),
|
||||
DetectConflicts(group),
|
||||
createdAt,
|
||||
null);
|
||||
|
||||
linksets.Add(linkset);
|
||||
}
|
||||
|
||||
// Apply cursor-based pagination
|
||||
var ordered = linksets
|
||||
.OrderByDescending(ls => ls.CreatedAt)
|
||||
.ThenBy(ls => ls.AdvisoryId, StringComparer.Ordinal)
|
||||
.AsEnumerable();
|
||||
|
||||
if (cursor is not null)
|
||||
{
|
||||
ordered = ordered.Where(ls =>
|
||||
ls.CreatedAt < cursor.CreatedAt ||
|
||||
(ls.CreatedAt == cursor.CreatedAt &&
|
||||
string.Compare(ls.AdvisoryId, cursor.AdvisoryId, StringComparison.Ordinal) > 0));
|
||||
}
|
||||
|
||||
return ordered.Take(limit).ToList();
|
||||
}
|
||||
|
||||
private static AdvisoryLinksetNormalized? BuildNormalized(IEnumerable<AdvisoryObservation> observations)
|
||||
{
|
||||
var purls = observations
|
||||
.SelectMany(o => o.Linkset.Purls.IsDefaultOrEmpty ? Enumerable.Empty<string>() : o.Linkset.Purls)
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(p => p, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
var cpes = observations
|
||||
.SelectMany(o => o.Linkset.Cpes.IsDefaultOrEmpty ? Enumerable.Empty<string>() : o.Linkset.Cpes)
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(c => c, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
if (purls.Length == 0 && cpes.Length == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new AdvisoryLinksetNormalized(
|
||||
purls.Length > 0 ? purls : null,
|
||||
cpes.Length > 0 ? cpes : null,
|
||||
null,
|
||||
null,
|
||||
null);
|
||||
}
|
||||
|
||||
private static AdvisoryLinksetProvenance BuildProvenance(
|
||||
IEnumerable<AdvisoryObservation> observations,
|
||||
DateTimeOffset now)
|
||||
{
|
||||
var hashes = observations
|
||||
.Select(o => o.ObservationId)
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(h => h, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new AdvisoryLinksetProvenance(
|
||||
hashes,
|
||||
"read-through-cache",
|
||||
null);
|
||||
}
|
||||
|
||||
private static double ComputeConfidence(IEnumerable<AdvisoryObservation> observations)
|
||||
{
|
||||
// Simple confidence: based on number of corroborating observations
|
||||
var count = observations.Count();
|
||||
return count switch
|
||||
{
|
||||
1 => 0.5,
|
||||
2 => 0.7,
|
||||
3 => 0.85,
|
||||
_ => Math.Min(1.0, 0.85 + (count - 3) * 0.03)
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryLinksetConflict> DetectConflicts(
|
||||
IEnumerable<AdvisoryObservation> observations)
|
||||
{
|
||||
var conflicts = new List<AdvisoryLinksetConflict>();
|
||||
var obsList = observations.ToList();
|
||||
|
||||
if (obsList.Count <= 1)
|
||||
{
|
||||
return conflicts;
|
||||
}
|
||||
|
||||
// Detect PURL conflicts (same package, different versions mentioned)
|
||||
var purlsByPackage = obsList
|
||||
.SelectMany(o => o.Linkset.Purls.IsDefaultOrEmpty ? Enumerable.Empty<string>() : o.Linkset.Purls)
|
||||
.Where(p => p.Contains('@'))
|
||||
.GroupBy(p => p.Split('@')[0], StringComparer.Ordinal)
|
||||
.Where(g => g.Distinct(StringComparer.Ordinal).Count() > 1);
|
||||
|
||||
foreach (var group in purlsByPackage)
|
||||
{
|
||||
var values = group.Distinct(StringComparer.Ordinal).ToImmutableArray();
|
||||
conflicts.Add(new AdvisoryLinksetConflict(
|
||||
"purl_version",
|
||||
"Multiple versions specified for same package",
|
||||
values,
|
||||
null));
|
||||
}
|
||||
|
||||
return conflicts;
|
||||
}
|
||||
|
||||
private sealed class VendorUpstreamComparer : IEqualityComparer<(string Vendor, string UpstreamId)>
|
||||
{
|
||||
public bool Equals((string Vendor, string UpstreamId) x, (string Vendor, string UpstreamId) y)
|
||||
=> StringComparer.OrdinalIgnoreCase.Equals(x.Vendor, y.Vendor)
|
||||
&& StringComparer.Ordinal.Equals(x.UpstreamId, y.UpstreamId);
|
||||
|
||||
public int GetHashCode((string Vendor, string UpstreamId) obj)
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.Add(obj.Vendor, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.UpstreamId, StringComparer.Ordinal);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user