feat: Enhance MongoDB storage with event publishing and outbox support
- Added `MongoAdvisoryObservationEventPublisher` and `NatsAdvisoryObservationEventPublisher` for event publishing. - Registered `IAdvisoryObservationEventPublisher` to choose between NATS and MongoDB based on configuration. - Introduced `MongoAdvisoryObservationEventOutbox` for outbox pattern implementation. - Updated service collection to include new event publishers and outbox. - Added a new hosted service `AdvisoryObservationTransportWorker` for processing events. feat: Update project dependencies - Added `NATS.Client.Core` package to the project for NATS integration. test: Add unit tests for AdvisoryLinkset normalization - Created `AdvisoryLinksetNormalizationConfidenceTests` to validate confidence score calculations. fix: Adjust confidence assertion in `AdvisoryObservationAggregationTests` - Updated confidence assertion to allow a range instead of a fixed value. test: Implement tests for AdvisoryObservationEventFactory - Added `AdvisoryObservationEventFactoryTests` to ensure correct mapping and hashing of observation events. chore: Configure test project for Findings Ledger - Created `Directory.Build.props` for test project configuration. - Added `StellaOps.Findings.Ledger.Exports.Unit.csproj` for unit tests related to findings ledger exports. feat: Implement export contracts for findings ledger - Defined export request and response contracts in `ExportContracts.cs`. - Created various export item records for findings, VEX, advisories, and SBOMs. feat: Add export functionality to Findings Ledger Web Service - Implemented endpoints for exporting findings, VEX, advisories, and SBOMs. - Integrated `ExportQueryService` for handling export logic and pagination. test: Add tests for Node language analyzer phase 22 - Implemented `NodePhase22SampleLoaderTests` to validate loading of NDJSON fixtures. - Created sample NDJSON file for testing. chore: Set up isolated test environment for Node tests - Added `node-isolated.runsettings` for isolated test execution. - Created `node-tests-isolated.sh` script for running tests in isolation.
This commit is contained in:
master
@@ -0,0 +1,122 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Core;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal.Phase22;
|
||||
|
||||
internal static class NodePhase22SampleLoader
|
||||
{
|
||||
private const string EnvKey = "SCANNER_NODE_PHASE22_FIXTURE";
|
||||
private const string DefaultFileName = "node-phase22-sample.ndjson";
|
||||
|
||||
public static async ValueTask<IReadOnlyCollection<LanguageComponentRecord>> TryLoadAsync(
|
||||
string rootPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var fixturePath = Environment.GetEnvironmentVariable(EnvKey);
|
||||
if (string.IsNullOrWhiteSpace(fixturePath))
|
||||
{
|
||||
fixturePath = Path.Combine(rootPath, DefaultFileName);
|
||||
if (!File.Exists(fixturePath))
|
||||
{
|
||||
// fallback to docs sample if tests point to repo root
|
||||
var repoRoot = FindRepoRoot(rootPath);
|
||||
var fromDocs = Path.Combine(repoRoot, "docs", "samples", "scanner", "node-phase22", DefaultFileName);
|
||||
fixturePath = File.Exists(fromDocs) ? fromDocs : fixturePath;
|
||||
}
|
||||
}
|
||||
|
||||
if (!File.Exists(fixturePath))
|
||||
{
|
||||
return Array.Empty<LanguageComponentRecord>();
|
||||
}
|
||||
|
||||
var records = new List<LanguageComponentRecord>();
|
||||
await using var stream = File.OpenRead(fixturePath);
|
||||
using var reader = new StreamReader(stream);
|
||||
|
||||
string? line;
|
||||
while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) is not null)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
using var jsonDoc = JsonDocument.Parse(line);
|
||||
var root = jsonDoc.RootElement;
|
||||
if (!root.TryGetProperty("type", out var typeProp))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!string.Equals(typeProp.GetString(), "component", StringComparison.Ordinal))
|
||||
{
|
||||
continue; // only components are mapped into LanguageComponentRecords for now
|
||||
}
|
||||
|
||||
var componentType = root.GetProperty("componentType").GetString() ?? "pkg";
|
||||
var path = root.GetProperty("path").GetString() ?? string.Empty;
|
||||
var reason = root.TryGetProperty("reason", out var reasonProp) ? reasonProp.GetString() : null;
|
||||
var format = root.TryGetProperty("format", out var formatProp) ? formatProp.GetString() : null;
|
||||
var confidence = root.TryGetProperty("confidence", out var confProp) && confProp.TryGetDouble(out var conf)
|
||||
? conf.ToString("0.00", CultureInfo.InvariantCulture)
|
||||
: null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var metadata = new List<KeyValuePair<string, string?>>();
|
||||
if (!string.IsNullOrWhiteSpace(reason)) metadata.Add(new("reason", reason));
|
||||
if (!string.IsNullOrWhiteSpace(format)) metadata.Add(new("format", format));
|
||||
if (!string.IsNullOrWhiteSpace(confidence)) metadata.Add(new("confidence", confidence));
|
||||
|
||||
var typeTag = componentType switch
|
||||
{
|
||||
"native" => "node:native",
|
||||
"wasm" => "node:wasm",
|
||||
_ => "node:bundle"
|
||||
};
|
||||
|
||||
var name = Path.GetFileName(path);
|
||||
var record = LanguageComponentRecord.FromExplicitKey(
|
||||
analyzerId: "node-phase22",
|
||||
componentKey: path,
|
||||
purl: null,
|
||||
name: name,
|
||||
version: null,
|
||||
type: typeTag,
|
||||
metadata: metadata,
|
||||
evidence: null,
|
||||
usedByEntrypoint: false);
|
||||
|
||||
records.Add(record);
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
private static string FindRepoRoot(string start)
|
||||
{
|
||||
var current = new DirectoryInfo(start);
|
||||
while (current is not null && current.Exists)
|
||||
{
|
||||
if (File.Exists(Path.Combine(current.FullName, "README.md")))
|
||||
{
|
||||
return current.FullName;
|
||||
}
|
||||
|
||||
current = current.Parent;
|
||||
}
|
||||
|
||||
return start;
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang.Node.Internal;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Node;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Node.Internal;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Node.Internal.Phase22;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Node;
|
||||
|
||||
public sealed class NodeLanguageAnalyzer : ILanguageAnalyzer
|
||||
{
|
||||
@@ -13,11 +14,11 @@ public sealed class NodeLanguageAnalyzer : ILanguageAnalyzer
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(writer);
|
||||
|
||||
var lockData = await NodeLockData.LoadAsync(context.RootPath, cancellationToken).ConfigureAwait(false);
|
||||
var packages = NodePackageCollector.CollectPackages(context, lockData, cancellationToken);
|
||||
|
||||
foreach (var package in packages.OrderBy(static p => p.ComponentKey, StringComparer.Ordinal))
|
||||
{
|
||||
var lockData = await NodeLockData.LoadAsync(context.RootPath, cancellationToken).ConfigureAwait(false);
|
||||
var packages = NodePackageCollector.CollectPackages(context, lockData, cancellationToken);
|
||||
|
||||
foreach (var package in packages.OrderBy(static p => p.ComponentKey, StringComparer.Ordinal))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var metadata = package.CreateMetadata();
|
||||
@@ -29,9 +30,16 @@ public sealed class NodeLanguageAnalyzer : ILanguageAnalyzer
|
||||
name: package.Name,
|
||||
version: package.Version,
|
||||
type: "npm",
|
||||
metadata: metadata,
|
||||
evidence: evidence,
|
||||
usedByEntrypoint: package.IsUsedByEntrypoint);
|
||||
}
|
||||
}
|
||||
}
|
||||
metadata: metadata,
|
||||
evidence: evidence,
|
||||
usedByEntrypoint: package.IsUsedByEntrypoint);
|
||||
}
|
||||
|
||||
// Optional Phase 22 prep path: ingest precomputed bundle/native/WASM AOC records from NDJSON fixture
|
||||
var phase22Records = await NodePhase22SampleLoader.TryLoadAsync(context.RootPath, cancellationToken).ConfigureAwait(false);
|
||||
if (phase22Records.Count > 0)
|
||||
{
|
||||
writer.AddRange(phase22Records);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user