stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details
AOC Guard CI / aoc-guard (push) Has been cancelled
Details
AOC Guard CI / aoc-verify (push) Has been cancelled
Details
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Details
Export Center CI / export-ci (push) Has been cancelled
Details
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Details
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Details
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Details
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Details
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Details
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Details
Signals CI & Image / signals-ci (push) Has been cancelled
Details
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Details
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Details

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions
Download Patch File Download Diff File Expand all files Collapse all files

162
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/Configuration/OsvOptions.cs
View File

@@ -1,81 +1,81 @@
using System.Diagnostics.CodeAnalysis;
namespace StellaOps.Concelier.Connector.Osv.Configuration;
public sealed class OsvOptions
{
public const string HttpClientName = "source.osv";
public Uri BaseUri { get; set; } = new("https://osv-vulnerabilities.storage.googleapis.com/", UriKind.Absolute);
public IReadOnlyList<string> Ecosystems { get; set; } = new[] { "PyPI", "npm", "Maven", "Go", "crates" };
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(14);
public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromMinutes(10);
public int MaxAdvisoriesPerFetch { get; set; } = 250;
public string ArchiveFileName { get; set; } = "all.zip";
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
public TimeSpan HttpTimeout { get; set; } = TimeSpan.FromMinutes(3);
[MemberNotNull(nameof(BaseUri), nameof(Ecosystems), nameof(ArchiveFileName))]
public void Validate()
{
if (BaseUri is null || !BaseUri.IsAbsoluteUri)
{
throw new InvalidOperationException("OSV base URI must be an absolute URI.");
}
if (string.IsNullOrWhiteSpace(ArchiveFileName))
{
throw new InvalidOperationException("OSV archive file name must be provided.");
}
if (!ArchiveFileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("OSV archive file name must be a .zip resource.");
}
if (Ecosystems is null || Ecosystems.Count == 0)
{
throw new InvalidOperationException("At least one OSV ecosystem must be configured.");
}
foreach (var ecosystem in Ecosystems)
{
if (string.IsNullOrWhiteSpace(ecosystem))
{
throw new InvalidOperationException("Ecosystem names cannot be null or whitespace.");
}
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("Initial backfill window must be positive.");
}
if (ModifiedTolerance < TimeSpan.Zero)
{
throw new InvalidOperationException("Modified tolerance cannot be negative.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException("Max advisories per fetch must be greater than zero.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("Request delay cannot be negative.");
}
if (HttpTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("HTTP timeout must be positive.");
}
}
}
using System.Diagnostics.CodeAnalysis;
namespace StellaOps.Concelier.Connector.Osv.Configuration;
public sealed class OsvOptions
{
public const string HttpClientName = "source.osv";
public Uri BaseUri { get; set; } = new("https://osv-vulnerabilities.storage.googleapis.com/", UriKind.Absolute);
public IReadOnlyList<string> Ecosystems { get; set; } = new[] { "PyPI", "npm", "Maven", "Go", "crates" };
public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(14);
public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromMinutes(10);
public int MaxAdvisoriesPerFetch { get; set; } = 250;
public string ArchiveFileName { get; set; } = "all.zip";
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
public TimeSpan HttpTimeout { get; set; } = TimeSpan.FromMinutes(3);
[MemberNotNull(nameof(BaseUri), nameof(Ecosystems), nameof(ArchiveFileName))]
public void Validate()
{
if (BaseUri is null || !BaseUri.IsAbsoluteUri)
{
throw new InvalidOperationException("OSV base URI must be an absolute URI.");
}
if (string.IsNullOrWhiteSpace(ArchiveFileName))
{
throw new InvalidOperationException("OSV archive file name must be provided.");
}
if (!ArchiveFileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("OSV archive file name must be a .zip resource.");
}
if (Ecosystems is null || Ecosystems.Count == 0)
{
throw new InvalidOperationException("At least one OSV ecosystem must be configured.");
}
foreach (var ecosystem in Ecosystems)
{
if (string.IsNullOrWhiteSpace(ecosystem))
{
throw new InvalidOperationException("Ecosystem names cannot be null or whitespace.");
}
}
if (InitialBackfill <= TimeSpan.Zero)
{
throw new InvalidOperationException("Initial backfill window must be positive.");
}
if (ModifiedTolerance < TimeSpan.Zero)
{
throw new InvalidOperationException("Modified tolerance cannot be negative.");
}
if (MaxAdvisoriesPerFetch <= 0)
{
throw new InvalidOperationException("Max advisories per fetch must be greater than zero.");
}
if (RequestDelay < TimeSpan.Zero)
{
throw new InvalidOperationException("Request delay cannot be negative.");
}
if (HttpTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("HTTP timeout must be positive.");
}
}
}

580
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/Internal/OsvCursor.cs
View File

@@ -1,290 +1,290 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Bson;
namespace StellaOps.Concelier.Connector.Osv.Internal;
internal sealed record OsvCursor(
IReadOnlyDictionary<string, DateTimeOffset?> LastModifiedByEcosystem,
IReadOnlyDictionary<string, IReadOnlyCollection<string>> ProcessedIdsByEcosystem,
IReadOnlyDictionary<string, OsvArchiveMetadata> ArchiveMetadataByEcosystem,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
private static readonly IReadOnlyDictionary<string, DateTimeOffset?> EmptyLastModified =
new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
private static readonly IReadOnlyDictionary<string, IReadOnlyCollection<string>> EmptyProcessedIds =
new Dictionary<string, IReadOnlyCollection<string>>(StringComparer.OrdinalIgnoreCase);
private static readonly IReadOnlyDictionary<string, OsvArchiveMetadata> EmptyArchiveMetadata =
new Dictionary<string, OsvArchiveMetadata>(StringComparer.OrdinalIgnoreCase);
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
private static readonly IReadOnlyCollection<string> EmptyStringList = Array.Empty<string>();
public static OsvCursor Empty { get; } = new(EmptyLastModified, EmptyProcessedIds, EmptyArchiveMetadata, EmptyGuidList, EmptyGuidList);
public BsonDocument ToBsonDocument()
{
var document = new BsonDocument
{
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
};
if (LastModifiedByEcosystem.Count > 0)
{
var lastModifiedDoc = new BsonDocument();
foreach (var (ecosystem, timestamp) in LastModifiedByEcosystem)
{
lastModifiedDoc[ecosystem] = timestamp.HasValue ? BsonValue.Create(timestamp.Value.UtcDateTime) : BsonNull.Value;
}
document["lastModified"] = lastModifiedDoc;
}
if (ProcessedIdsByEcosystem.Count > 0)
{
var processedDoc = new BsonDocument();
foreach (var (ecosystem, ids) in ProcessedIdsByEcosystem)
{
processedDoc[ecosystem] = new BsonArray(ids.Select(id => id));
}
document["processed"] = processedDoc;
}
if (ArchiveMetadataByEcosystem.Count > 0)
{
var metadataDoc = new BsonDocument();
foreach (var (ecosystem, metadata) in ArchiveMetadataByEcosystem)
{
var element = new BsonDocument();
if (!string.IsNullOrWhiteSpace(metadata.ETag))
{
element["etag"] = metadata.ETag;
}
if (metadata.LastModified.HasValue)
{
element["lastModified"] = metadata.LastModified.Value.UtcDateTime;
}
metadataDoc[ecosystem] = element;
}
document["archive"] = metadataDoc;
}
return document;
}
public static OsvCursor FromBson(BsonDocument? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var lastModified = ReadLastModified(document.TryGetValue("lastModified", out var lastModifiedValue) ? lastModifiedValue : null);
var processed = ReadProcessedIds(document.TryGetValue("processed", out var processedValue) ? processedValue : null);
var archiveMetadata = ReadArchiveMetadata(document.TryGetValue("archive", out var archiveValue) ? archiveValue : null);
var pendingDocuments = ReadGuidList(document, "pendingDocuments");
var pendingMappings = ReadGuidList(document, "pendingMappings");
return new OsvCursor(lastModified, processed, archiveMetadata, pendingDocuments, pendingMappings);
}
public DateTimeOffset? GetLastModified(string ecosystem)
{
ArgumentException.ThrowIfNullOrEmpty(ecosystem);
return LastModifiedByEcosystem.TryGetValue(ecosystem, out var value) ? value : null;
}
public bool HasProcessedId(string ecosystem, string id)
{
ArgumentException.ThrowIfNullOrEmpty(ecosystem);
ArgumentException.ThrowIfNullOrEmpty(id);
return ProcessedIdsByEcosystem.TryGetValue(ecosystem, out var ids)
&& ids.Contains(id, StringComparer.OrdinalIgnoreCase);
}
public OsvCursor WithLastModified(string ecosystem, DateTimeOffset timestamp, IEnumerable<string> processedIds)
{
ArgumentException.ThrowIfNullOrEmpty(ecosystem);
var lastModified = new Dictionary<string, DateTimeOffset?>(LastModifiedByEcosystem, StringComparer.OrdinalIgnoreCase)
{
[ecosystem] = timestamp.ToUniversalTime(),
};
var processed = new Dictionary<string, IReadOnlyCollection<string>>(ProcessedIdsByEcosystem, StringComparer.OrdinalIgnoreCase)
{
[ecosystem] = processedIds?.Where(static id => !string.IsNullOrWhiteSpace(id))
.Select(static id => id.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray() ?? EmptyStringList,
};
return this with { LastModifiedByEcosystem = lastModified, ProcessedIdsByEcosystem = processed };
}
public OsvCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
public OsvCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
public OsvCursor AddProcessedId(string ecosystem, string id)
{
ArgumentException.ThrowIfNullOrEmpty(ecosystem);
ArgumentException.ThrowIfNullOrEmpty(id);
var processed = new Dictionary<string, IReadOnlyCollection<string>>(ProcessedIdsByEcosystem, StringComparer.OrdinalIgnoreCase);
if (!processed.TryGetValue(ecosystem, out var ids))
{
ids = EmptyStringList;
}
var set = new HashSet<string>(ids, StringComparer.OrdinalIgnoreCase)
{
id.Trim(),
};
processed[ecosystem] = set.ToArray();
return this with { ProcessedIdsByEcosystem = processed };
}
public bool TryGetArchiveMetadata(string ecosystem, out OsvArchiveMetadata metadata)
{
ArgumentException.ThrowIfNullOrEmpty(ecosystem);
return ArchiveMetadataByEcosystem.TryGetValue(ecosystem, out metadata!);
}
public OsvCursor WithArchiveMetadata(string ecosystem, string? etag, DateTimeOffset? lastModified)
{
ArgumentException.ThrowIfNullOrEmpty(ecosystem);
var metadata = new Dictionary<string, OsvArchiveMetadata>(ArchiveMetadataByEcosystem, StringComparer.OrdinalIgnoreCase)
{
[ecosystem] = new OsvArchiveMetadata(etag?.Trim(), lastModified?.ToUniversalTime()),
};
return this with { ArchiveMetadataByEcosystem = metadata };
}
private static IReadOnlyDictionary<string, DateTimeOffset?> ReadLastModified(BsonValue? value)
{
if (value is not BsonDocument document)
{
return EmptyLastModified;
}
var dictionary = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
foreach (var element in document.Elements)
{
if (element.Value is null || element.Value.IsBsonNull)
{
dictionary[element.Name] = null;
continue;
}
dictionary[element.Name] = ParseDate(element.Value);
}
return dictionary;
}
private static IReadOnlyDictionary<string, IReadOnlyCollection<string>> ReadProcessedIds(BsonValue? value)
{
if (value is not BsonDocument document)
{
return EmptyProcessedIds;
}
var dictionary = new Dictionary<string, IReadOnlyCollection<string>>(StringComparer.OrdinalIgnoreCase);
foreach (var element in document.Elements)
{
if (element.Value is not BsonArray array)
{
continue;
}
var ids = new List<string>(array.Count);
foreach (var idValue in array)
{
if (idValue?.BsonType == BsonType.String)
{
var str = idValue.AsString.Trim();
if (!string.IsNullOrWhiteSpace(str))
{
ids.Add(str);
}
}
}
dictionary[element.Name] = ids.Count == 0
? EmptyStringList
: ids.Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
}
return dictionary;
}
private static IReadOnlyDictionary<string, OsvArchiveMetadata> ReadArchiveMetadata(BsonValue? value)
{
if (value is not BsonDocument document)
{
return EmptyArchiveMetadata;
}
var dictionary = new Dictionary<string, OsvArchiveMetadata>(StringComparer.OrdinalIgnoreCase);
foreach (var element in document.Elements)
{
if (element.Value is not BsonDocument metadataDocument)
{
continue;
}
string? etag = metadataDocument.TryGetValue("etag", out var etagValue) && etagValue.IsString ? etagValue.AsString : null;
DateTimeOffset? lastModified = metadataDocument.TryGetValue("lastModified", out var lastModifiedValue)
? ParseDate(lastModifiedValue)
: null;
dictionary[element.Name] = new OsvArchiveMetadata(etag, lastModified);
}
return dictionary.Count == 0 ? EmptyArchiveMetadata : dictionary;
}
private static IReadOnlyCollection<Guid> ReadGuidList(BsonDocument document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
{
return EmptyGuidList;
}
var list = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element.ToString(), out var guid))
{
list.Add(guid);
}
}
return list;
}
private static DateTimeOffset? ParseDate(BsonValue value)
{
return value.BsonType switch
{
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
}
internal sealed record OsvArchiveMetadata(string? ETag, DateTimeOffset? LastModified);
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Concelier.Documents;
namespace StellaOps.Concelier.Connector.Osv.Internal;
internal sealed record OsvCursor(
IReadOnlyDictionary<string, DateTimeOffset?> LastModifiedByEcosystem,
IReadOnlyDictionary<string, IReadOnlyCollection<string>> ProcessedIdsByEcosystem,
IReadOnlyDictionary<string, OsvArchiveMetadata> ArchiveMetadataByEcosystem,
IReadOnlyCollection<Guid> PendingDocuments,
IReadOnlyCollection<Guid> PendingMappings)
{
private static readonly IReadOnlyDictionary<string, DateTimeOffset?> EmptyLastModified =
new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
private static readonly IReadOnlyDictionary<string, IReadOnlyCollection<string>> EmptyProcessedIds =
new Dictionary<string, IReadOnlyCollection<string>>(StringComparer.OrdinalIgnoreCase);
private static readonly IReadOnlyDictionary<string, OsvArchiveMetadata> EmptyArchiveMetadata =
new Dictionary<string, OsvArchiveMetadata>(StringComparer.OrdinalIgnoreCase);
private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>();
private static readonly IReadOnlyCollection<string> EmptyStringList = Array.Empty<string>();
public static OsvCursor Empty { get; } = new(EmptyLastModified, EmptyProcessedIds, EmptyArchiveMetadata, EmptyGuidList, EmptyGuidList);
public DocumentObject ToDocumentObject()
{
var document = new DocumentObject
{
["pendingDocuments"] = new DocumentArray(PendingDocuments.Select(id => id.ToString())),
["pendingMappings"] = new DocumentArray(PendingMappings.Select(id => id.ToString())),
};
if (LastModifiedByEcosystem.Count > 0)
{
var lastModifiedDoc = new DocumentObject();
foreach (var (ecosystem, timestamp) in LastModifiedByEcosystem)
{
lastModifiedDoc[ecosystem] = timestamp.HasValue ? DocumentValue.Create(timestamp.Value.UtcDateTime) : DocumentNull.Value;
}
document["lastModified"] = lastModifiedDoc;
}
if (ProcessedIdsByEcosystem.Count > 0)
{
var processedDoc = new DocumentObject();
foreach (var (ecosystem, ids) in ProcessedIdsByEcosystem)
{
processedDoc[ecosystem] = new DocumentArray(ids.Select(id => id));
}
document["processed"] = processedDoc;
}
if (ArchiveMetadataByEcosystem.Count > 0)
{
var metadataDoc = new DocumentObject();
foreach (var (ecosystem, metadata) in ArchiveMetadataByEcosystem)
{
var element = new DocumentObject();
if (!string.IsNullOrWhiteSpace(metadata.ETag))
{
element["etag"] = metadata.ETag;
}
if (metadata.LastModified.HasValue)
{
element["lastModified"] = metadata.LastModified.Value.UtcDateTime;
}
metadataDoc[ecosystem] = element;
}
document["archive"] = metadataDoc;
}
return document;
}
public static OsvCursor FromBson(DocumentObject? document)
{
if (document is null || document.ElementCount == 0)
{
return Empty;
}
var lastModified = ReadLastModified(document.TryGetValue("lastModified", out var lastModifiedValue) ? lastModifiedValue : null);
var processed = ReadProcessedIds(document.TryGetValue("processed", out var processedValue) ? processedValue : null);
var archiveMetadata = ReadArchiveMetadata(document.TryGetValue("archive", out var archiveValue) ? archiveValue : null);
var pendingDocuments = ReadGuidList(document, "pendingDocuments");
var pendingMappings = ReadGuidList(document, "pendingMappings");
return new OsvCursor(lastModified, processed, archiveMetadata, pendingDocuments, pendingMappings);
}
public DateTimeOffset? GetLastModified(string ecosystem)
{
ArgumentException.ThrowIfNullOrEmpty(ecosystem);
return LastModifiedByEcosystem.TryGetValue(ecosystem, out var value) ? value : null;
}
public bool HasProcessedId(string ecosystem, string id)
{
ArgumentException.ThrowIfNullOrEmpty(ecosystem);
ArgumentException.ThrowIfNullOrEmpty(id);
return ProcessedIdsByEcosystem.TryGetValue(ecosystem, out var ids)
&& ids.Contains(id, StringComparer.OrdinalIgnoreCase);
}
public OsvCursor WithLastModified(string ecosystem, DateTimeOffset timestamp, IEnumerable<string> processedIds)
{
ArgumentException.ThrowIfNullOrEmpty(ecosystem);
var lastModified = new Dictionary<string, DateTimeOffset?>(LastModifiedByEcosystem, StringComparer.OrdinalIgnoreCase)
{
[ecosystem] = timestamp.ToUniversalTime(),
};
var processed = new Dictionary<string, IReadOnlyCollection<string>>(ProcessedIdsByEcosystem, StringComparer.OrdinalIgnoreCase)
{
[ecosystem] = processedIds?.Where(static id => !string.IsNullOrWhiteSpace(id))
.Select(static id => id.Trim())
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray() ?? EmptyStringList,
};
return this with { LastModifiedByEcosystem = lastModified, ProcessedIdsByEcosystem = processed };
}
public OsvCursor WithPendingDocuments(IEnumerable<Guid> ids)
=> this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList };
public OsvCursor WithPendingMappings(IEnumerable<Guid> ids)
=> this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList };
public OsvCursor AddProcessedId(string ecosystem, string id)
{
ArgumentException.ThrowIfNullOrEmpty(ecosystem);
ArgumentException.ThrowIfNullOrEmpty(id);
var processed = new Dictionary<string, IReadOnlyCollection<string>>(ProcessedIdsByEcosystem, StringComparer.OrdinalIgnoreCase);
if (!processed.TryGetValue(ecosystem, out var ids))
{
ids = EmptyStringList;
}
var set = new HashSet<string>(ids, StringComparer.OrdinalIgnoreCase)
{
id.Trim(),
};
processed[ecosystem] = set.ToArray();
return this with { ProcessedIdsByEcosystem = processed };
}
public bool TryGetArchiveMetadata(string ecosystem, out OsvArchiveMetadata metadata)
{
ArgumentException.ThrowIfNullOrEmpty(ecosystem);
return ArchiveMetadataByEcosystem.TryGetValue(ecosystem, out metadata!);
}
public OsvCursor WithArchiveMetadata(string ecosystem, string? etag, DateTimeOffset? lastModified)
{
ArgumentException.ThrowIfNullOrEmpty(ecosystem);
var metadata = new Dictionary<string, OsvArchiveMetadata>(ArchiveMetadataByEcosystem, StringComparer.OrdinalIgnoreCase)
{
[ecosystem] = new OsvArchiveMetadata(etag?.Trim(), lastModified?.ToUniversalTime()),
};
return this with { ArchiveMetadataByEcosystem = metadata };
}
private static IReadOnlyDictionary<string, DateTimeOffset?> ReadLastModified(DocumentValue? value)
{
if (value is not DocumentObject document)
{
return EmptyLastModified;
}
var dictionary = new Dictionary<string, DateTimeOffset?>(StringComparer.OrdinalIgnoreCase);
foreach (var element in document.Elements)
{
if (element.Value is null || element.Value.IsDocumentNull)
{
dictionary[element.Name] = null;
continue;
}
dictionary[element.Name] = ParseDate(element.Value);
}
return dictionary;
}
private static IReadOnlyDictionary<string, IReadOnlyCollection<string>> ReadProcessedIds(DocumentValue? value)
{
if (value is not DocumentObject document)
{
return EmptyProcessedIds;
}
var dictionary = new Dictionary<string, IReadOnlyCollection<string>>(StringComparer.OrdinalIgnoreCase);
foreach (var element in document.Elements)
{
if (element.Value is not DocumentArray array)
{
continue;
}
var ids = new List<string>(array.Count);
foreach (var idValue in array)
{
if (idValue?.DocumentType == DocumentType.String)
{
var str = idValue.AsString.Trim();
if (!string.IsNullOrWhiteSpace(str))
{
ids.Add(str);
}
}
}
dictionary[element.Name] = ids.Count == 0
? EmptyStringList
: ids.Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
}
return dictionary;
}
private static IReadOnlyDictionary<string, OsvArchiveMetadata> ReadArchiveMetadata(DocumentValue? value)
{
if (value is not DocumentObject document)
{
return EmptyArchiveMetadata;
}
var dictionary = new Dictionary<string, OsvArchiveMetadata>(StringComparer.OrdinalIgnoreCase);
foreach (var element in document.Elements)
{
if (element.Value is not DocumentObject metadataDocument)
{
continue;
}
string? etag = metadataDocument.TryGetValue("etag", out var etagValue) && etagValue.IsString ? etagValue.AsString : null;
DateTimeOffset? lastModified = metadataDocument.TryGetValue("lastModified", out var lastModifiedValue)
? ParseDate(lastModifiedValue)
: null;
dictionary[element.Name] = new OsvArchiveMetadata(etag, lastModified);
}
return dictionary.Count == 0 ? EmptyArchiveMetadata : dictionary;
}
private static IReadOnlyCollection<Guid> ReadGuidList(DocumentObject document, string field)
{
if (!document.TryGetValue(field, out var value) || value is not DocumentArray array)
{
return EmptyGuidList;
}
var list = new List<Guid>(array.Count);
foreach (var element in array)
{
if (Guid.TryParse(element.ToString(), out var guid))
{
list.Add(guid);
}
}
return list;
}
private static DateTimeOffset? ParseDate(DocumentValue value)
{
return value.DocumentType switch
{
DocumentType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
DocumentType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
_ => null,
};
}
}
internal sealed record OsvArchiveMetadata(string? ETag, DateTimeOffset? LastModified);

1634
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/Internal/OsvMapper.cs
View File

File diff suppressed because it is too large Load Diff

202
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/Internal/OsvVulnerabilityDto.cs
View File

@@ -1,36 +1,36 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Osv.Internal;
internal sealed record OsvVulnerabilityDto
{
[JsonPropertyName("id")]
public string Id { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("details")]
public string? Details { get; init; }
[JsonPropertyName("aliases")]
public IReadOnlyList<string>? Aliases { get; init; }
[JsonPropertyName("related")]
public IReadOnlyList<string>? Related { get; init; }
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("severity")]
public IReadOnlyList<OsvSeverityDto>? Severity { get; init; }
using System;
using System.Collections.Generic;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.Connector.Osv.Internal;
internal sealed record OsvVulnerabilityDto
{
[JsonPropertyName("id")]
public string Id { get; init; } = string.Empty;
[JsonPropertyName("summary")]
public string? Summary { get; init; }
[JsonPropertyName("details")]
public string? Details { get; init; }
[JsonPropertyName("aliases")]
public IReadOnlyList<string>? Aliases { get; init; }
[JsonPropertyName("related")]
public IReadOnlyList<string>? Related { get; init; }
[JsonPropertyName("published")]
public DateTimeOffset? Published { get; init; }
[JsonPropertyName("modified")]
public DateTimeOffset? Modified { get; init; }
[JsonPropertyName("severity")]
public IReadOnlyList<OsvSeverityDto>? Severity { get; init; }
[JsonPropertyName("references")]
public IReadOnlyList<OsvReferenceDto>? References { get; init; }
@@ -39,20 +39,20 @@ internal sealed record OsvVulnerabilityDto
[JsonPropertyName("credits")]
public IReadOnlyList<OsvCreditDto>? Credits { get; init; }
[JsonPropertyName("database_specific")]
public JsonElement DatabaseSpecific { get; init; }
}
internal sealed record OsvSeverityDto
{
[JsonPropertyName("type")]
public string? Type { get; init; }
[JsonPropertyName("score")]
public string? Score { get; init; }
}
[JsonPropertyName("database_specific")]
public JsonElement DatabaseSpecific { get; init; }
}
internal sealed record OsvSeverityDto
{
[JsonPropertyName("type")]
public string? Type { get; init; }
[JsonPropertyName("score")]
public string? Score { get; init; }
}
internal sealed record OsvReferenceDto
{
[JsonPropertyName("type")]
@@ -73,57 +73,57 @@ internal sealed record OsvCreditDto
[JsonPropertyName("contact")]
public IReadOnlyList<string>? Contact { get; init; }
}
internal sealed record OsvAffectedPackageDto
{
[JsonPropertyName("package")]
public OsvPackageDto? Package { get; init; }
[JsonPropertyName("ranges")]
public IReadOnlyList<OsvRangeDto>? Ranges { get; init; }
[JsonPropertyName("versions")]
public IReadOnlyList<string>? Versions { get; init; }
[JsonPropertyName("ecosystem_specific")]
public JsonElement EcosystemSpecific { get; init; }
}
internal sealed record OsvPackageDto
{
[JsonPropertyName("ecosystem")]
public string? Ecosystem { get; init; }
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("purl")]
public string? Purl { get; init; }
}
internal sealed record OsvRangeDto
{
[JsonPropertyName("type")]
public string? Type { get; init; }
[JsonPropertyName("events")]
public IReadOnlyList<OsvEventDto>? Events { get; init; }
[JsonPropertyName("repo")]
public string? Repository { get; init; }
}
internal sealed record OsvEventDto
{
[JsonPropertyName("introduced")]
public string? Introduced { get; init; }
[JsonPropertyName("fixed")]
public string? Fixed { get; init; }
[JsonPropertyName("last_affected")]
public string? LastAffected { get; init; }
[JsonPropertyName("limit")]
public string? Limit { get; init; }
}
internal sealed record OsvAffectedPackageDto
{
[JsonPropertyName("package")]
public OsvPackageDto? Package { get; init; }
[JsonPropertyName("ranges")]
public IReadOnlyList<OsvRangeDto>? Ranges { get; init; }
[JsonPropertyName("versions")]
public IReadOnlyList<string>? Versions { get; init; }
[JsonPropertyName("ecosystem_specific")]
public JsonElement EcosystemSpecific { get; init; }
}
internal sealed record OsvPackageDto
{
[JsonPropertyName("ecosystem")]
public string? Ecosystem { get; init; }
[JsonPropertyName("name")]
public string? Name { get; init; }
[JsonPropertyName("purl")]
public string? Purl { get; init; }
}
internal sealed record OsvRangeDto
{
[JsonPropertyName("type")]
public string? Type { get; init; }
[JsonPropertyName("events")]
public IReadOnlyList<OsvEventDto>? Events { get; init; }
[JsonPropertyName("repo")]
public string? Repository { get; init; }
}
internal sealed record OsvEventDto
{
[JsonPropertyName("introduced")]
public string? Introduced { get; init; }
[JsonPropertyName("fixed")]
public string? Fixed { get; init; }
[JsonPropertyName("last_affected")]
public string? LastAffected { get; init; }
[JsonPropertyName("limit")]
public string? Limit { get; init; }
}

92
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/Jobs.cs
View File

@@ -1,46 +1,46 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Osv;
internal static class OsvJobKinds
{
public const string Fetch = "source:osv:fetch";
public const string Parse = "source:osv:parse";
public const string Map = "source:osv:map";
}
internal sealed class OsvFetchJob : IJob
{
private readonly OsvConnector _connector;
public OsvFetchJob(OsvConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class OsvParseJob : IJob
{
private readonly OsvConnector _connector;
public OsvParseJob(OsvConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class OsvMapJob : IJob
{
private readonly OsvConnector _connector;
public OsvMapJob(OsvConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}
using System;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Concelier.Core.Jobs;
namespace StellaOps.Concelier.Connector.Osv;
internal static class OsvJobKinds
{
public const string Fetch = "source:osv:fetch";
public const string Parse = "source:osv:parse";
public const string Map = "source:osv:map";
}
internal sealed class OsvFetchJob : IJob
{
private readonly OsvConnector _connector;
public OsvFetchJob(OsvConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.FetchAsync(context.Services, cancellationToken);
}
internal sealed class OsvParseJob : IJob
{
private readonly OsvConnector _connector;
public OsvParseJob(OsvConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.ParseAsync(context.Services, cancellationToken);
}
internal sealed class OsvMapJob : IJob
{
private readonly OsvConnector _connector;
public OsvMapJob(OsvConnector connector)
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
=> _connector.MapAsync(context.Services, cancellationToken);
}

8
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/OsvConnector.cs
View File

@@ -11,8 +11,8 @@ using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Bson;
using StellaOps.Concelier.Bson.IO;
using StellaOps.Concelier.Documents;
using StellaOps.Concelier.Documents.IO;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
@@ -188,7 +188,7 @@ public sealed class OsvConnector : IFeedConnector
}
var sanitized = JsonSerializer.Serialize(dto, SerializerOptions);
var payload = StellaOps.Concelier.Bson.BsonDocument.Parse(sanitized);
var payload = StellaOps.Concelier.Documents.DocumentObject.Parse(sanitized);
var dtoRecord = new DtoRecord(
Guid.NewGuid(),
document.Id,
@@ -302,7 +302,7 @@ public sealed class OsvConnector : IFeedConnector
private async Task UpdateCursorAsync(OsvCursor cursor, CancellationToken cancellationToken)
{
var document = cursor.ToBsonDocument();
var document = cursor.ToDocumentObject();
await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
}

40
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/OsvConnectorPlugin.cs
View File

@@ -1,20 +1,20 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Osv;
public sealed class OsvConnectorPlugin : IConnectorPlugin
{
public string Name => SourceName;
public static string SourceName => "osv";
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<OsvConnector>(services);
}
}
using System;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
namespace StellaOps.Concelier.Connector.Osv;
public sealed class OsvConnectorPlugin : IConnectorPlugin
{
public string Name => SourceName;
public static string SourceName => "osv";
public bool IsAvailable(IServiceProvider services) => services is not null;
public IFeedConnector Create(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
return ActivatorUtilities.CreateInstance<OsvConnector>(services);
}
}

106
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/OsvDependencyInjectionRoutine.cs
View File

@@ -1,53 +1,53 @@
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Osv.Configuration;
namespace StellaOps.Concelier.Connector.Osv;
public sealed class OsvDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:osv";
private const string FetchCron = "0,20,40 * * * *";
private const string ParseCron = "5,25,45 * * * *";
private const string MapCron = "10,30,50 * * * *";
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(15);
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(20);
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(20);
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(10);
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddOsvConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
var scheduler = new JobSchedulerBuilder(services);
scheduler
.AddJob<OsvFetchJob>(
OsvJobKinds.Fetch,
cronExpression: FetchCron,
timeout: FetchTimeout,
leaseDuration: LeaseDuration)
.AddJob<OsvParseJob>(
OsvJobKinds.Parse,
cronExpression: ParseCron,
timeout: ParseTimeout,
leaseDuration: LeaseDuration)
.AddJob<OsvMapJob>(
OsvJobKinds.Map,
cronExpression: MapCron,
timeout: MapTimeout,
leaseDuration: LeaseDuration);
return services;
}
}
using System;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.DependencyInjection;
using StellaOps.Concelier.Core.Jobs;
using StellaOps.Concelier.Connector.Osv.Configuration;
namespace StellaOps.Concelier.Connector.Osv;
public sealed class OsvDependencyInjectionRoutine : IDependencyInjectionRoutine
{
private const string ConfigurationSection = "concelier:sources:osv";
private const string FetchCron = "0,20,40 * * * *";
private const string ParseCron = "5,25,45 * * * *";
private const string MapCron = "10,30,50 * * * *";
private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(15);
private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(20);
private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(20);
private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(10);
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
services.AddOsvConnector(options =>
{
configuration.GetSection(ConfigurationSection).Bind(options);
options.Validate();
});
var scheduler = new JobSchedulerBuilder(services);
scheduler
.AddJob<OsvFetchJob>(
OsvJobKinds.Fetch,
cronExpression: FetchCron,
timeout: FetchTimeout,
leaseDuration: LeaseDuration)
.AddJob<OsvParseJob>(
OsvJobKinds.Parse,
cronExpression: ParseCron,
timeout: ParseTimeout,
leaseDuration: LeaseDuration)
.AddJob<OsvMapJob>(
OsvJobKinds.Map,
cronExpression: MapCron,
timeout: MapTimeout,
leaseDuration: LeaseDuration);
return services;
}
}

46
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/OsvServiceCollectionExtensions.cs
View File

@@ -1,31 +1,31 @@
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Common.Http;
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Connector.Common.Http;
using StellaOps.Concelier.Connector.Osv.Configuration;
using StellaOps.Concelier.Connector.Osv.Internal;
namespace StellaOps.Concelier.Connector.Osv;
public static class OsvServiceCollectionExtensions
{
public static IServiceCollection AddOsvConnector(this IServiceCollection services, Action<OsvOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<OsvOptions>()
.Configure(configure)
.PostConfigure(static opts => opts.Validate());
namespace StellaOps.Concelier.Connector.Osv;
public static class OsvServiceCollectionExtensions
{
public static IServiceCollection AddOsvConnector(this IServiceCollection services, Action<OsvOptions> configure)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<OsvOptions>()
.Configure(configure)
.PostConfigure(static opts => opts.Validate());
services.AddSourceHttpClient(OsvOptions.HttpClientName, (sp, clientOptions) =>
{
var options = sp.GetRequiredService<IOptions<OsvOptions>>().Value;
clientOptions.BaseAddress = options.BaseUri;
clientOptions.Timeout = options.HttpTimeout;
clientOptions.UserAgent = "StellaOps.Concelier.OSV/1.0";
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseUri.Host);
clientOptions.UserAgent = "StellaOps.Concelier.OSV/1.0";
clientOptions.AllowedHosts.Clear();
clientOptions.AllowedHosts.Add(options.BaseUri.Host);
clientOptions.DefaultRequestHeaders["Accept"] = "application/zip";
});
@@ -35,5 +35,5 @@ public static class OsvServiceCollectionExtensions
services.AddTransient<OsvParseJob>();
services.AddTransient<OsvMapJob>();
return services;
}
}
}
}

6
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/Properties/AssemblyInfo.cs
View File

@@ -1,3 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("FixtureUpdater")]
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("FixtureUpdater")]