stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 12 Releases Wiki Activity

feat: Add documentation and task tracking for Sprints 508 to 514 in Ops & Offline

Browse Source 
- Created detailed markdown files for Sprints 508 (Ops Offline Kit), 509 (Samples), 510 (AirGap), 511 (Api), 512 (Bench), 513 (Provenance), and 514 (Sovereign Crypto Enablement) outlining tasks, dependencies, and owners.
- Introduced a comprehensive Reachability Evidence Delivery Guide to streamline the reachability signal process.
- Implemented unit tests for Advisory AI to block known injection patterns and redact secrets.
- Added AuthoritySenderConstraintHelper to manage sender constraints in OpenIddict transactions.
This commit is contained in:
master
2025-11-08 23:18:28 +02:00
parent 536f6249a6
commit ae69b1a8a1
187 changed files with 4326 additions and 3196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchService.cs
View File

@@ -6,7 +6,6 @@ using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -19,6 +18,7 @@ using StellaOps.Concelier.RawModels;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Documents;
using System.Text.Json;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Connector.Common.Fetch;
@@ -40,15 +40,17 @@ public sealed class SourceFetchService
private readonly IAdvisoryRawWriteGuard _guard;
private readonly IAdvisoryLinksetMapper _linksetMapper;
private readonly string _connectorVersion;
public SourceFetchService(
IHttpClientFactory httpClientFactory,
RawDocumentStorage rawDocumentStorage,
IDocumentStore documentStore,
ILogger<SourceFetchService> logger,
private readonly ICryptoHash _hash;
public SourceFetchService(
IHttpClientFactory httpClientFactory,
RawDocumentStorage rawDocumentStorage,
IDocumentStore documentStore,
ILogger<SourceFetchService> logger,
IJitterSource jitterSource,
IAdvisoryRawWriteGuard guard,
IAdvisoryLinksetMapper linksetMapper,
ICryptoHash hash,
TimeProvider? timeProvider = null,
IOptionsMonitor<SourceHttpClientOptions>? httpClientOptions = null,
IOptions<MongoStorageOptions>? storageOptions = null)
@@ -60,6 +62,7 @@ public sealed class SourceFetchService
_jitterSource = jitterSource ?? throw new ArgumentNullException(nameof(jitterSource));
_guard = guard ?? throw new ArgumentNullException(nameof(guard));
_linksetMapper = linksetMapper ?? throw new ArgumentNullException(nameof(linksetMapper));
_hash = hash ?? throw new ArgumentNullException(nameof(hash));
_timeProvider = timeProvider ?? TimeProvider.System;
_httpClientOptions = httpClientOptions ?? throw new ArgumentNullException(nameof(httpClientOptions));
_storageOptions = storageOptions ?? throw new ArgumentNullException(nameof(storageOptions));
@@ -103,7 +106,7 @@ public sealed class SourceFetchService
}
var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
var contentHash = Convert.ToHexString(SHA256.HashData(contentBytes)).ToLowerInvariant();
var contentHash = _hash.ComputeHashHex(contentBytes, HashAlgorithms.Sha256);
var fetchedAt = _timeProvider.GetUtcNow();
var contentType = response.Content.Headers.ContentType?.ToString();

71
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/State/SourceStateSeedProcessor.cs
View File

@@ -1,10 +1,10 @@
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Bson;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Documents;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Bson;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Connector.Common.State;
@@ -15,23 +15,26 @@ public sealed class SourceStateSeedProcessor
{
private readonly IDocumentStore _documentStore;
private readonly RawDocumentStorage _rawDocumentStorage;
private readonly ISourceStateRepository _stateRepository;
private readonly TimeProvider _timeProvider;
private readonly ILogger<SourceStateSeedProcessor> _logger;
public SourceStateSeedProcessor(
IDocumentStore documentStore,
RawDocumentStorage rawDocumentStorage,
ISourceStateRepository stateRepository,
TimeProvider? timeProvider = null,
ILogger<SourceStateSeedProcessor>? logger = null)
{
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? NullLogger<SourceStateSeedProcessor>.Instance;
}
private readonly ISourceStateRepository _stateRepository;
private readonly TimeProvider _timeProvider;
private readonly ILogger<SourceStateSeedProcessor> _logger;
private readonly ICryptoHash _hash;
public SourceStateSeedProcessor(
IDocumentStore documentStore,
RawDocumentStorage rawDocumentStorage,
ISourceStateRepository stateRepository,
ICryptoHash hash,
TimeProvider? timeProvider = null,
ILogger<SourceStateSeedProcessor>? logger = null)
{
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_hash = hash ?? throw new ArgumentNullException(nameof(hash));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? NullLogger<SourceStateSeedProcessor>.Instance;
}
public async Task<SourceStateSeedResult> ProcessAsync(SourceStateSeedSpecification specification, CancellationToken cancellationToken)
{
@@ -138,7 +141,7 @@ public sealed class SourceStateSeedProcessor
_logger.LogWarning("Seed document URI '{Uri}' does not appear to be absolute.", document.Uri);
}
var sha256 = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
var contentHash = _hash.ComputeHashHex(payload, HashAlgorithms.Sha256);
var existing = await _documentStore.FindBySourceAndUriAsync(source, document.Uri, cancellationToken).ConfigureAwait(false);
@@ -168,12 +171,12 @@ public sealed class SourceStateSeedProcessor
var metadata = CloneDictionary(document.Metadata);
var record = new DocumentRecord(
document.DocumentId ?? existing?.Id ?? Guid.NewGuid(),
source,
document.Uri,
document.FetchedAt ?? completedAt,
sha256,
var record = new DocumentRecord(
document.DocumentId ?? existing?.Id ?? Guid.NewGuid(),
source,
document.Uri,
document.FetchedAt ?? completedAt,
contentHash,
string.IsNullOrWhiteSpace(document.Status) ? DocumentStatuses.PendingParse : document.Status,
document.ContentType,
headers,
@@ -181,9 +184,9 @@ public sealed class SourceStateSeedProcessor
document.Etag,
document.LastModified,
gridId,
document.ExpiresAt);
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
document.ExpiresAt);
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
documentIds.Add(upserted.Id);

3
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj
View File

@@ -17,6 +17,7 @@
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj" />
</ItemGroup>
</Project>
</Project>

3
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Ubuntu/StellaOps.Concelier.Connector.Distro.Ubuntu.csproj
View File

@@ -14,5 +14,6 @@
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
</ItemGroup>
</Project>
</Project>

57
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuConnector.cs
View File

@@ -1,22 +1,22 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Globalization;
using System.Text;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration;
using StellaOps.Concelier.Connector.Distro.Ubuntu.Internal;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Plugin;
using System.Globalization;
using System.Text;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.Connector.Common;
using StellaOps.Concelier.Connector.Common.Fetch;
using StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration;
using StellaOps.Concelier.Connector.Distro.Ubuntu.Internal;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Plugin;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Connector.Distro.Ubuntu;
@@ -29,8 +29,9 @@ public sealed class UbuntuConnector : IFeedConnector
private readonly IAdvisoryStore _advisoryStore;
private readonly ISourceStateRepository _stateRepository;
private readonly UbuntuOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<UbuntuConnector> _logger;
private readonly TimeProvider _timeProvider;
private readonly ILogger<UbuntuConnector> _logger;
private readonly ICryptoHash _hash;
private static readonly Action<ILogger, string, int, Exception?> LogMapped =
LoggerMessage.Define<string, int>(
@@ -45,9 +46,10 @@ public sealed class UbuntuConnector : IFeedConnector
IDtoStore dtoStore,
IAdvisoryStore advisoryStore,
ISourceStateRepository stateRepository,
IOptions<UbuntuOptions> options,
TimeProvider? timeProvider,
ILogger<UbuntuConnector> logger)
IOptions<UbuntuOptions> options,
TimeProvider? timeProvider,
ILogger<UbuntuConnector> logger,
ICryptoHash cryptoHash)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
@@ -56,9 +58,10 @@ public sealed class UbuntuConnector : IFeedConnector
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_hash = cryptoHash ?? throw new ArgumentNullException(nameof(cryptoHash));
}
public string SourceName => UbuntuConnectorPlugin.SourceName;
@@ -418,9 +421,9 @@ public sealed class UbuntuConnector : IFeedConnector
private static string ComputeNoticeHash(BsonDocument document)
{
var bytes = document.ToBson();
var hash = SHA256.HashData(bytes);
return Convert.ToHexString(hash).ToLowerInvariant();
}
var hash = _hash.ComputeHash(bytes, HashAlgorithms.Sha256);
return Convert.ToHexString(hash).ToLowerInvariant();
}
private static BsonDocument ToBson(UbuntuNoticeDto notice)
{

45
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/NvdConnector.cs
View File

@@ -1,5 +1,4 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging;
@@ -17,8 +16,9 @@ using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Concelier.Storage.Mongo.ChangeHistory;
using StellaOps.Plugin;
using Json.Schema;
using StellaOps.Plugin;
using Json.Schema;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Connector.Nvd;
@@ -32,10 +32,11 @@ public sealed class NvdConnector : IFeedConnector
private readonly IChangeHistoryStore _changeHistoryStore;
private readonly ISourceStateRepository _stateRepository;
private readonly IJsonSchemaValidator _schemaValidator;
private readonly NvdOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<NvdConnector> _logger;
private readonly NvdDiagnostics _diagnostics;
private readonly NvdOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<NvdConnector> _logger;
private readonly NvdDiagnostics _diagnostics;
private readonly ICryptoHash _hash;
private static readonly JsonSchema Schema = NvdSchemaProvider.Schema;
@@ -48,10 +49,11 @@ public sealed class NvdConnector : IFeedConnector
IChangeHistoryStore changeHistoryStore,
ISourceStateRepository stateRepository,
IJsonSchemaValidator schemaValidator,
IOptions<NvdOptions> options,
NvdDiagnostics diagnostics,
TimeProvider? timeProvider,
ILogger<NvdConnector> logger)
IOptions<NvdOptions> options,
NvdDiagnostics diagnostics,
ICryptoHash hash,
TimeProvider? timeProvider,
ILogger<NvdConnector> logger)
{
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
@@ -63,10 +65,11 @@ public sealed class NvdConnector : IFeedConnector
_schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_options.Validate();
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_hash = hash ?? throw new ArgumentNullException(nameof(hash));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public string SourceName => NvdConnectorPlugin.SourceName;
@@ -524,12 +527,12 @@ public sealed class NvdConnector : IFeedConnector
private static string SerializeElement(JsonElement element)
=> JsonSerializer.Serialize(element, new JsonSerializerOptions { WriteIndented = false });
private static string ComputeHash(string snapshot)
{
var bytes = Encoding.UTF8.GetBytes(snapshot);
var hash = SHA256.HashData(bytes);
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
}
private string ComputeHash(string snapshot)
{
var bytes = Encoding.UTF8.GetBytes(snapshot);
var hex = _hash.ComputeHashHex(bytes, HashAlgorithms.Sha256);
return $"sha256:{hex}";
}
private async Task<NvdCursor> GetCursorAsync(CancellationToken cancellationToken)
{

3
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Nvd/StellaOps.Concelier.Connector.Nvd.csproj
View File

@@ -9,10 +9,11 @@
<EmbeddedResource Include="Schemas\nvd-vulnerability.schema.json" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" />
</ItemGroup>
</Project>
</Project>

17
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/OsvConnector.cs
View File

@@ -5,9 +5,8 @@ using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@@ -21,9 +20,10 @@ using StellaOps.Concelier.Connector.Osv.Configuration;
using StellaOps.Concelier.Connector.Osv.Internal;
using StellaOps.Concelier.Storage.Mongo;
using StellaOps.Concelier.Storage.Mongo.Advisories;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Plugin;
using StellaOps.Concelier.Storage.Mongo.Documents;
using StellaOps.Concelier.Storage.Mongo.Dtos;
using StellaOps.Plugin;
using StellaOps.Cryptography;
namespace StellaOps.Concelier.Connector.Osv;
@@ -45,6 +45,7 @@ public sealed class OsvConnector : IFeedConnector
private readonly TimeProvider _timeProvider;
private readonly ILogger<OsvConnector> _logger;
private readonly OsvDiagnostics _diagnostics;
private readonly ICryptoHash _hash;
public OsvConnector(
IHttpClientFactory httpClientFactory,
@@ -55,6 +56,7 @@ public sealed class OsvConnector : IFeedConnector
ISourceStateRepository stateRepository,
IOptions<OsvOptions> options,
OsvDiagnostics diagnostics,
ICryptoHash hash,
TimeProvider? timeProvider,
ILogger<OsvConnector> logger)
{
@@ -66,6 +68,7 @@ public sealed class OsvConnector : IFeedConnector
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_hash = hash ?? throw new ArgumentNullException(nameof(hash));
_options.Validate();
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -415,7 +418,7 @@ public sealed class OsvConnector : IFeedConnector
}
var documentUri = BuildDocumentUri(ecosystem, dto.Id);
var sha256 = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant();
var sha256 = _hash.ComputeHashHex(bytes, HashAlgorithms.Sha256);
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false);
if (existing is not null && string.Equals(existing.Sha256, sha256, StringComparison.OrdinalIgnoreCase))

3
src/Concelier/__Libraries/StellaOps.Concelier.Connector.Osv/StellaOps.Concelier.Connector.Osv.csproj
View File

@@ -7,6 +7,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" />
<ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
@@ -21,4 +22,4 @@
<_Parameter1>StellaOps.Concelier.Connector.Osv.Tests</_Parameter1>
</AssemblyAttribute>
</ItemGroup>
</Project>
</Project>