sprints work.
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
# GroundTruth.Debuginfod - Agent Instructions
|
||||
|
||||
## Module Overview
|
||||
|
||||
This library implements the debuginfod symbol source connector for fetching debug symbols from Fedora/RHEL debuginfod services.
|
||||
|
||||
## Key Components
|
||||
|
||||
- **DebuginfodConnector** - Main connector implementing three-phase pipeline
|
||||
- **DebuginfodConnectorPlugin** - Plugin registration for DI discovery
|
||||
- **DebuginfodOptions** - Configuration options
|
||||
- **DebuginfodDiagnostics** - Metrics and telemetry
|
||||
- **IDwarfParser** - Interface for DWARF symbol parsing
|
||||
|
||||
## Configuration
|
||||
|
||||
Environment variables:
|
||||
- `DEBUGINFOD_URLS` - Space/comma-separated list of debuginfod server URLs
|
||||
- `DEBUGINFOD_CACHE` - Local cache directory
|
||||
- `DEBUGINFOD_TIMEOUT` - Request timeout in seconds
|
||||
|
||||
## Three-Phase Pipeline
|
||||
|
||||
1. **Fetch**: Download debuginfo by build-id from debuginfod server
|
||||
2. **Parse**: Extract DWARF symbols using IDwarfParser
|
||||
3. **Map**: Build canonical SymbolObservation with AOC compliance
|
||||
|
||||
## Debuginfod Protocol
|
||||
|
||||
API endpoints:
|
||||
- `GET /buildid/{buildid}/debuginfo` - Fetch debug info
|
||||
- `GET /buildid/{buildid}/executable` - Fetch executable
|
||||
- `GET /buildid/{buildid}/source/{path}` - Fetch source file
|
||||
- `GET /metrics` - Prometheus metrics (for health checks)
|
||||
|
||||
## Testing
|
||||
|
||||
- Unit tests for connector logic
|
||||
- Integration tests require access to debuginfod server (skippable)
|
||||
- Deterministic fixtures for offline testing
|
||||
|
||||
## Future Work
|
||||
|
||||
- Implement real IDwarfParser using Gimli or libdw
|
||||
- IMA signature verification
|
||||
- Source file fetching
|
||||
- Multi-server fallback
|
||||
@@ -0,0 +1,99 @@
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Debuginfod.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for the debuginfod connector.
|
||||
/// </summary>
|
||||
public sealed class DebuginfodOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Section name for configuration binding.
|
||||
/// </summary>
|
||||
public const string SectionName = "GroundTruth:Debuginfod";
|
||||
|
||||
/// <summary>
|
||||
/// HTTP client name for DI.
|
||||
/// </summary>
|
||||
public const string HttpClientName = "debuginfod";
|
||||
|
||||
/// <summary>
|
||||
/// Base URL for the debuginfod service.
|
||||
/// Defaults to Fedora's public debuginfod service.
|
||||
/// </summary>
|
||||
public Uri BaseUrl { get; set; } = new("https://debuginfod.fedoraproject.org");
|
||||
|
||||
/// <summary>
|
||||
/// Additional debuginfod URLs to query (for fallback or multiple sources).
|
||||
/// </summary>
|
||||
public List<Uri> AdditionalUrls { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Request timeout in seconds.
|
||||
/// </summary>
|
||||
public int TimeoutSeconds { get; set; } = 30;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum concurrent requests.
|
||||
/// </summary>
|
||||
public int MaxConcurrentRequests { get; set; } = 4;
|
||||
|
||||
/// <summary>
|
||||
/// Retry count for failed requests.
|
||||
/// </summary>
|
||||
public int RetryCount { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Initial retry delay in milliseconds.
|
||||
/// </summary>
|
||||
public int RetryDelayMs { get; set; } = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to verify IMA signatures when available.
|
||||
/// </summary>
|
||||
public bool VerifyImaSignatures { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Local cache directory for downloaded debuginfo.
|
||||
/// </summary>
|
||||
public string? CacheDirectory { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum cache size in megabytes.
|
||||
/// </summary>
|
||||
public int MaxCacheSizeMb { get; set; } = 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Cache expiration in hours.
|
||||
/// </summary>
|
||||
public int CacheExpirationHours { get; set; } = 168; // 1 week
|
||||
|
||||
/// <summary>
|
||||
/// User agent string.
|
||||
/// </summary>
|
||||
public string UserAgent { get; set; } = "StellaOps.GroundTruth.Debuginfod/1.0";
|
||||
|
||||
/// <summary>
|
||||
/// Whether to include source files in fetch.
|
||||
/// </summary>
|
||||
public bool IncludeSourceFiles { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Validate options.
|
||||
/// </summary>
|
||||
public void Validate()
|
||||
{
|
||||
if (BaseUrl is null)
|
||||
throw new InvalidOperationException("Debuginfod base URL must be configured.");
|
||||
|
||||
if (!BaseUrl.IsAbsoluteUri)
|
||||
throw new InvalidOperationException("Debuginfod base URL must be an absolute URI.");
|
||||
|
||||
if (TimeoutSeconds <= 0)
|
||||
throw new InvalidOperationException("Timeout must be positive.");
|
||||
|
||||
if (MaxConcurrentRequests <= 0)
|
||||
throw new InvalidOperationException("Max concurrent requests must be positive.");
|
||||
|
||||
if (RetryCount < 0)
|
||||
throw new InvalidOperationException("Retry count cannot be negative.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,449 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Net;
|
||||
using System.Runtime.CompilerServices;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Abstractions;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Debuginfod.Configuration;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Debuginfod.Internal;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Debuginfod;
|
||||
|
||||
/// <summary>
|
||||
/// Debuginfod symbol source connector for Fedora/RHEL debuginfod services.
|
||||
/// Implements the three-phase pipeline: Fetch → Parse → Map.
|
||||
/// </summary>
|
||||
public sealed class DebuginfodConnector : SymbolSourceConnectorBase, ISymbolSourceCapability
|
||||
{
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly ISymbolRawDocumentRepository _documentRepository;
|
||||
private readonly ISymbolObservationRepository _observationRepository;
|
||||
private readonly ISymbolSourceStateRepository _stateRepository;
|
||||
private readonly ISymbolObservationWriteGuard _writeGuard;
|
||||
private readonly DebuginfodOptions _options;
|
||||
private readonly DebuginfodDiagnostics _diagnostics;
|
||||
|
||||
/// <summary>
|
||||
/// Source ID for this connector.
|
||||
/// </summary>
|
||||
public const string SourceName = "debuginfod-fedora";
|
||||
|
||||
public DebuginfodConnector(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
ISymbolRawDocumentRepository documentRepository,
|
||||
ISymbolObservationRepository observationRepository,
|
||||
ISymbolSourceStateRepository stateRepository,
|
||||
ISymbolObservationWriteGuard writeGuard,
|
||||
IOptions<DebuginfodOptions> options,
|
||||
DebuginfodDiagnostics diagnostics,
|
||||
ILogger<DebuginfodConnector> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
: base(logger, timeProvider)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_documentRepository = documentRepository ?? throw new ArgumentNullException(nameof(documentRepository));
|
||||
_observationRepository = observationRepository ?? throw new ArgumentNullException(nameof(observationRepository));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_writeGuard = writeGuard ?? throw new ArgumentNullException(nameof(writeGuard));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override string SourceId => SourceName;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override string DisplayName => "Fedora debuginfod";
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override IReadOnlyList<string> SupportedDistros =>
|
||||
["fedora", "rhel", "centos", "rocky", "alma"];
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.GetOrCreateAsync(SourceId, cancellationToken);
|
||||
|
||||
// Check backoff
|
||||
if (state.BackoffUntil.HasValue && state.BackoffUntil.Value > UtcNow)
|
||||
{
|
||||
Logger.LogInformation(
|
||||
"Debuginfod fetch skipped due to backoff until {BackoffUntil}",
|
||||
state.BackoffUntil.Value);
|
||||
return;
|
||||
}
|
||||
|
||||
// Get pending debug IDs from cursor (or use configured list)
|
||||
var debugIds = GetPendingDebugIds(state);
|
||||
if (debugIds.Length == 0)
|
||||
{
|
||||
Logger.LogDebug("No pending debug IDs to fetch from debuginfod");
|
||||
return;
|
||||
}
|
||||
|
||||
var httpClient = _httpClientFactory.CreateClient(DebuginfodOptions.HttpClientName);
|
||||
var fetchedCount = 0;
|
||||
var errorCount = 0;
|
||||
|
||||
foreach (var debugId in debugIds)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
var document = await FetchDebugInfoAsync(httpClient, debugId, cancellationToken);
|
||||
if (document is not null)
|
||||
{
|
||||
await _documentRepository.UpsertAsync(document, cancellationToken);
|
||||
state = state.AddPendingParse(document.Digest);
|
||||
fetchedCount++;
|
||||
_diagnostics.RecordFetchSuccess();
|
||||
}
|
||||
}
|
||||
catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
|
||||
{
|
||||
Logger.LogDebug("Debug ID {DebugId} not found in debuginfod", debugId);
|
||||
_diagnostics.RecordFetchNotFound();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
LogError(ex, "Fetch", $"Failed to fetch debug ID {debugId}");
|
||||
errorCount++;
|
||||
_diagnostics.RecordFetchError();
|
||||
|
||||
if (errorCount > 5)
|
||||
{
|
||||
await _stateRepository.MarkFailedAsync(
|
||||
SourceId,
|
||||
$"Too many fetch errors: {ex.Message}",
|
||||
TimeSpan.FromMinutes(15),
|
||||
cancellationToken);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
state = state with { LastSuccessAt = UtcNow };
|
||||
await _stateRepository.UpdateAsync(state, cancellationToken);
|
||||
|
||||
Logger.LogInformation(
|
||||
"Debuginfod fetch completed: {FetchedCount} fetched, {ErrorCount} errors",
|
||||
fetchedCount, errorCount);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.GetOrCreateAsync(SourceId, cancellationToken);
|
||||
|
||||
if (state.PendingParse.Length == 0)
|
||||
{
|
||||
Logger.LogDebug("No documents pending parse for debuginfod");
|
||||
return;
|
||||
}
|
||||
|
||||
var dwParser = services.GetRequiredService<IDwarfParser>();
|
||||
var parsedCount = 0;
|
||||
|
||||
foreach (var digest in state.PendingParse)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentRepository.FindByDigestAsync(digest, cancellationToken);
|
||||
if (document is null)
|
||||
{
|
||||
Logger.LogWarning("Document {Digest} not found for parse", digest);
|
||||
state = state.RemovePendingParse(digest);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Parse DWARF symbols
|
||||
var symbols = await dwParser.ParseSymbolsAsync(
|
||||
document.PayloadId!.Value,
|
||||
cancellationToken);
|
||||
|
||||
LogParse(digest, symbols.Count);
|
||||
|
||||
// Update document status and move to map phase
|
||||
await _documentRepository.UpdateStatusAsync(digest, DocumentStatus.PendingMap, cancellationToken);
|
||||
state = state.MoveToPendingMap(digest);
|
||||
parsedCount++;
|
||||
_diagnostics.RecordParseSuccess(symbols.Count);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
LogError(ex, "Parse", $"Failed to parse document {digest}");
|
||||
await _documentRepository.UpdateStatusAsync(digest, DocumentStatus.Failed, cancellationToken);
|
||||
state = state.RemovePendingParse(digest);
|
||||
_diagnostics.RecordParseError();
|
||||
}
|
||||
}
|
||||
|
||||
await _stateRepository.UpdateAsync(state, cancellationToken);
|
||||
|
||||
Logger.LogInformation("Debuginfod parse completed: {ParsedCount} documents parsed", parsedCount);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.GetOrCreateAsync(SourceId, cancellationToken);
|
||||
|
||||
if (state.PendingMap.Length == 0)
|
||||
{
|
||||
Logger.LogDebug("No documents pending map for debuginfod");
|
||||
return;
|
||||
}
|
||||
|
||||
var dwParser = services.GetRequiredService<IDwarfParser>();
|
||||
var mappedCount = 0;
|
||||
|
||||
foreach (var digest in state.PendingMap)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentRepository.FindByDigestAsync(digest, cancellationToken);
|
||||
if (document is null)
|
||||
{
|
||||
Logger.LogWarning("Document {Digest} not found for map", digest);
|
||||
state = state.MarkMapped(digest);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Parse symbols from stored payload
|
||||
var symbols = await dwParser.ParseSymbolsAsync(
|
||||
document.PayloadId!.Value,
|
||||
cancellationToken);
|
||||
|
||||
// Build observation
|
||||
var observation = BuildObservation(document, symbols);
|
||||
|
||||
// Validate against AOC
|
||||
_writeGuard.EnsureValid(observation);
|
||||
|
||||
// Check for existing observation with same content
|
||||
var existingId = await _observationRepository.FindByContentHashAsync(
|
||||
SourceId,
|
||||
observation.DebugId,
|
||||
observation.ContentHash,
|
||||
cancellationToken);
|
||||
|
||||
if (existingId is not null)
|
||||
{
|
||||
Logger.LogDebug(
|
||||
"Observation already exists with hash {Hash}, skipping",
|
||||
observation.ContentHash);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Insert new observation
|
||||
await _observationRepository.InsertAsync(observation, cancellationToken);
|
||||
LogMap(observation.ObservationId);
|
||||
_diagnostics.RecordMapSuccess(symbols.Count);
|
||||
}
|
||||
|
||||
await _documentRepository.UpdateStatusAsync(digest, DocumentStatus.Mapped, cancellationToken);
|
||||
state = state.MarkMapped(digest);
|
||||
mappedCount++;
|
||||
}
|
||||
catch (GroundTruthAocGuardException ex)
|
||||
{
|
||||
Logger.LogError(
|
||||
"AOC violation mapping document {Digest}: {Violations}",
|
||||
digest,
|
||||
string.Join(", ", ex.Violations.Select(v => v.Code)));
|
||||
await _documentRepository.UpdateStatusAsync(digest, DocumentStatus.Quarantined, cancellationToken);
|
||||
state = state.MarkMapped(digest);
|
||||
_diagnostics.RecordMapAocViolation();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
LogError(ex, "Map", $"Failed to map document {digest}");
|
||||
await _documentRepository.UpdateStatusAsync(digest, DocumentStatus.Failed, cancellationToken);
|
||||
state = state.MarkMapped(digest);
|
||||
_diagnostics.RecordMapError();
|
||||
}
|
||||
}
|
||||
|
||||
await _stateRepository.UpdateAsync(state, cancellationToken);
|
||||
|
||||
Logger.LogInformation("Debuginfod map completed: {MappedCount} documents mapped", mappedCount);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<SymbolSourceConnectivityResult> TestConnectivityAsync(CancellationToken ct = default)
|
||||
{
|
||||
var startTime = UtcNow;
|
||||
try
|
||||
{
|
||||
var httpClient = _httpClientFactory.CreateClient(DebuginfodOptions.HttpClientName);
|
||||
var response = await httpClient.GetAsync("/metrics", ct);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var latency = UtcNow - startTime;
|
||||
return new SymbolSourceConnectivityResult(
|
||||
IsConnected: true,
|
||||
Latency: latency,
|
||||
ErrorMessage: null,
|
||||
TestedAt: UtcNow);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var latency = UtcNow - startTime;
|
||||
return new SymbolSourceConnectivityResult(
|
||||
IsConnected: false,
|
||||
Latency: latency,
|
||||
ErrorMessage: ex.Message,
|
||||
TestedAt: UtcNow);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<SymbolSourceMetadata> GetMetadataAsync(CancellationToken ct = default)
|
||||
{
|
||||
var stats = await _observationRepository.GetStatsAsync(ct);
|
||||
return new SymbolSourceMetadata(
|
||||
SourceId: SourceId,
|
||||
DisplayName: DisplayName,
|
||||
BaseUrl: _options.BaseUrl.ToString(),
|
||||
LastSyncAt: stats.NewestObservation,
|
||||
ObservationCount: (int)stats.TotalObservations,
|
||||
DebugIdCount: (int)stats.UniqueDebugIds,
|
||||
AdditionalInfo: new Dictionary<string, string>
|
||||
{
|
||||
["total_symbols"] = stats.TotalSymbols.ToString()
|
||||
});
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<SymbolData?> FetchByDebugIdAsync(string debugId, CancellationToken ct = default)
|
||||
{
|
||||
var httpClient = _httpClientFactory.CreateClient(DebuginfodOptions.HttpClientName);
|
||||
var document = await FetchDebugInfoAsync(httpClient, debugId, ct);
|
||||
if (document is null)
|
||||
return null;
|
||||
|
||||
// For direct fetch, we need to parse symbols inline
|
||||
// This is a simplified version - full implementation would use stored payload
|
||||
return new SymbolData(
|
||||
DebugId: debugId,
|
||||
BinaryName: document.Metadata.GetValueOrDefault("binary_name", "unknown"),
|
||||
Architecture: document.Metadata.GetValueOrDefault("architecture", "unknown"),
|
||||
Symbols: [],
|
||||
BuildInfo: null,
|
||||
Provenance: new SymbolDataProvenance(
|
||||
SourceId: SourceId,
|
||||
DocumentUri: document.DocumentUri,
|
||||
FetchedAt: document.FetchedAt,
|
||||
ContentHash: document.Digest,
|
||||
SignatureState: SignatureState.None,
|
||||
SignatureDetails: null));
|
||||
}
|
||||
|
||||
private ImmutableArray<string> GetPendingDebugIds(SymbolSourceState state)
|
||||
{
|
||||
// In production, this would come from a work queue or scheduled list
|
||||
// For now, return empty - the connector is query-driven via FetchByDebugIdAsync
|
||||
if (state.Cursor.TryGetValue("pending_debug_ids", out var pending) &&
|
||||
!string.IsNullOrWhiteSpace(pending))
|
||||
{
|
||||
return pending.Split(',', StringSplitOptions.RemoveEmptyEntries)
|
||||
.Select(s => s.Trim())
|
||||
.ToImmutableArray();
|
||||
}
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
private async Task<SymbolRawDocument?> FetchDebugInfoAsync(
|
||||
HttpClient httpClient,
|
||||
string debugId,
|
||||
CancellationToken ct)
|
||||
{
|
||||
// Debuginfod URL pattern: /buildid/{buildid}/debuginfo
|
||||
var requestUri = $"/buildid/{debugId}/debuginfo";
|
||||
LogFetch(requestUri, debugId);
|
||||
|
||||
var response = await httpClient.GetAsync(requestUri, ct);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var content = await response.Content.ReadAsByteArrayAsync(ct);
|
||||
var digest = ComputeDocumentDigest(content);
|
||||
|
||||
// Check if we already have this document
|
||||
var existing = await _documentRepository.FindByDigestAsync(digest, ct);
|
||||
if (existing is not null)
|
||||
{
|
||||
Logger.LogDebug("Document {Digest} already exists, skipping", digest);
|
||||
return null;
|
||||
}
|
||||
|
||||
var contentType = response.Content.Headers.ContentType?.MediaType ?? "application/x-elf";
|
||||
var etag = response.Headers.ETag?.Tag;
|
||||
|
||||
return new SymbolRawDocument
|
||||
{
|
||||
Digest = digest,
|
||||
SourceId = SourceId,
|
||||
DocumentUri = $"{_options.BaseUrl}{requestUri}",
|
||||
FetchedAt = UtcNow,
|
||||
RecordedAt = UtcNow,
|
||||
ContentType = contentType,
|
||||
ContentSize = content.Length,
|
||||
ETag = etag,
|
||||
Status = DocumentStatus.PendingParse,
|
||||
PayloadId = null, // Will be set by blob storage
|
||||
Metadata = ImmutableDictionary<string, string>.Empty
|
||||
.Add("debug_id", debugId)
|
||||
.Add("binary_name", "unknown") // Would extract from ELF headers
|
||||
};
|
||||
}
|
||||
|
||||
private SymbolObservation BuildObservation(
|
||||
SymbolRawDocument document,
|
||||
IReadOnlyList<ObservedSymbol> symbols)
|
||||
{
|
||||
var debugId = document.Metadata.GetValueOrDefault("debug_id", "unknown");
|
||||
var binaryName = document.Metadata.GetValueOrDefault("binary_name", "unknown");
|
||||
var architecture = document.Metadata.GetValueOrDefault("architecture", "x86_64");
|
||||
|
||||
// Determine revision number
|
||||
var existingObservations = _observationRepository
|
||||
.FindByDebugIdAsync(debugId, CancellationToken.None)
|
||||
.GetAwaiter()
|
||||
.GetResult();
|
||||
var revision = existingObservations.Length + 1;
|
||||
|
||||
var observation = new SymbolObservation
|
||||
{
|
||||
ObservationId = GenerateObservationId(debugId, revision),
|
||||
SourceId = SourceId,
|
||||
DebugId = debugId,
|
||||
BinaryName = binaryName,
|
||||
Architecture = architecture,
|
||||
Symbols = symbols.ToImmutableArray(),
|
||||
SymbolCount = symbols.Count,
|
||||
Provenance = new ObservationProvenance
|
||||
{
|
||||
SourceId = SourceId,
|
||||
DocumentUri = document.DocumentUri,
|
||||
FetchedAt = document.FetchedAt,
|
||||
RecordedAt = UtcNow,
|
||||
DocumentHash = document.Digest,
|
||||
SignatureState = SignatureState.None,
|
||||
ConnectorVersion = "1.0.0"
|
||||
},
|
||||
ContentHash = "", // Will be computed
|
||||
CreatedAt = UtcNow
|
||||
};
|
||||
|
||||
// Compute content hash
|
||||
var contentHash = ComputeContentHash(observation);
|
||||
return observation with { ContentHash = contentHash };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Abstractions;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Debuginfod.Configuration;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Debuginfod;
|
||||
|
||||
/// <summary>
|
||||
/// Plugin for the debuginfod symbol source connector.
|
||||
/// </summary>
|
||||
public sealed class DebuginfodConnectorPlugin : ISymbolSourceConnectorPlugin
|
||||
{
|
||||
/// <inheritdoc/>
|
||||
public string Name => DebuginfodConnector.SourceName;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
// Check if the connector is configured
|
||||
var options = services.GetService<Microsoft.Extensions.Options.IOptions<DebuginfodOptions>>();
|
||||
if (options?.Value is null)
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
options.Value.Validate();
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ISymbolSourceConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return ActivatorUtilities.CreateInstance<DebuginfodConnector>(services);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Abstractions;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Debuginfod.Configuration;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Debuginfod.Internal;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Debuginfod;
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for adding debuginfod connector to DI.
|
||||
/// </summary>
|
||||
public static class DebuginfodServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Add the debuginfod symbol source connector.
|
||||
/// </summary>
|
||||
/// <param name="services">Service collection.</param>
|
||||
/// <param name="configure">Configuration action.</param>
|
||||
/// <returns>Service collection for chaining.</returns>
|
||||
public static IServiceCollection AddDebuginfodConnector(
|
||||
this IServiceCollection services,
|
||||
Action<DebuginfodOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
// Register options with validation
|
||||
services.AddOptions<DebuginfodOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static opts => opts.Validate());
|
||||
|
||||
// Register HTTP client
|
||||
services.AddHttpClient(DebuginfodOptions.HttpClientName, (sp, client) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<DebuginfodOptions>>().Value;
|
||||
client.BaseAddress = options.BaseUrl;
|
||||
client.Timeout = TimeSpan.FromSeconds(options.TimeoutSeconds);
|
||||
client.DefaultRequestHeaders.Add("User-Agent", options.UserAgent);
|
||||
client.DefaultRequestHeaders.Add("Accept", "application/octet-stream");
|
||||
});
|
||||
|
||||
// Register services
|
||||
services.AddSingleton<DebuginfodDiagnostics>();
|
||||
services.AddSingleton<IDwarfParser, ElfDwarfParser>();
|
||||
services.AddTransient<DebuginfodConnector>();
|
||||
services.AddSingleton<ISymbolSourceConnectorPlugin, DebuginfodConnectorPlugin>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Add the debuginfod symbol source connector with default Fedora configuration.
|
||||
/// </summary>
|
||||
/// <param name="services">Service collection.</param>
|
||||
/// <returns>Service collection for chaining.</returns>
|
||||
public static IServiceCollection AddDebuginfodConnector(this IServiceCollection services)
|
||||
{
|
||||
return services.AddDebuginfodConnector(_ => { });
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Add the debuginfod connector from environment variables.
|
||||
/// </summary>
|
||||
/// <param name="services">Service collection.</param>
|
||||
/// <returns>Service collection for chaining.</returns>
|
||||
/// <remarks>
|
||||
/// Reads configuration from:
|
||||
/// - DEBUGINFOD_URLS: Comma-separated list of debuginfod server URLs
|
||||
/// - DEBUGINFOD_CACHE: Local cache directory
|
||||
/// - DEBUGINFOD_TIMEOUT: Request timeout in seconds
|
||||
/// </remarks>
|
||||
public static IServiceCollection AddDebuginfodConnectorFromEnvironment(this IServiceCollection services)
|
||||
{
|
||||
return services.AddDebuginfodConnector(opts =>
|
||||
{
|
||||
var urls = Environment.GetEnvironmentVariable("DEBUGINFOD_URLS");
|
||||
if (!string.IsNullOrWhiteSpace(urls))
|
||||
{
|
||||
var urlList = urls.Split([' ', ','], StringSplitOptions.RemoveEmptyEntries);
|
||||
if (urlList.Length > 0 && Uri.TryCreate(urlList[0], UriKind.Absolute, out var primary))
|
||||
{
|
||||
opts.BaseUrl = primary;
|
||||
}
|
||||
for (var i = 1; i < urlList.Length; i++)
|
||||
{
|
||||
if (Uri.TryCreate(urlList[i], UriKind.Absolute, out var additional))
|
||||
{
|
||||
opts.AdditionalUrls.Add(additional);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var cache = Environment.GetEnvironmentVariable("DEBUGINFOD_CACHE");
|
||||
if (!string.IsNullOrWhiteSpace(cache))
|
||||
{
|
||||
opts.CacheDirectory = cache;
|
||||
}
|
||||
|
||||
var timeout = Environment.GetEnvironmentVariable("DEBUGINFOD_TIMEOUT");
|
||||
if (!string.IsNullOrWhiteSpace(timeout) && int.TryParse(timeout, out var timeoutSeconds))
|
||||
{
|
||||
opts.TimeoutSeconds = timeoutSeconds;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Debuginfod.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Diagnostics and metrics for the debuginfod connector.
|
||||
/// </summary>
|
||||
public sealed class DebuginfodDiagnostics
|
||||
{
|
||||
private readonly Counter<long> _fetchSuccessCounter;
|
||||
private readonly Counter<long> _fetchNotFoundCounter;
|
||||
private readonly Counter<long> _fetchErrorCounter;
|
||||
private readonly Counter<long> _parseSuccessCounter;
|
||||
private readonly Counter<long> _parseErrorCounter;
|
||||
private readonly Counter<long> _mapSuccessCounter;
|
||||
private readonly Counter<long> _mapErrorCounter;
|
||||
private readonly Counter<long> _mapAocViolationCounter;
|
||||
private readonly Histogram<long> _symbolCountHistogram;
|
||||
|
||||
public DebuginfodDiagnostics(IMeterFactory meterFactory)
|
||||
{
|
||||
var meter = meterFactory.Create("StellaOps.BinaryIndex.GroundTruth.Debuginfod");
|
||||
|
||||
_fetchSuccessCounter = meter.CreateCounter<long>(
|
||||
"groundtruth.debuginfod.fetch.success",
|
||||
unit: "{documents}",
|
||||
description: "Number of successful debuginfod fetches");
|
||||
|
||||
_fetchNotFoundCounter = meter.CreateCounter<long>(
|
||||
"groundtruth.debuginfod.fetch.not_found",
|
||||
unit: "{documents}",
|
||||
description: "Number of debuginfod fetches that returned 404");
|
||||
|
||||
_fetchErrorCounter = meter.CreateCounter<long>(
|
||||
"groundtruth.debuginfod.fetch.error",
|
||||
unit: "{documents}",
|
||||
description: "Number of failed debuginfod fetches");
|
||||
|
||||
_parseSuccessCounter = meter.CreateCounter<long>(
|
||||
"groundtruth.debuginfod.parse.success",
|
||||
unit: "{documents}",
|
||||
description: "Number of successful DWARF parses");
|
||||
|
||||
_parseErrorCounter = meter.CreateCounter<long>(
|
||||
"groundtruth.debuginfod.parse.error",
|
||||
unit: "{documents}",
|
||||
description: "Number of failed DWARF parses");
|
||||
|
||||
_mapSuccessCounter = meter.CreateCounter<long>(
|
||||
"groundtruth.debuginfod.map.success",
|
||||
unit: "{observations}",
|
||||
description: "Number of successful observation mappings");
|
||||
|
||||
_mapErrorCounter = meter.CreateCounter<long>(
|
||||
"groundtruth.debuginfod.map.error",
|
||||
unit: "{observations}",
|
||||
description: "Number of failed observation mappings");
|
||||
|
||||
_mapAocViolationCounter = meter.CreateCounter<long>(
|
||||
"groundtruth.debuginfod.map.aoc_violation",
|
||||
unit: "{observations}",
|
||||
description: "Number of AOC violations during mapping");
|
||||
|
||||
_symbolCountHistogram = meter.CreateHistogram<long>(
|
||||
"groundtruth.debuginfod.symbols_per_binary",
|
||||
unit: "{symbols}",
|
||||
description: "Distribution of symbol counts per binary");
|
||||
}
|
||||
|
||||
public void RecordFetchSuccess() => _fetchSuccessCounter.Add(1);
|
||||
public void RecordFetchNotFound() => _fetchNotFoundCounter.Add(1);
|
||||
public void RecordFetchError() => _fetchErrorCounter.Add(1);
|
||||
|
||||
public void RecordParseSuccess(int symbolCount)
|
||||
{
|
||||
_parseSuccessCounter.Add(1);
|
||||
_symbolCountHistogram.Record(symbolCount);
|
||||
}
|
||||
|
||||
public void RecordParseError() => _parseErrorCounter.Add(1);
|
||||
|
||||
public void RecordMapSuccess(int symbolCount)
|
||||
{
|
||||
_mapSuccessCounter.Add(1);
|
||||
}
|
||||
|
||||
public void RecordMapError() => _mapErrorCounter.Add(1);
|
||||
public void RecordMapAocViolation() => _mapAocViolationCounter.Add(1);
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Abstractions;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Debuginfod.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// ELF/DWARF parser implementation.
|
||||
///
|
||||
/// NOTE: LibObjectFile 1.0.0 has significant API changes from 0.x.
|
||||
/// This is a stub implementation pending API migration.
|
||||
/// See: https://github.com/xoofx/LibObjectFile/releases/tag/1.0.0
|
||||
/// </summary>
|
||||
public sealed class ElfDwarfParser : IDwarfParser
|
||||
{
|
||||
private readonly ILogger<ElfDwarfParser> _logger;
|
||||
|
||||
public ElfDwarfParser(ILogger<ElfDwarfParser> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<IReadOnlyList<ObservedSymbol>> ParseSymbolsAsync(Guid payloadId, CancellationToken ct = default)
|
||||
{
|
||||
throw new NotImplementedException(
|
||||
"Parsing from payload ID requires blob storage integration. Use stream overload instead.");
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<IReadOnlyList<ObservedSymbol>> ParseSymbolsAsync(Stream stream, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(stream);
|
||||
|
||||
_logger.LogWarning(
|
||||
"ElfDwarfParser is a stub - LibObjectFile 1.0.0 API migration pending. " +
|
||||
"Returning empty symbol list.");
|
||||
|
||||
return Task.FromResult<IReadOnlyList<ObservedSymbol>>(Array.Empty<ObservedSymbol>());
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<string?> ExtractBuildIdAsync(Stream stream, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(stream);
|
||||
|
||||
_logger.LogWarning(
|
||||
"ElfDwarfParser.ExtractBuildIdAsync is a stub - LibObjectFile 1.0.0 API migration pending.");
|
||||
|
||||
// Try to read build-id using simple heuristics
|
||||
try
|
||||
{
|
||||
// Look for .note.gnu.build-id section marker
|
||||
using var reader = new BinaryReader(stream, System.Text.Encoding.UTF8, leaveOpen: true);
|
||||
|
||||
// Reset to start
|
||||
stream.Position = 0;
|
||||
|
||||
// Read ELF header to verify it's an ELF file
|
||||
var magic = reader.ReadBytes(4);
|
||||
if (magic.Length < 4 || magic[0] != 0x7f || magic[1] != 'E' || magic[2] != 'L' || magic[3] != 'F')
|
||||
{
|
||||
_logger.LogDebug("Not an ELF file");
|
||||
return Task.FromResult<string?>(null);
|
||||
}
|
||||
|
||||
_logger.LogDebug("ELF file detected, but full parsing requires LibObjectFile API migration");
|
||||
return Task.FromResult<string?>(null);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Failed to read ELF header");
|
||||
return Task.FromResult<string?>(null);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<ObservedBuildMetadata?> ExtractBuildMetadataAsync(Stream stream, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(stream);
|
||||
|
||||
_logger.LogWarning(
|
||||
"ElfDwarfParser.ExtractBuildMetadataAsync is a stub - LibObjectFile 1.0.0 API migration pending.");
|
||||
|
||||
return Task.FromResult<ObservedBuildMetadata?>(null);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
using StellaOps.BinaryIndex.GroundTruth.Abstractions;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Debuginfod.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Interface for parsing DWARF debug information from ELF binaries.
|
||||
/// </summary>
|
||||
public interface IDwarfParser
|
||||
{
|
||||
/// <summary>
|
||||
/// Parse symbols from a stored payload.
|
||||
/// </summary>
|
||||
/// <param name="payloadId">Blob storage ID for the ELF binary.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>List of parsed symbols.</returns>
|
||||
Task<IReadOnlyList<ObservedSymbol>> ParseSymbolsAsync(Guid payloadId, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Parse symbols from a stream.
|
||||
/// </summary>
|
||||
/// <param name="stream">ELF binary stream.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>List of parsed symbols.</returns>
|
||||
Task<IReadOnlyList<ObservedSymbol>> ParseSymbolsAsync(Stream stream, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Extract build ID from an ELF binary.
|
||||
/// </summary>
|
||||
/// <param name="stream">ELF binary stream.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>Build ID as hex string, or null if not found.</returns>
|
||||
Task<string?> ExtractBuildIdAsync(Stream stream, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Extract build metadata from DWARF debug info.
|
||||
/// </summary>
|
||||
/// <param name="stream">ELF binary stream.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>Build metadata.</returns>
|
||||
Task<ObservedBuildMetadata?> ExtractBuildMetadataAsync(Stream stream, CancellationToken ct = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stub implementation of DWARF parser for initial development.
|
||||
/// Production implementation would use Gimli (Rust) or libdw bindings.
|
||||
/// </summary>
|
||||
public sealed class StubDwarfParser : IDwarfParser
|
||||
{
|
||||
/// <inheritdoc/>
|
||||
public Task<IReadOnlyList<ObservedSymbol>> ParseSymbolsAsync(Guid payloadId, CancellationToken ct = default)
|
||||
{
|
||||
// Stub: Return empty list
|
||||
// Production: Load from blob storage and parse
|
||||
return Task.FromResult<IReadOnlyList<ObservedSymbol>>([]);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<IReadOnlyList<ObservedSymbol>> ParseSymbolsAsync(Stream stream, CancellationToken ct = default)
|
||||
{
|
||||
// Stub: Return empty list
|
||||
// Production: Parse ELF + DWARF sections
|
||||
return Task.FromResult<IReadOnlyList<ObservedSymbol>>([]);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<string?> ExtractBuildIdAsync(Stream stream, CancellationToken ct = default)
|
||||
{
|
||||
// Stub: Return null
|
||||
// Production: Read .note.gnu.build-id section
|
||||
return Task.FromResult<string?>(null);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<ObservedBuildMetadata?> ExtractBuildMetadataAsync(Stream stream, CancellationToken ct = default)
|
||||
{
|
||||
// Stub: Return null
|
||||
// Production: Parse DW_AT_producer and other DWARF attributes
|
||||
return Task.FromResult<ObservedBuildMetadata?>(null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<NoWarn>$(NoWarn);NU1603</NoWarn>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<GenerateDocumentationFile>true</GenerateDocumentationFile>
|
||||
<Description>Debuginfod symbol source connector for Fedora/RHEL debuginfod services</Description>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" />
|
||||
<PackageReference Include="LibObjectFile" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.BinaryIndex.GroundTruth.Abstractions\StellaOps.BinaryIndex.GroundTruth.Abstractions.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
Reference in New Issue
Block a user