Add determinism tests for verdict artifact generation and update SHA256 sums script

- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering. - Created helper methods for generating sample verdict inputs and computing canonical hashes. - Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics. - Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
2025-12-24 02:17:34 +02:00
parent e59921374e
commit 7503c19b8f
390 changed files with 37389 additions and 5380 deletions
--- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Cisco/CiscoCsafParserSnapshotTests.cs
+++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Cisco/CiscoCsafParserSnapshotTests.cs
@@ -0,0 +1,215 @@
+// -----------------------------------------------------------------------------
+// CiscoCsafParserSnapshotTests.cs
+// Sprint: SPRINT_5100_0007_0005
+// Task: CONN-FIX-005
+// Description: Cisco CSAF parser snapshot tests for fixture validation
+// -----------------------------------------------------------------------------
+
+using System.Text.Json;
+using FluentAssertions;
+using StellaOps.Concelier.Connector.Vndr.Cisco.Internal;
+using Xunit;
+
+namespace StellaOps.Concelier.Connector.Vndr.Cisco.Tests.Cisco;
+
+/// <summary>
+/// Parser snapshot tests for the Cisco CSAF connector.
+/// Verifies that raw CSAF JSON fixtures parse to expected CiscoCsafData output.
+/// </summary>
+public sealed class CiscoCsafParserSnapshotTests
+{
+    private static readonly string BaseDirectory = AppContext.BaseDirectory;
+    private static readonly string FixturesDirectory = Path.Combine(BaseDirectory, "Cisco", "Fixtures");
+    private static readonly string ExpectedDirectory = Path.Combine(BaseDirectory, "Expected");
+    private static readonly JsonSerializerOptions SerializerOptions = new()
+    {
+        WriteIndented = true,
+        PropertyNamingPolicy = JsonNamingPolicy.CamelCase
+    };
+
+    [Fact]
+    [Trait("Lane", "Unit")]
+    [Trait("Category", "Snapshot")]
+    public void ParseTypicalCsaf_ProducesExpectedCsafData()
+    {
+        // Arrange
+        var csafJson = ReadFixture("cisco-csaf-typical.json");
+        var expectedJson = ReadExpected("cisco-csaf-typical.csafdata.json");
+
+        // Act
+        var csafData = CiscoCsafParser.Parse(csafJson);
+        var actualJson = SerializeCsafData(csafData);
+
+        // Assert
+        actualJson.Should().Be(expectedJson,
+            "typical CSAF fixture should produce expected CsafData output");
+    }
+
+    [Fact]
+    [Trait("Lane", "Unit")]
+    [Trait("Category", "Snapshot")]
+    public void ParseMultiCveCsaf_ProducesExpectedCsafData()
+    {
+        // Arrange
+        var csafJson = ReadFixture("cisco-csaf-edge-multi-cve.json");
+        var expectedJson = ReadExpected("cisco-csaf-edge-multi-cve.csafdata.json");
+
+        // Act
+        var csafData = CiscoCsafParser.Parse(csafJson);
+        var actualJson = SerializeCsafData(csafData);
+
+        // Assert
+        actualJson.Should().Be(expectedJson,
+            "multi-CVE CSAF fixture should produce expected CsafData output");
+    }
+
+    [Fact]
+    [Trait("Lane", "Unit")]
+    [Trait("Category", "Determinism")]
+    public void ParseTypicalCsaf_IsDeterministic()
+    {
+        // Arrange
+        var csafJson = ReadFixture("cisco-csaf-typical.json");
+
+        // Act
+        var results = new List<string>();
+        for (int i = 0; i < 3; i++)
+        {
+            var csafData = CiscoCsafParser.Parse(csafJson);
+            results.Add(SerializeCsafData(csafData));
+        }
+
+        // Assert
+        results.Distinct().Should().HaveCount(1,
+            "parsing CSAF multiple times should produce identical output");
+    }
+
+    [Fact]
+    [Trait("Lane", "Unit")]
+    [Trait("Category", "Resilience")]
+    public void ParseMissingTracking_HandlesGracefully()
+    {
+        // Arrange
+        var csafJson = ReadFixture("cisco-csaf-error-missing-tracking.json");
+
+        // Act
+        var csafData = CiscoCsafParser.Parse(csafJson);
+
+        // Assert - parser should not throw, just return empty/default data
+        csafData.Should().NotBeNull();
+    }
+
+    [Fact]
+    [Trait("Lane", "Unit")]
+    [Trait("Category", "Resilience")]
+    public void ParseInvalidJson_ThrowsJsonException()
+    {
+        // Arrange
+        var invalidJson = ReadFixture("cisco-csaf-error-invalid-json.json");
+
+        // Act & Assert
+        var act = () => CiscoCsafParser.Parse(invalidJson);
+        act.Should().Throw<JsonException>("invalid JSON should throw JsonException");
+    }
+
+    [Fact]
+    [Trait("Lane", "Unit")]
+    [Trait("Category", "Parser")]
+    public void CsafParser_ExtractsProducts()
+    {
+        // Arrange
+        var csafJson = ReadFixture("cisco-csaf-typical.json");
+
+        // Act
+        var csafData = CiscoCsafParser.Parse(csafJson);
+
+        // Assert
+        csafData.Products.Should().NotBeEmpty("CSAF should contain product definitions");
+        csafData.Products.Should().ContainKey("CSCWA12345");
+    }
+
+    [Fact]
+    [Trait("Lane", "Unit")]
+    [Trait("Category", "Parser")]
+    public void CsafParser_ExtractsProductStatuses()
+    {
+        // Arrange
+        var csafJson = ReadFixture("cisco-csaf-typical.json");
+
+        // Act
+        var csafData = CiscoCsafParser.Parse(csafJson);
+
+        // Assert
+        csafData.ProductStatuses.Should().NotBeEmpty("CSAF should contain product status mappings");
+        csafData.ProductStatuses.Should().ContainKey("CSCWA12345");
+        csafData.ProductStatuses["CSCWA12345"].Should().Contain("known_affected");
+    }
+
+    [Fact]
+    [Trait("Lane", "Unit")]
+    [Trait("Category", "Parser")]
+    public void CsafParser_ExtractsMultipleProducts()
+    {
+        // Arrange
+        var csafJson = ReadFixture("cisco-csaf-edge-multi-cve.json");
+
+        // Act
+        var csafData = CiscoCsafParser.Parse(csafJson);
+
+        // Assert
+        csafData.Products.Should().HaveCountGreaterThanOrEqualTo(3, "multi-CVE CSAF should contain multiple products");
+        csafData.ProductStatuses.Should().HaveCountGreaterThanOrEqualTo(3, "multi-CVE CSAF should have status for each product");
+    }
+
+    [Fact]
+    [Trait("Lane", "Unit")]
+    [Trait("Category", "Parser")]
+    public void CsafParser_EmptyContent_ReturnsEmptyData()
+    {
+        // Arrange & Act
+        var csafData = CiscoCsafParser.Parse(string.Empty);
+
+        // Assert
+        csafData.Products.Should().BeEmpty();
+        csafData.ProductStatuses.Should().BeEmpty();
+    }
+
+    private static string SerializeCsafData(CiscoCsafData csafData)
+    {
+        var result = new
+        {
+            products = csafData.Products
+                .OrderBy(p => p.Key, StringComparer.OrdinalIgnoreCase)
+                .Select(p => new
+                {
+                    productId = p.Key,
+                    name = p.Value.Name
+                })
+                .ToList(),
+            productStatuses = csafData.ProductStatuses
+                .OrderBy(s => s.Key, StringComparer.OrdinalIgnoreCase)
+                .Select(s => new
+                {
+                    productId = s.Key,
+                    statuses = s.Value.OrderBy(x => x, StringComparer.OrdinalIgnoreCase).ToList()
+                })
+                .ToList()
+        };
+
+        return JsonSerializer.Serialize(result, SerializerOptions)
+            .Replace("\r\n", "\n")
+            .TrimEnd();
+    }
+
+    private static string ReadFixture(string fileName)
+    {
+        var path = Path.Combine(FixturesDirectory, fileName);
+        return File.ReadAllText(path);
+    }
+
+    private static string ReadExpected(string fileName)
+    {
+        var path = Path.Combine(ExpectedDirectory, fileName);
+        return File.ReadAllText(path).Replace("\r\n", "\n").TrimEnd();
+    }
+}
--- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Cisco/Fixtures/cisco-csaf-edge-multi-cve.json
+++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Cisco/Fixtures/cisco-csaf-edge-multi-cve.json
@@ -0,0 +1,93 @@
+{
+  "document": {
+    "aggregate_severity": {
+      "text": "Critical"
+    },
+    "lang": "en",
+    "notes": [
+      {
+        "category": "summary",
+        "text": "Multiple vulnerabilities in Cisco Unified Communications Manager affecting multiple products and CVEs."
+      }
+    ],
+    "references": [
+      {
+        "category": "self",
+        "summary": "Cisco Security Advisory",
+        "url": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-multi-2025"
+      }
+    ],
+    "title": "Cisco Unified Communications Manager Multiple Vulnerabilities",
+    "tracking": {
+      "id": "cisco-sa-multi-2025",
+      "initial_release_date": "2025-11-01T00:00:00+00:00",
+      "current_release_date": "2025-11-15T00:00:00+00:00"
+    }
+  },
+  "product_tree": {
+    "full_product_names": [
+      {
+        "name": "Cisco Unified Communications Manager 14.0",
+        "product_id": "CUCM-14.0"
+      },
+      {
+        "name": "Cisco Unified Communications Manager IM and Presence 14.0",
+        "product_id": "CUCM-IMP-14.0"
+      },
+      {
+        "name": "Cisco Unity Connection 14.0",
+        "product_id": "CUC-14.0"
+      }
+    ]
+  },
+  "vulnerabilities": [
+    {
+      "cve": "CVE-2025-1001",
+      "scores": [
+        {
+          "cvss_v3": {
+            "baseScore": 9.8,
+            "baseSeverity": "CRITICAL",
+            "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H",
+            "version": "3.1"
+          }
+        }
+      ],
+      "product_status": {
+        "known_affected": ["CUCM-14.0", "CUCM-IMP-14.0"]
+      }
+    },
+    {
+      "cve": "CVE-2025-1002",
+      "scores": [
+        {
+          "cvss_v3": {
+            "baseScore": 7.5,
+            "baseSeverity": "HIGH",
+            "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N",
+            "version": "3.1"
+          }
+        }
+      ],
+      "product_status": {
+        "known_affected": ["CUCM-14.0", "CUC-14.0"]
+      }
+    },
+    {
+      "cve": "CVE-2025-1003",
+      "scores": [
+        {
+          "cvss_v3": {
+            "baseScore": 5.3,
+            "baseSeverity": "MEDIUM",
+            "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N",
+            "version": "3.1"
+          }
+        }
+      ],
+      "product_status": {
+        "known_affected": ["CUCM-IMP-14.0", "CUC-14.0"]
+      }
+    }
+  ]
+}
--- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Cisco/Fixtures/cisco-csaf-error-invalid-json.json
+++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Cisco/Fixtures/cisco-csaf-error-invalid-json.json
@@ -0,0 +1,9 @@
+{
+  "document": {
+    "aggregate_severity": {
+      "text": "High"
+    },
+    "lang": "en",
+    "title": "Invalid JSON - unclosed brace",
+    "tracking": {
+      "id": "cisco-sa-invalid"
--- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Cisco/Fixtures/cisco-csaf-error-missing-tracking.json
+++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Cisco/Fixtures/cisco-csaf-error-missing-tracking.json
@@ -0,0 +1,14 @@
+{
+  "document": {
+    "aggregate_severity": {
+      "text": "High"
+    },
+    "lang": "en",
+    "title": "Malformed CSAF - Missing tracking"
+  },
+  "vulnerabilities": [
+    {
+      "cve": "CVE-2025-9999"
+    }
+  ]
+}
--- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Cisco/Fixtures/cisco-csaf-typical.json
+++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Cisco/Fixtures/cisco-csaf-typical.json
@@ -0,0 +1,62 @@
+{
+  "document": {
+    "aggregate_severity": {
+      "text": "High"
+    },
+    "lang": "en",
+    "notes": [
+      {
+        "category": "summary",
+        "text": "A vulnerability in the web UI of Cisco IOS XE Software could allow an authenticated remote attacker to execute arbitrary commands."
+      }
+    ],
+    "references": [
+      {
+        "category": "self",
+        "summary": "Cisco Security Advisory",
+        "url": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-test-2025"
+      }
+    ],
+    "title": "Cisco IOS XE Software Web UI Command Injection Vulnerability",
+    "tracking": {
+      "id": "cisco-sa-test-2025",
+      "initial_release_date": "2025-10-01T00:00:00+00:00",
+      "current_release_date": "2025-10-02T00:00:00+00:00"
+    }
+  },
+  "product_tree": {
+    "full_product_names": [
+      {
+        "name": "Cisco IOS XE Software 17.6.1",
+        "product_id": "CSCWA12345"
+      }
+    ]
+  },
+  "vulnerabilities": [
+    {
+      "cve": "CVE-2025-0001",
+      "references": [
+        {
+          "category": "external",
+          "summary": "CVE record",
+          "url": "https://www.cve.org/CVERecord?id=CVE-2025-0001"
+        }
+      ],
+      "scores": [
+        {
+          "cvss_v3": {
+            "baseScore": 8.8,
+            "baseSeverity": "HIGH",
+            "vectorString": "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H",
+            "version": "3.1"
+          }
+        }
+      ],
+      "product_status": {
+        "known_affected": [
+          "CSCWA12345"
+        ]
+      }
+    }
+  ]
+}
--- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Expected/cisco-csaf-edge-multi-cve.csafdata.json
+++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Expected/cisco-csaf-edge-multi-cve.csafdata.json
@@ -0,0 +1,36 @@
+{
+  "products": [
+    {
+      "productId": "CUC-14.0",
+      "name": "Cisco Unity Connection 14.0"
+    },
+    {
+      "productId": "CUCM-14.0",
+      "name": "Cisco Unified Communications Manager 14.0"
+    },
+    {
+      "productId": "CUCM-IMP-14.0",
+      "name": "Cisco Unified Communications Manager IM and Presence 14.0"
+    }
+  ],
+  "productStatuses": [
+    {
+      "productId": "CUC-14.0",
+      "statuses": [
+        "known_affected"
+      ]
+    },
+    {
+      "productId": "CUCM-14.0",
+      "statuses": [
+        "known_affected"
+      ]
+    },
+    {
+      "productId": "CUCM-IMP-14.0",
+      "statuses": [
+        "known_affected"
+      ]
+    }
+  ]
+}
--- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Expected/cisco-csaf-typical.canonical.json
+++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Expected/cisco-csaf-typical.canonical.json
@@ -0,0 +1,117 @@
+{
+  "advisoryKey": "cisco-sa-test-2025",
+  "affectedPackages": [
+    {
+      "type": "vendor",
+      "identifier": "Cisco IOS XE Software 17.6.1",
+      "platform": null,
+      "versionRanges": [
+        {
+          "fixedVersion": null,
+          "introducedVersion": null,
+          "lastAffectedVersion": null,
+          "primitives": {
+            "evr": null,
+            "hasVendorExtensions": true,
+            "nevra": null,
+            "semVer": null,
+            "vendorExtensions": {
+              "productId": "CSCWA12345"
+            }
+          },
+          "provenance": {
+            "source": "vndr.cisco",
+            "kind": "csaf",
+            "value": "CSCWA12345",
+            "decisionReason": null,
+            "recordedAt": "2025-10-02T00:00:00+00:00",
+            "fieldMask": ["affectedpackages[].versionranges[]"]
+          },
+          "rangeExpression": "CSCWA12345",
+          "rangeKind": "vendor"
+        }
+      ],
+      "normalizedVersions": [],
+      "statuses": [
+        {
+          "provenance": {
+            "source": "vndr.cisco",
+            "kind": "csaf-status",
+            "value": "known_affected",
+            "decisionReason": null,
+            "recordedAt": "2025-10-02T00:00:00+00:00",
+            "fieldMask": ["affectedpackages[].statuses[]"]
+          },
+          "status": "affected"
+        }
+      ],
+      "provenance": [
+        {
+          "source": "vndr.cisco",
+          "kind": "csaf",
+          "value": "CSCWA12345",
+          "decisionReason": null,
+          "recordedAt": "2025-10-02T00:00:00+00:00",
+          "fieldMask": ["affectedpackages[]"]
+        }
+      ]
+    }
+  ],
+  "aliases": [
+    "cisco-sa-test-2025",
+    "CVE-2025-0001"
+  ],
+  "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H",
+  "credits": [],
+  "cvssMetrics": [
+    {
+      "baseScore": 8.8,
+      "baseSeverity": "high",
+      "provenance": {
+        "source": "vndr.cisco",
+        "kind": "cvss",
+        "value": "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H",
+        "decisionReason": null,
+        "recordedAt": "2025-10-02T00:00:00+00:00",
+        "fieldMask": ["cvssmetrics[]"]
+      },
+      "vector": "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H",
+      "version": "3.1"
+    }
+  ],
+  "cwes": [],
+  "description": "A vulnerability in the web UI of Cisco IOS XE Software could allow an authenticated remote attacker to execute arbitrary commands.",
+  "exploitKnown": false,
+  "language": "en",
+  "modified": "2025-10-02T00:00:00+00:00",
+  "provenance": [
+    {
+      "source": "vndr.cisco",
+      "kind": "csaf",
+      "value": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-test-2025",
+      "decisionReason": null,
+      "recordedAt": "2025-10-02T00:00:00+00:00",
+      "fieldMask": ["advisory"]
+    }
+  ],
+  "published": "2025-10-01T00:00:00+00:00",
+  "references": [
+    {
+      "kind": "self",
+      "provenance": {
+        "source": "vndr.cisco",
+        "kind": "reference",
+        "value": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-test-2025",
+        "decisionReason": null,
+        "recordedAt": "2025-10-02T00:00:00+00:00",
+        "fieldMask": ["references[]"]
+      },
+      "sourceTag": "Cisco Security Advisory",
+      "summary": null,
+      "url": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-test-2025"
+    }
+  ],
+  "severity": "high",
+  "summary": "A vulnerability in the web UI of Cisco IOS XE Software could allow an authenticated remote attacker to execute arbitrary commands.",
+  "title": "Cisco IOS XE Software Web UI Command Injection Vulnerability"
+}
--- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Expected/cisco-csaf-typical.csafdata.json
+++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/Expected/cisco-csaf-typical.csafdata.json
@@ -0,0 +1,16 @@
+{
+  "products": [
+    {
+      "productId": "CSCWA12345",
+      "name": "Cisco IOS XE Software 17.6.1"
+    }
+  ],
+  "productStatuses": [
+    {
+      "productId": "CSCWA12345",
+      "statuses": [
+        "known_affected"
+      ]
+    }
+  ]
+}
--- a/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests.csproj
+++ b/src/Concelier/__Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests.csproj
@@ -10,9 +10,15 @@
    <ProjectReference Include="../../__Libraries/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
    <ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
    <ProjectReference Include="../../__Libraries/StellaOps.Concelier.Connector.Vndr.Cisco/StellaOps.Concelier.Connector.Vndr.Cisco.csproj" />
+    <ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
  </ItemGroup>

  <ItemGroup>
    <PackageReference Include="FluentAssertions" Version="6.12.0" />
  </ItemGroup>
+
+  <ItemGroup>
+    <None Include="Cisco/Fixtures/*.json" CopyToOutputDirectory="Always" />
+    <None Include="Expected/*.json" CopyToOutputDirectory="Always" />
+  </ItemGroup>
 </Project>