stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

devops folders consolidate

Browse Source
This commit is contained in:
master
2026-01-25 23:27:41 +02:00
parent 6e687b523a
commit a50bbb38ef
334 changed files with 35079 additions and 5569 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
devops/tools/ops-scripts/check-advisory-raw-duplicates.js
View File

@@ -1,77 +0,0 @@
/**
* Aggregation helper that surfaces advisory_raw duplicate candidates prior to enabling the
* idempotency unique index. Intended for staging/offline snapshots.
*
* Usage:
* mongo concelier ops/devops/scripts/check-advisory-raw-duplicates.js
*
* Environment variables:
* LIMIT - optional cap on number of duplicate groups to print (default 50).
*/
(function () {
function toInt(value, fallback) {
var parsed = parseInt(value, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
var limit = typeof LIMIT !== "undefined" ? toInt(LIMIT, 50) : 50;
var database = db.getName ? db.getSiblingDB(db.getName()) : db;
if (!database) {
throw new Error("Unable to resolve database handle");
}
print("");
print("== advisory_raw duplicate audit ==");
print("Database: " + database.getName());
print("Limit : " + limit);
print("");
var pipeline = [
{
$group: {
_id: {
vendor: "$source.vendor",
upstreamId: "$upstream.upstream_id",
contentHash: "$upstream.content_hash",
tenant: "$tenant"
},
ids: { $addToSet: "$_id" },
count: { $sum: 1 }
}
},
{ $match: { count: { $gt: 1 } } },
{
$project: {
_id: 0,
vendor: "$_id.vendor",
upstreamId: "$_id.upstreamId",
contentHash: "$_id.contentHash",
tenant: "$_id.tenant",
count: 1,
ids: 1
}
},
{ $sort: { count: -1, vendor: 1, upstreamId: 1 } },
{ $limit: limit }
];
var cursor = database.getCollection("advisory_raw").aggregate(pipeline, { allowDiskUse: true });
var any = false;
while (cursor.hasNext()) {
var doc = cursor.next();
any = true;
print("---");
print("vendor : " + doc.vendor);
print("upstream_id : " + doc.upstreamId);
print("tenant : " + doc.tenant);
print("content_hash: " + doc.contentHash);
print("count : " + doc.count);
print("ids : " + doc.ids.join(", "));
}
if (!any) {
print("No duplicate advisory_raw documents detected.");
}
print("");
})();

46
devops/tools/ops-scripts/check-advisory-raw-duplicates.sql Normal file
View File

@@ -0,0 +1,46 @@
-- Advisory raw duplicate detection query
-- Surfaces advisory_raw duplicate candidates prior to enabling the idempotency unique index.
-- Intended for staging/offline snapshots.
--
-- Usage:
-- psql -d concelier -f ops/devops/tools/ops-scripts/check-advisory-raw-duplicates.sql
--
-- Environment variables:
-- LIMIT - optional cap on number of duplicate groups to print (default 50).
\echo '== advisory_raw duplicate audit =='
\conninfo
WITH duplicates AS (
SELECT
source_vendor,
upstream_id,
content_hash,
tenant,
COUNT(*) as count,
ARRAY_AGG(id) as ids
FROM advisory_raw
GROUP BY source_vendor, upstream_id, content_hash, tenant
HAVING COUNT(*) > 1
ORDER BY COUNT(*) DESC, source_vendor, upstream_id
LIMIT COALESCE(NULLIF(:'LIMIT', '')::INT, 50)
)
SELECT
'vendor: ' || source_vendor || E'\n' ||
'upstream_id: ' || upstream_id || E'\n' ||
'tenant: ' || COALESCE(tenant, 'NULL') || E'\n' ||
'content_hash: ' || content_hash || E'\n' ||
'count: ' || count || E'\n' ||
'ids: ' || ARRAY_TO_STRING(ids, ', ') AS duplicate_info
FROM duplicates;
SELECT CASE WHEN COUNT(*) = 0
THEN 'No duplicate advisory_raw documents detected.'
ELSE 'Found ' || COUNT(*) || ' duplicate groups.'
END as status
FROM (
SELECT 1 FROM advisory_raw
GROUP BY source_vendor, upstream_id, content_hash, tenant
HAVING COUNT(*) > 1
LIMIT 1
) t;

100
devops/tools/ops-scripts/rollback-lnm-backfill.js
View File

@@ -1,100 +0,0 @@
/**
* Rollback script for LNM-21-102-DEV legacy advisory backfill migration.
* Removes backfilled observations and linksets by querying the backfill_marker field,
* then clears the tombstone markers from advisory_raw.
*
* Usage:
* mongo concelier ops/devops/scripts/rollback-lnm-backfill.js
*
* Environment variables:
* DRY_RUN - if set to "1", only reports what would be deleted without making changes.
* BATCH_SIZE - optional batch size for deletions (default 500).
*
* After running this script, delete the migration record:
* db.schema_migrations.deleteOne({ _id: "20251127_lnm_legacy_backfill" })
*
* Then restart the Concelier service.
*/
(function () {
var BACKFILL_MARKER = "lnm_21_102_dev";
function toInt(value, fallback) {
var parsed = parseInt(value, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
function toBool(value) {
return value === "1" || value === "true" || value === true;
}
var dryRun = typeof DRY_RUN !== "undefined" ? toBool(DRY_RUN) : false;
var batchSize = typeof BATCH_SIZE !== "undefined" ? toInt(BATCH_SIZE, 500) : 500;
var database = db.getName ? db.getSiblingDB(db.getName()) : db;
if (!database) {
throw new Error("Unable to resolve database handle");
}
print("");
print("== LNM-21-102-DEV Backfill Rollback ==");
print("Database : " + database.getName());
print("Dry Run : " + dryRun);
print("Batch Size: " + batchSize);
print("");
// Step 1: Count and delete backfilled observations
var observationsCollection = database.getCollection("advisory_observations");
var observationsFilter = { backfill_marker: BACKFILL_MARKER };
var observationsCount = observationsCollection.countDocuments(observationsFilter);
print("Found " + observationsCount + " backfilled observations to remove.");
if (!dryRun && observationsCount > 0) {
var obsResult = observationsCollection.deleteMany(observationsFilter);
print("Deleted " + obsResult.deletedCount + " observations.");
}
// Step 2: Count and delete backfilled linksets
var linksetsCollection = database.getCollection("advisory_linksets");
var linksetsFilter = { backfill_marker: BACKFILL_MARKER };
var linksetsCount = linksetsCollection.countDocuments(linksetsFilter);
print("Found " + linksetsCount + " backfilled linksets to remove.");
if (!dryRun && linksetsCount > 0) {
var linkResult = linksetsCollection.deleteMany(linksetsFilter);
print("Deleted " + linkResult.deletedCount + " linksets.");
}
// Step 3: Clear tombstone markers from advisory_raw
var rawCollection = database.getCollection("advisory_raw");
var rawFilter = { backfill_marker: BACKFILL_MARKER };
var rawCount = rawCollection.countDocuments(rawFilter);
print("Found " + rawCount + " advisory_raw documents with tombstone markers to clear.");
if (!dryRun && rawCount > 0) {
var rawResult = rawCollection.updateMany(rawFilter, { $unset: { backfill_marker: "" } });
print("Cleared tombstone markers from " + rawResult.modifiedCount + " advisory_raw documents.");
}
// Step 4: Summary
print("");
print("== Rollback Summary ==");
if (dryRun) {
print("DRY RUN - No changes were made.");
print("Would delete " + observationsCount + " observations.");
print("Would delete " + linksetsCount + " linksets.");
print("Would clear " + rawCount + " tombstone markers.");
} else {
print("Observations deleted: " + observationsCount);
print("Linksets deleted : " + linksetsCount);
print("Tombstones cleared : " + rawCount);
}
print("");
print("Next steps:");
print("1. Delete the migration record:");
print(' db.schema_migrations.deleteOne({ _id: "20251127_lnm_legacy_backfill" })');
print("2. Restart the Concelier service.");
print("");
})();

60
devops/tools/ops-scripts/rollback-lnm-backfill.sql Normal file
View File

@@ -0,0 +1,60 @@
-- Rollback script for LNM-21-102-DEV legacy advisory backfill migration.
-- Removes backfilled observations and linksets by querying the backfill_marker field,
-- then clears the tombstone markers from advisory_raw.
--
-- Usage:
-- psql -d concelier -f ops/devops/tools/ops-scripts/rollback-lnm-backfill.sql
--
-- Environment variables:
-- DRY_RUN - if set to "1", only reports what would be deleted without making changes.
--
-- After running this script, delete the migration record:
-- DELETE FROM schema_migrations WHERE id = '20251127_lnm_legacy_backfill';
--
-- Then restart the Concelier service.
\echo ''
\echo '== LNM-21-102-DEV Backfill Rollback =='
\conninfo
-- Count backfilled observations
SELECT 'Found ' || COUNT(*) || ' backfilled observations to remove.' as status
FROM advisory_observations
WHERE backfill_marker = 'lnm_21_102_dev';
-- Count backfilled linksets
SELECT 'Found ' || COUNT(*) || ' backfilled linksets to remove.' as status
FROM advisory_linksets
WHERE backfill_marker = 'lnm_21_102_dev';
-- Count advisory_raw tombstone markers
SELECT 'Found ' || COUNT(*) || ' advisory_raw documents with tombstone markers to clear.' as status
FROM advisory_raw
WHERE backfill_marker = 'lnm_21_102_dev';
-- Only execute if not DRY_RUN
\if :{?DRY_RUN}
\echo 'DRY RUN mode - no changes made'
\echo 'Set DRY_RUN=0 or omit it to execute the rollback'
\else
-- Step 1: Delete backfilled observations
DELETE FROM advisory_observations WHERE backfill_marker = 'lnm_21_102_dev';
\echo 'Deleted observations'
-- Step 2: Delete backfilled linksets
DELETE FROM advisory_linksets WHERE backfill_marker = 'lnm_21_102_dev';
\echo 'Deleted linksets'
-- Step 3: Clear tombstone markers from advisory_raw
UPDATE advisory_raw SET backfill_marker = NULL WHERE backfill_marker = 'lnm_21_102_dev';
\echo 'Cleared tombstone markers'
\endif
\echo ''
\echo '== Rollback Summary =='
\echo ''
\echo 'Next steps:'
\echo '1. Delete the migration record:'
\echo ' DELETE FROM schema_migrations WHERE id = ''20251127_lnm_legacy_backfill'';'
\echo '2. Restart the Concelier service.'
\echo ''