Refactor code structure and optimize performance across multiple modules
This commit is contained in:
StellaOps Bot
97
devops/logging/filebeat.yml
Normal file
97
devops/logging/filebeat.yml
Normal file
@@ -0,0 +1,97 @@
|
||||
# StellaOps Filebeat Configuration
|
||||
# Ships logs to Elasticsearch/Logstash for centralized logging
|
||||
|
||||
filebeat.inputs:
|
||||
# Application logs (JSON format from Serilog)
|
||||
- type: log
|
||||
enabled: true
|
||||
paths:
|
||||
- /var/log/stellaops/*/*.log
|
||||
json.keys_under_root: true
|
||||
json.add_error_key: true
|
||||
json.message_key: message
|
||||
json.overwrite_keys: true
|
||||
fields:
|
||||
log_type: application
|
||||
fields_under_root: true
|
||||
multiline:
|
||||
type: pattern
|
||||
pattern: '^\[?[0-9]{4}-[0-9]{2}-[0-9]{2}'
|
||||
negate: true
|
||||
match: after
|
||||
|
||||
# Container logs (stdout/stderr)
|
||||
- type: container
|
||||
enabled: true
|
||||
paths:
|
||||
- /var/lib/docker/containers/*/*.log
|
||||
processors:
|
||||
- add_kubernetes_metadata:
|
||||
host: ${NODE_NAME}
|
||||
matchers:
|
||||
- logs_path:
|
||||
logs_path: "/var/lib/docker/containers/"
|
||||
|
||||
# Processors for all inputs
|
||||
processors:
|
||||
- add_host_metadata:
|
||||
when.not.contains.tags: forwarded
|
||||
- add_cloud_metadata: ~
|
||||
- add_docker_metadata: ~
|
||||
- decode_json_fields:
|
||||
fields: ["message"]
|
||||
target: ""
|
||||
overwrite_keys: true
|
||||
when:
|
||||
has_fields: ["message"]
|
||||
- drop_fields:
|
||||
fields: ["agent.ephemeral_id", "agent.id", "agent.name"]
|
||||
ignore_missing: true
|
||||
|
||||
# Output configuration
|
||||
output.elasticsearch:
|
||||
enabled: ${FILEBEAT_ELASTICSEARCH_ENABLED:false}
|
||||
hosts: ["${ELASTICSEARCH_HOST:localhost}:${ELASTICSEARCH_PORT:9200}"]
|
||||
protocol: "${ELASTICSEARCH_PROTOCOL:http}"
|
||||
username: "${ELASTICSEARCH_USERNAME:}"
|
||||
password: "${ELASTICSEARCH_PASSWORD:}"
|
||||
index: "stellaops-%{[fields.log_type]}-%{+yyyy.MM.dd}"
|
||||
ssl:
|
||||
enabled: ${ELASTICSEARCH_SSL_ENABLED:false}
|
||||
verification_mode: "${ELASTICSEARCH_SSL_VERIFICATION:full}"
|
||||
|
||||
output.logstash:
|
||||
enabled: ${FILEBEAT_LOGSTASH_ENABLED:false}
|
||||
hosts: ["${LOGSTASH_HOST:localhost}:${LOGSTASH_PORT:5044}"]
|
||||
ssl:
|
||||
enabled: ${LOGSTASH_SSL_ENABLED:false}
|
||||
|
||||
# Fallback to file output (useful for air-gapped environments)
|
||||
output.file:
|
||||
enabled: ${FILEBEAT_FILE_ENABLED:false}
|
||||
path: "/var/log/filebeat"
|
||||
filename: stellaops-filebeat
|
||||
rotate_every_kb: 10240
|
||||
number_of_files: 7
|
||||
|
||||
# Logging
|
||||
logging.level: info
|
||||
logging.to_files: true
|
||||
logging.files:
|
||||
path: /var/log/filebeat
|
||||
name: filebeat
|
||||
keepfiles: 7
|
||||
permissions: 0640
|
||||
|
||||
# Index Lifecycle Management
|
||||
setup.ilm:
|
||||
enabled: true
|
||||
rollover_alias: "stellaops"
|
||||
pattern: "{now/d}-000001"
|
||||
policy_name: "stellaops-ilm-policy"
|
||||
|
||||
# Kibana dashboards
|
||||
setup.kibana:
|
||||
enabled: ${KIBANA_ENABLED:false}
|
||||
host: "${KIBANA_HOST:localhost}:${KIBANA_PORT:5601}"
|
||||
protocol: "${KIBANA_PROTOCOL:http}"
|
||||
83
devops/logging/logrotate.conf
Normal file
83
devops/logging/logrotate.conf
Normal file
@@ -0,0 +1,83 @@
|
||||
# StellaOps Logrotate Configuration
|
||||
# Place in /etc/logrotate.d/stellaops
|
||||
|
||||
/var/log/stellaops/*/*.log {
|
||||
daily
|
||||
rotate 14
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 0640 stellaops stellaops
|
||||
sharedscripts
|
||||
dateext
|
||||
dateformat -%Y%m%d
|
||||
|
||||
# Size-based rotation (rotate if larger than 100MB regardless of time)
|
||||
maxsize 100M
|
||||
|
||||
# Minimum size before considering rotation
|
||||
minsize 1M
|
||||
|
||||
postrotate
|
||||
# Signal services to reopen log files if needed
|
||||
# Most Serilog file sinks handle this automatically
|
||||
/bin/true
|
||||
endscript
|
||||
}
|
||||
|
||||
# Scanner service specific (higher volume)
|
||||
/var/log/stellaops/scanner/*.log {
|
||||
daily
|
||||
rotate 7
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 0640 stellaops stellaops
|
||||
sharedscripts
|
||||
dateext
|
||||
maxsize 200M
|
||||
}
|
||||
|
||||
# Concelier service (vulnerability processing)
|
||||
/var/log/stellaops/concelier/*.log {
|
||||
daily
|
||||
rotate 14
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 0640 stellaops stellaops
|
||||
sharedscripts
|
||||
dateext
|
||||
maxsize 150M
|
||||
}
|
||||
|
||||
# Authority service (signing operations - keep longer for audit)
|
||||
/var/log/stellaops/authority/*.log {
|
||||
daily
|
||||
rotate 30
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 0640 stellaops stellaops
|
||||
sharedscripts
|
||||
dateext
|
||||
maxsize 50M
|
||||
}
|
||||
|
||||
# Router/Gateway logs
|
||||
/var/log/stellaops/router/*.log {
|
||||
daily
|
||||
rotate 7
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 0640 stellaops stellaops
|
||||
sharedscripts
|
||||
dateext
|
||||
maxsize 100M
|
||||
}
|
||||
62
devops/logging/serilog.json.template
Normal file
62
devops/logging/serilog.json.template
Normal file
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"Serilog": {
|
||||
"Using": [
|
||||
"Serilog.Sinks.Console",
|
||||
"Serilog.Sinks.File",
|
||||
"Serilog.Enrichers.Thread",
|
||||
"Serilog.Enrichers.Environment"
|
||||
],
|
||||
"MinimumLevel": {
|
||||
"Default": "Information",
|
||||
"Override": {
|
||||
"Microsoft": "Warning",
|
||||
"Microsoft.AspNetCore": "Warning",
|
||||
"Microsoft.EntityFrameworkCore": "Warning",
|
||||
"System": "Warning",
|
||||
"System.Net.Http": "Warning",
|
||||
"Grpc": "Warning"
|
||||
}
|
||||
},
|
||||
"WriteTo": [
|
||||
{
|
||||
"Name": "Console",
|
||||
"Args": {
|
||||
"outputTemplate": "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} {Level:u3}] [{SourceContext}] {Message:lj}{NewLine}{Exception}",
|
||||
"theme": "Serilog.Sinks.SystemConsole.Themes.AnsiConsoleTheme::Code, Serilog.Sinks.Console"
|
||||
}
|
||||
},
|
||||
{
|
||||
"Name": "File",
|
||||
"Args": {
|
||||
"path": "/var/log/stellaops/${STELLAOPS_SERVICE_NAME:-.}/stellaops-.log",
|
||||
"rollingInterval": "Day",
|
||||
"retainedFileCountLimit": 14,
|
||||
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] [{SourceContext}] [{TraceId}] {Message:lj}{NewLine}{Exception}",
|
||||
"fileSizeLimitBytes": 104857600,
|
||||
"rollOnFileSizeLimit": true
|
||||
}
|
||||
}
|
||||
],
|
||||
"Enrich": [
|
||||
"FromLogContext",
|
||||
"WithMachineName",
|
||||
"WithThreadId",
|
||||
"WithEnvironmentName",
|
||||
"WithProcessId"
|
||||
],
|
||||
"Properties": {
|
||||
"Application": "StellaOps",
|
||||
"ServiceName": "${STELLAOPS_SERVICE_NAME:-Unknown}",
|
||||
"Environment": "${ASPNETCORE_ENVIRONMENT:-Production}"
|
||||
},
|
||||
"Filter": [
|
||||
{
|
||||
"Name": "ByExcluding",
|
||||
"Args": {
|
||||
"expression": "Contains(@Message, 'Executing endpoint')"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"AllowedHosts": "*"
|
||||
}
|
||||
Reference in New Issue
Block a user