Skip to content

Commit 7441ee2

Browse files
simongdaviessimongdavies
committed
added change detection
Signed-off-by: Simon Davies <[email protected]>
1 parent f2102bd commit 7441ee2

File tree

1 file changed

+216
-4
lines changed

1 file changed

+216
-4
lines changed

‎.github/workflows/dep_rust.yml

Lines changed: 216 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,157 @@ jobs:
6464
- name: fmt
6565
run: just fmt-check
6666

67+
- name: Start continuous device monitoring
68+
if: runner.os == 'Linux'
69+
run: |
70+
echo "Starting continuous /dev/kvm monitoring with inotify..."
71+
72+
# Create monitoring script using inotify
73+
cat > /tmp/monitor_kvm_device.sh << 'EOF'
74+
#!/bin/bash
75+
LOGFILE="/tmp/kvm_device_changes.log"
76+
BASELINE_FILE="/tmp/kvm_baseline.txt"
77+
78+
echo "$(date): Starting KVM device monitoring with inotify" >> "$LOGFILE"
79+
80+
# Function to record current state
81+
# Function to record current device state
82+
record_current_state() {
83+
if [ -e /dev/kvm ]; then
84+
ls -la /dev/kvm > "$BASELINE_FILE" 2>/dev/null
85+
echo "$(date): Initial state recorded: $(cat $BASELINE_FILE)" >> "$LOGFILE"
86+
else
87+
echo "NO_DEVICE" > "$BASELINE_FILE"
88+
echo "$(date): /dev/kvm does not exist" >> "$LOGFILE"
89+
fi
90+
}
91+
92+
# Function to handle device changes
93+
handle_change() {
94+
local event_type="$1"
95+
local timestamp=$(date '+%Y-%m-%d %H:%M:%S.%3N')
96+
echo "$timestamp: inotify event: $event_type" >> "$LOGFILE"
97+
98+
if [ -e /dev/kvm ]; then
99+
CURRENT_STATE=$(ls -la /dev/kvm 2>/dev/null)
100+
101+
if [ -f "$BASELINE_FILE" ]; then
102+
BASELINE_STATE=$(cat "$BASELINE_FILE" 2>/dev/null)
103+
104+
if [ "$CURRENT_STATE" != "$BASELINE_STATE" ] && [ "$BASELINE_STATE" != "NO_DEVICE" ]; then
105+
echo "$timestamp: CHANGE DETECTED!" >> "$LOGFILE"
106+
echo "$timestamp: Event type: $event_type" >> "$LOGFILE"
107+
echo "$timestamp: Previous: $BASELINE_STATE" >> "$LOGFILE"
108+
echo "$timestamp: Current: $CURRENT_STATE" >> "$LOGFILE"
109+
110+
# Only capture detailed diagnostics on actual changes
111+
echo "$timestamp: Active processes:" >> "$LOGFILE"
112+
ps aux | grep -E "(udev|systemd|kvm|qemu|hyperlight)" | grep -v grep >> "$LOGFILE" 2>/dev/null || echo "No relevant processes" >> "$LOGFILE"
113+
114+
# Check recent audit events (limit to avoid spam)
115+
echo "$timestamp: Recent audit events (last 5):" >> "$LOGFILE"
116+
sudo ausearch -k device_changes,hypervisor_kvm,permission_syscalls -ts recent 2>/dev/null | tail -5 >> "$LOGFILE" || echo "No recent audit events" >> "$LOGFILE"
117+
118+
elif [ "$BASELINE_STATE" = "NO_DEVICE" ] && [ -n "$CURRENT_STATE" ]; then
119+
echo "$timestamp: DEVICE CREATED!" >> "$LOGFILE"
120+
echo "$timestamp: Event type: $event_type" >> "$LOGFILE"
121+
echo "$timestamp: New device: $CURRENT_STATE" >> "$LOGFILE"
122+
123+
# Capture device creation events
124+
echo "$timestamp: Device creation audit events:" >> "$LOGFILE"
125+
sudo ausearch -k device_changes,hypervisor_kvm -ts recent 2>/dev/null >> "$LOGFILE" || echo "No device creation audit events" >> "$LOGFILE"
126+
fi
127+
fi
128+
129+
# Update baseline
130+
echo "$CURRENT_STATE" > "$BASELINE_FILE"
131+
else
132+
if [ -f "$BASELINE_FILE" ] && [ "$(cat "$BASELINE_FILE")" != "NO_DEVICE" ]; then
133+
echo "$timestamp: DEVICE REMOVED!" >> "$LOGFILE"
134+
echo "$timestamp: Event type: $event_type" >> "$LOGFILE"
135+
echo "$timestamp: Previous state: $(cat "$BASELINE_FILE")" >> "$LOGFILE"
136+
echo "NO_DEVICE" > "$BASELINE_FILE"
137+
fi
138+
fi
139+
}
140+
141+
# Record initial state
142+
record_current_state
143+
144+
# Ensure inotify tools are available
145+
if ! command -v inotifywait >/dev/null 2>&1; then
146+
echo "$(date): inotifywait not available, installing..." >> "$LOGFILE"
147+
sudo apt update && sudo apt install -y inotify-tools || { echo "$(date): FATAL: Failed to install inotify-tools" >> "$LOGFILE"; exit 1; }
148+
fi
149+
150+
echo "$(date): Starting inotify monitoring on /dev/" >> "$LOGFILE"
151+
152+
# Monitor the /dev directory for KVM device changes
153+
# We monitor /dev because /dev/kvm might not exist initially
154+
inotifywait -m -e create,delete,modify,attrib,moved_to,moved_from /dev/ 2>/dev/null | while read path action file; do
155+
if [[ "$file" == "kvm" ]]; then
156+
echo "$(date): inotify event on $file: $action" >> "$LOGFILE"
157+
handle_change "$action ($file)"
158+
fi
159+
done &
160+
DEV_MONITOR_PID=$!
161+
echo "$(date): Started /dev/ monitor with PID: $DEV_MONITOR_PID" >> "$LOGFILE"
162+
163+
# Also monitor /dev/kvm directly if it exists
164+
if [ -e /dev/kvm ]; then
165+
echo "$(date): Also monitoring /dev/kvm directly" >> "$LOGFILE"
166+
inotifywait -m -e modify,attrib,access /dev/kvm 2>/dev/null | while read path action file; do
167+
echo "$(date): Direct inotify event on /dev/kvm: $action" >> "$LOGFILE"
168+
handle_change "direct_kvm_$action"
169+
done &
170+
KVM_MONITOR_PID=$!
171+
echo "$(date): Started direct /dev/kvm monitor with PID: $KVM_MONITOR_PID" >> "$LOGFILE"
172+
fi
173+
174+
# Periodic check to ensure monitor processes are still running
175+
while true; do
176+
sleep 30
177+
if ! kill -0 $DEV_MONITOR_PID 2>/dev/null; then
178+
echo "$(date): WARNING: /dev/ monitor process died, restarting..." >> "$LOGFILE"
179+
break
180+
fi
181+
echo "$(date): Monitors still active" >> "$LOGFILE"
182+
done
183+
EOF
184+
185+
chmod +x /tmp/monitor_kvm_device.sh
186+
187+
# Start monitoring in background
188+
/tmp/monitor_kvm_device.sh &
189+
MONITOR_PID=$!
190+
echo "Started device monitoring with PID: $MONITOR_PID"
191+
echo $MONITOR_PID > /tmp/kvm_monitor.pid
192+
193+
# Create cleanup function
194+
cleanup_monitor() {
195+
echo "Cleaning up monitoring processes..."
196+
if [ -f /tmp/kvm_monitor.pid ]; then
197+
MAIN_PID=$(cat /tmp/kvm_monitor.pid)
198+
echo "Terminating main monitor process: $MAIN_PID"
199+
kill $MAIN_PID 2>/dev/null || echo "Main process already terminated"
200+
fi
201+
202+
# Kill any remaining inotifywait processes
203+
pkill -f "inotifywait.*kvm" 2>/dev/null || echo "No inotifywait processes to clean up"
204+
pkill -f "monitor_kvm_device" 2>/dev/null || echo "No monitor script processes to clean up"
205+
206+
echo "Monitor cleanup completed"
207+
}
208+
209+
# Set up cleanup trap
210+
trap cleanup_monitor EXIT
211+
212+
# Give monitor time to establish baseline and start inotify
213+
sleep 3
214+
215+
echo "Monitor status:"
216+
ps aux | grep "monitor_kvm_device\|inotifywait" | grep -v grep || echo "Monitor process not found"
217+
67218
- name: clippy
68219
run: |
69220
just clippy ${{ matrix.config }}
@@ -157,18 +308,79 @@ jobs:
157308
just bench-ci main ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
158309
if: ${{ matrix.config == 'release' }}
159310

311+
# Show device change monitoring results on failure
312+
- name: Show KVM device change analysis on failure
313+
if: failure() && runner.os == 'Linux'
314+
run: |
315+
echo "=== KVM Device Change Analysis (Job Failed) ==="
316+
echo "Timestamp: $(date)"
317+
echo ""
318+
319+
# Stop the monitoring process
320+
if [ -f /tmp/kvm_monitor.pid ]; then
321+
MONITOR_PID=$(cat /tmp/kvm_monitor.pid)
322+
echo "Stopping monitor process (PID: $MONITOR_PID)..."
323+
kill $MONITOR_PID 2>/dev/null || echo "Monitor process already stopped"
324+
sleep 2
325+
fi
326+
327+
echo "=== Device Change Log Analysis ==="
328+
if [ -f /tmp/kvm_device_changes.log ]; then
329+
echo "🔍 Changes detected during job execution:"
330+
cat /tmp/kvm_device_changes.log
331+
else
332+
echo "📝 No device changes detected (log file not found)"
333+
fi
334+
echo ""
335+
336+
echo "=== What Caused Permission/Ownership Changes ==="
337+
if [ -f /tmp/kvm_device_changes.log ] && grep -q "CHANGE DETECTED" /tmp/kvm_device_changes.log; then
338+
echo "⚠️ Device permissions or ownership changed during the job!"
339+
echo ""
340+
echo "Summary of changes:"
341+
grep -A 2 "CHANGE DETECTED" /tmp/kvm_device_changes.log || echo "Could not extract change summary"
342+
echo ""
343+
echo "Processes that may have caused changes:"
344+
grep -A 10 "Active processes:" /tmp/kvm_device_changes.log | head -20 || echo "No process information captured"
345+
echo ""
346+
echo "Recent audit events related to changes:"
347+
grep -A 10 "Recent audit events:" /tmp/kvm_device_changes.log | head -20 || echo "No audit events captured"
348+
else
349+
echo "✅ No device permission/ownership changes detected during job execution"
350+
fi
351+
160352
# Always check KVM device status at the end - runs regardless of job success/failure
161353
- name: Final KVM device status check
162354
if: always() && runner.os == 'Linux'
163355
run: |
164356
echo "=== Final KVM Device Status Check ==="
165357
echo "Timestamp: $(date)"
166358
echo ""
359+
360+
# Stop the monitoring process if still running
361+
if [ -f /tmp/kvm_monitor.pid ]; then
362+
MONITOR_PID=$(cat /tmp/kvm_monitor.pid)
363+
echo "Stopping monitor process (PID: $MONITOR_PID)..."
364+
kill $MONITOR_PID 2>/dev/null || echo "Monitor process already stopped"
365+
sleep 2
366+
fi
367+
167368
echo "KVM device listing:"
168369
sudo ls -al /dev/kvm 2>/dev/null || echo "❌ /dev/kvm device not found or not accessible"
169370
echo ""
170-
echo "MSHV device listing:"
171-
sudo ls -al /dev/mshv 2>/dev/null || echo "❌ /dev/mshv device not found or not accessible"
172-
echo ""
173371
echo "All hypervisor-related devices:"
174-
sudo ls -al /dev/ | grep -E "(kvm|mshv|hyperv)" 2>/dev/null || echo "No hypervisor devices found in /dev/"
372+
sudo ls -al /dev/ | grep -E "(kvm|hyperv)" 2>/dev/null || echo "No hypervisor devices found in /dev/"
373+
echo ""
374+
375+
echo "=== Complete Device Change Summary ==="
376+
if [ -f /tmp/kvm_device_changes.log ]; then
377+
echo "📊 Full monitoring log:"
378+
cat /tmp/kvm_device_changes.log
379+
echo ""
380+
echo "🔢 Change statistics:"
381+
echo "Device creation events: $(grep -c "DEVICE CREATED" /tmp/kvm_device_changes.log 2>/dev/null || echo "0")"
382+
echo "Device removal events: $(grep -c "DEVICE REMOVED" /tmp/kvm_device_changes.log 2>/dev/null || echo "0")"
383+
echo "Permission/ownership changes: $(grep -c "CHANGE DETECTED" /tmp/kvm_device_changes.log 2>/dev/null || echo "0")"
384+
else
385+
echo "📝 No monitoring log available"
386+
fi

0 commit comments

Comments
 (0)