Recover python frames when BPF fails to read PyCodeObject

When bpf_probe_read_user fails to read a PyCodeObject (e.g. page
swapped out), push the frame with codeobject_id=0 instead of aborting
the unwind. This preserves the rest of the stack trace.

On the agent side, handle ebpfChecksum=0 in getCodeObject by skipping
the LRU cache (no checksum to validate against) and the staleness
check (no BPF reference to compare). The agent reads the code object
via process_vm_readv which supports page faults, so it can succeed
where BPF could not. Store the calculated checksum in the cache so
subsequent frames with a real BPF checksum can match.

Author: gnurizen

interpreter/python/python.go

@@ -454,10 +454,12 @@ func (p *pythonInstance) getCodeObject(addr libpf.Address,
 	if addr == 0 {
 		return nil, errors.New("failed to read code object: null pointer")
 	}
-	if value, ok := p.addrToCodeObject.Get(addr); ok {
-		m := value
-		if m.ebpfChecksum == ebpfChecksum {
-			return m, nil
+	if ebpfChecksum != 0 {
+		if value, ok := p.addrToCodeObject.Get(addr); ok {
+			m := value
+			if m.ebpfChecksum == ebpfChecksum {
+				return m, nil
+			}
 		}
 	}
 
@@ -512,7 +514,7 @@ func (p *pythonInstance) getCodeObject(addr libpf.Address,
 
 	ebpfChecksumCalculated := (argCount << 25) + (kwonlyArgCount << 18) +
 		(flags << 10) + firstLineNo
-	if ebpfChecksum != ebpfChecksumCalculated {
+	if ebpfChecksum != 0 && ebpfChecksum != ebpfChecksumCalculated {
 		return nil, fmt.Errorf("read code object was stale: %x != %x",
 			ebpfChecksum, ebpfChecksumCalculated)
 	}
@@ -533,7 +535,7 @@ func (p *pythonInstance) getCodeObject(addr libpf.Address,
 		sourceFileName: libpf.Intern(sourceFileName),
 		firstLineNo:    firstLineNo,
 		lineTable:      lineTable,
-		ebpfChecksum:   ebpfChecksum,
+		ebpfChecksum:   ebpfChecksumCalculated,
 	}
 	p.addrToCodeObject.Add(addr, pco)
 	return pco, nil

support/ebpf/python_tracer.ebpf.c

@@ -130,7 +130,12 @@ static EBPF_INLINE ErrorCode process_python_frame(
   if (bpf_probe_read_user(pss->code, sizeof(pss->code), py_codeobject)) {
     DEBUG_PRINT("Failed to read PyCodeObject at 0x%lx", (unsigned long)(py_codeobject));
     increment_metric(metricID_UnwindPythonErrBadCodeObjectArgCountAddr);
-    return ERR_PYTHON_BAD_CODE_OBJECT_ADDR;
+    // Push the frame with the code object address so the agent can try to
+    // read it via /proc/pid/mem (which supports page faults unlike BPF).
+    // codeobject_id=0 distinguishes this from a successful read.
+    file_id = (u64)py_codeobject;
+    lineno  = py_encode_lineno(0, (u32)py_f_lasti);
+    goto push_frame;
   }
 
   int py_argcount       = *(int *)(&pss->code[pyinfo->PyCodeObject_co_argcount]);