Not for review

.arclint

@@ -36,7 +36,8 @@
     "(^third_party/github.com/gogo/protobuf/gogoproto/gogo.proto$)",
     "(c_cpp_properties.json$)",
     "(go\\.mod$)",
-    "(go\\.sum$)"
+    "(go\\.sum$)",
+    "(src/stirling/source_connectors/socket_tracer/bcc_bpf/offsetgen_output.json)"
   ],
   "linters": {
     "build-linter": {
@@ -131,13 +132,24 @@
       "type": "goimports",
       "include": [
         "(\\.go$)"
+      ],
+      "exclude": [
+        "(src/stirling/testing/demo_apps/go_https/(server|client)/testdata/.*)",
+        "(src/stirling/source_connectors/socket_tracer/protocols/http2/testing/(go_grpc_server|go_grpc_client)/testdata/.*)"
       ]
     },
     "golangci-lint": {
       "type": "golangci-lint",
       "include": [
         "(\\.go$)"
       ],
+      "exclude": [
+        "(src/stirling/testing/demo_apps/go_https/(server|client)/testdata/.*)",
+        "(src/stirling/testing/demo_apps/go_https/server/https_server.go)",
+        "(src/stirling/source_connectors/socket_tracer/protocols/http2/testing/(go_grpc_server|go_grpc_client)/testdata/.*)",
+        "(src/vizier/services/query_broker/controllers)",
+        "(src/carnot/goplanner/logical_planner.go)"
+      ],
       "flags": [
         "--timeout=5m0s",
         "--out-format=checkstyle"
@@ -185,15 +197,6 @@
         "(^tools/arc_addons/.*\\.php$)"
       ]
     },
-    "proto-break-check": {
-      "type": "proto-break-check",
-      "include": [
-        "(.*\\.proto$)"
-      ],
-      "exclude": [
-        "(^src/stirling/source_connectors/dynamic_tracer/dynamic_tracing/ir/physicalpb/physical\\.proto$)"
-      ]
-    },
     "shellcheck": {
       "type": "shellcheck",
       "include": [

BUILD.bazel

@@ -39,6 +39,10 @@ gazelle(
 # gazelle:exclude **/*.pb.go
 # gazelle:exclude **/mock.go
 # gazelle:exclude external
+# gazelle:exclude src/stirling/testing/demo_apps/go_https/server/testdata/*
+# gazelle:exclude src/stirling/testing/demo_apps/go_https/client/testdata/*
+# gazelle:exclude src/stirling/source_connectors/socket_tracer/protocols/http2/testing/go_grpc_client/testdata/*
+# gazelle:exclude src/stirling/source_connectors/socket_tracer/protocols/http2/testing/go_grpc_server/testdata/*
 
 # Make gazelle not generate proto files. We need to use gogo proto and this does
 # not seem to work automatically right now. Keep an eye out on issue:

bazel/external/bcc.patch

@@ -0,0 +1,181 @@
+diff --git a/src/cc/CMakeLists.txt b/src/cc/CMakeLists.txt
+index 632adf36..5aadfd84 100644
+--- a/src/cc/CMakeLists.txt
++++ b/src/cc/CMakeLists.txt
+@@ -67,9 +67,6 @@ if(NOT CMAKE_USE_LIBBPF_PACKAGE)
+ endif()
+
+ set(bcc_common_sources bcc_common.cc bpf_module.cc bcc_btf.cc exported_files.cc)
+-if (${LLVM_PACKAGE_VERSION} VERSION_EQUAL 6 OR ${LLVM_PACKAGE_VERSION} VERSION_GREATER 6)
+-  set(bcc_common_sources ${bcc_common_sources} bcc_debug.cc)
+-endif()
+
+ if(ENABLE_LLVM_NATIVECODEGEN)
+ set(bcc_common_sources ${bcc_common_sources} bpf_module_rw_engine.cc)
+diff --git a/src/cc/bcc_btf.cc b/src/cc/bcc_btf.cc
+index be248612..f07853b5 100644
+--- a/src/cc/bcc_btf.cc
++++ b/src/cc/bcc_btf.cc
+@@ -367,9 +367,9 @@ int32_t BTFStringTable::addString(std::string S) {
+   return Offset;
+ }
+
+-BTF::BTF(bool debug, sec_map_def &sections) : debug_(debug),
++BTF::BTF(bool /*debug*/, sec_map_def &sections) : debug_(false),
+     btf_(nullptr), btf_ext_(nullptr), sections_(sections) {
+-  if (!debug)
++  if (!false)
+     libbpf_set_print(NULL);
+ }
+
+diff --git a/src/cc/bpf_module.cc b/src/cc/bpf_module.cc
+index c0dd0119..4160bc22 100644
+--- a/src/cc/bpf_module.cc
++++ b/src/cc/bpf_module.cc
+@@ -60,7 +60,6 @@
+ #include <vector>
+
+ #include "bcc_btf.h"
+-#include "bcc_debug.h"
+ #include "bcc_elf.h"
+ #include "bcc_libbpf_inc.h"
+ #include "common.h"
+@@ -162,8 +161,8 @@ BPFModule::BPFModule(unsigned flags, TableStorage *ts, bool rw_engine_enabled,
+   LLVMInitializeBPFAsmPrinter();
+ #if LLVM_VERSION_MAJOR >= 6
+   LLVMInitializeBPFAsmParser();
+-  if (flags & DEBUG_SOURCE)
+-    LLVMInitializeBPFDisassembler();
++  /* if (flags & DEBUG_SOURCE) */
++  /*   LLVMInitializeBPFDisassembler(); */
+ #endif
+   LLVMLinkInMCJIT(); /* call empty function to force linking of MCJIT */
+   if (!ts_) {
+@@ -283,8 +282,8 @@ void BPFModule::dump_ir(Module &mod) {
+
+ int BPFModule::run_pass_manager(Module &mod) {
+   if (verifyModule(mod, &errs())) {
+-    if (flags_ & DEBUG_LLVM_IR)
+-      dump_ir(mod);
++    /* if (flags_ & DEBUG_LLVM_IR) */
++    /*   dump_ir(mod); */
+     return -1;
+   }
+
+@@ -306,8 +305,8 @@ int BPFModule::run_pass_manager(Module &mod) {
+
+   // Add passes and run
+   MPM.addPass(AlwaysInlinerPass());
+-  if (flags_ & DEBUG_LLVM_IR)
+-    MPM.addPass(PrintModulePass(outs()));
++  /* if (flags_ & DEBUG_LLVM_IR) */
++  /*   MPM.addPass(PrintModulePass(outs())); */
+   MPM.run(mod, MAM);
+ #else
+   legacy::PassManager PM;
+@@ -323,8 +322,8 @@ int BPFModule::run_pass_manager(Module &mod) {
+    */
+   LLVMAddAlwaysInlinerPass(reinterpret_cast<LLVMPassManagerRef>(&PM));
+   PMB.populateModulePassManager(PM);
+-  if (flags_ & DEBUG_LLVM_IR)
+-    PM.add(createPrintModulePass(outs()));
++  /* if (flags_ & DEBUG_LLVM_IR) */
++  /*   PM.add(createPrintModulePass(outs())); */
+   PM.run(mod);
+ #endif
+
+@@ -594,11 +593,11 @@ int BPFModule::finalize() {
+   });
+   finalize_prog_func_info();
+
+-  if (flags_ & DEBUG_SOURCE) {
+-    SourceDebugger src_debugger(mod, *sections_p, *prog_func_info_, mod_src_,
+-                                src_dbg_fmap_);
+-    src_debugger.dump();
+-  }
++  /* if (flags_ & DEBUG_SOURCE) { */
++  /*   SourceDebugger src_debugger(mod, *sections_p, *prog_func_info_, mod_src_, */
++  /*                               src_dbg_fmap_); */
++  /*   src_debugger.dump(); */
++  /* } */
+
+   load_btf(*sections_p);
+   if (load_maps(*sections_p))
+diff --git a/src/cc/frontends/clang/b_frontend_action.cc b/src/cc/frontends/clang/b_frontend_action.cc
+index 95cec3cb..739fb7bb 100644
+--- a/src/cc/frontends/clang/b_frontend_action.cc
++++ b/src/cc/frontends/clang/b_frontend_action.cc
+@@ -1861,18 +1861,18 @@ void BFrontendAction::EndSourceFileAction() {
+   // Additional misc rewrites
+   DoMiscWorkAround();
+
+-  if (flags_ & DEBUG_PREPROCESSOR)
+-    rewriter_->getEditBuffer(rewriter_->getSourceMgr().getMainFileID()).write(llvm::errs());
++  /* if (flags_ & DEBUG_PREPROCESSOR) */
++  /*   rewriter_->getEditBuffer(rewriter_->getSourceMgr().getMainFileID()).write(llvm::errs()); */
+ #if LLVM_VERSION_MAJOR >= 9
+   llvm::raw_string_ostream tmp_os(mod_src_);
+   rewriter_->getEditBuffer(rewriter_->getSourceMgr().getMainFileID())
+       .write(tmp_os);
+ #else
+-  if (flags_ & DEBUG_SOURCE) {
+-    llvm::raw_string_ostream tmp_os(mod_src_);
+-    rewriter_->getEditBuffer(rewriter_->getSourceMgr().getMainFileID())
+-        .write(tmp_os);
+-  }
++  /* if (flags_ & DEBUG_SOURCE) { */
++  /*   llvm::raw_string_ostream tmp_os(mod_src_); */
++  /*   rewriter_->getEditBuffer(rewriter_->getSourceMgr().getMainFileID()) */
++  /*       .write(tmp_os); */
++  /* } */
+ #endif
+
+   for (auto func : func_range_) {
+diff --git a/src/cc/frontends/clang/loader.cc b/src/cc/frontends/clang/loader.cc
+index 7950cda4..df6614c0 100644
+--- a/src/cc/frontends/clang/loader.cc
++++ b/src/cc/frontends/clang/loader.cc
+@@ -231,8 +231,8 @@ int ClangLoader::parse(
+     }
+   }
+
+-  if (flags_ & DEBUG_PREPROCESSOR)
+-    std::cout << "Running from kernel directory at: " << kpath.c_str() << "\n";
++  /* if (flags_ & DEBUG_PREPROCESSOR) */
++  /*   std::cout << "Running from kernel directory at: " << kpath.c_str() << "\n"; */
+
+   // clang needs to run inside the kernel dir
+   DirStack dstack(kpath);
+@@ -288,13 +288,6 @@ int ClangLoader::parse(
+   vector<string> kflags;
+   if (kbuild_helper.get_flags(un.machine, &kflags))
+     return -1;
+-#if LLVM_VERSION_MAJOR >= 9
+-  flags_cstr.push_back("-g");
+-  flags_cstr.push_back("-gdwarf-4");
+-#else
+-  if (flags_ & DEBUG_SOURCE)
+-    flags_cstr.push_back("-g");
+-#endif
+   for (auto it = kflags.begin(); it != kflags.end(); ++it)
+     flags_cstr.push_back(it->c_str());
+
+@@ -447,12 +440,12 @@ int ClangLoader::do_compile(
+   // Initialize a compiler invocation object from the clang (-cc1) arguments.
+   const llvm::opt::ArgStringList &ccargs = cmd.getArguments();
+
+-  if (flags_ & DEBUG_PREPROCESSOR) {
+-    llvm::errs() << "clang";
+-    for (auto arg : ccargs)
+-      llvm::errs() << " " << arg;
+-    llvm::errs() << "\n";
+-  }
++  /* if (flags_ & DEBUG_PREPROCESSOR) { */
++  /*   llvm::errs() << "clang"; */
++  /*   for (auto arg : ccargs) */
++  /*     llvm::errs() << " " << arg; */
++  /*   llvm::errs() << "\n"; */
++  /* } */
+
+   // pre-compilation pass for generating tracepoint structures
+   CompilerInstance compiler0;

bazel/external/llvm.BUILD

@@ -46,11 +46,9 @@ cc_library(
         # LLVM Libraries.
         "lib/libLLVMWindowsManifest.a",
         "lib/libLLVMWindowsDriver.a",
-        "lib/libLLVMXRay.a",
         "lib/libLLVMLibDriver.a",
         "lib/libLLVMDlltoolDriver.a",
         "lib/libLLVMCoverage.a",
-        "lib/libLLVMLineEditor.a",
         "lib/libLLVMAArch64Disassembler.a",
         "lib/libLLVMAArch64AsmParser.a",
         "lib/libLLVMAArch64CodeGen.a",
@@ -63,7 +61,6 @@ cc_library(
         "lib/libLLVMX86CodeGen.a",
         "lib/libLLVMX86Desc.a",
         "lib/libLLVMX86Info.a",
-        "lib/libLLVMBPFDisassembler.a",
         "lib/libLLVMBPFAsmParser.a",
         "lib/libLLVMBPFCodeGen.a",
         "lib/libLLVMBPFDesc.a",
@@ -81,11 +78,9 @@ cc_library(
         "lib/libLLVMOption.a",
         "lib/libLLVMObjectYAML.a",
         "lib/libLLVMObjCopy.a",
-        "lib/libLLVMMCA.a",
         "lib/libLLVMMCDisassembler.a",
         "lib/libLLVMLTO.a",
         "lib/libLLVMCFGuard.a",
-        "lib/libLLVMFrontendOpenACC.a",
         "lib/libLLVMExtensions.a",
         "lib/libLLVMPasses.a",
         "lib/libLLVMObjCARCOpts.a",
@@ -105,7 +100,6 @@ cc_library(
         "lib/libLLVMAsmParser.a",
         "lib/libLLVMInterfaceStub.a",
         "lib/libLLVMFileCheck.a",
-        "lib/libLLVMFuzzMutate.a",
         "lib/libLLVMTarget.a",
         "lib/libLLVMScalarOpts.a",
         "lib/libLLVMInstCombine.a",
@@ -115,21 +109,17 @@ cc_library(
         "lib/libLLVMAnalysis.a",
         "lib/libLLVMProfileData.a",
         "lib/libLLVMSymbolize.a",
-        "lib/libLLVMDebugInfoPDB.a",
-        "lib/libLLVMDebugInfoMSF.a",
         "lib/libLLVMDebugInfoDWARF.a",
         "lib/libLLVMObject.a",
         "lib/libLLVMTextAPI.a",
         "lib/libLLVMMCParser.a",
         "lib/libLLVMMC.a",
         "lib/libLLVMDebugInfoCodeView.a",
         "lib/libLLVMBitReader.a",
-        "lib/libLLVMFuzzerCLI.a",
         "lib/libLLVMCore.a",
         "lib/libLLVMRemarks.a",
         "lib/libLLVMBitstreamReader.a",
         "lib/libLLVMBinaryFormat.a",
-        "lib/libLLVMTableGen.a",
         "lib/libLLVMSupport.a",
         "lib/libLLVMDemangle.a",
 

bazel/repositories.bzl

@@ -172,7 +172,7 @@ def _cc_deps():
     # Uncomment these to develop bcc and/or bpftrace locally. Should also comment out the corresponding _bazel_repo lines.
     # _local_repo("com_github_iovisor_bcc", build_file = "//bazel/external/local_dev:bcc.BUILD")
     # _local_repo("com_github_iovisor_bpftrace", build_file = "//bazel/external/local_dev:bpftrace.BUILD")
-    _bazel_repo("com_github_iovisor_bcc", build_file = "//bazel/external:bcc.BUILD")
+    _bazel_repo("com_github_iovisor_bcc", build_file = "//bazel/external:bcc.BUILD", patches= ["//bazel/external:bcc.patch"], patch_args = ["-p1"])
     _bazel_repo("com_github_iovisor_bpftrace", build_file = "//bazel/external:bpftrace.BUILD")
 
     # TODO(jps): For jattach, consider using a patch and directly pulling from upstream (vs. fork).

k8s/vizier/pem/base/pem_daemonset.yaml

@@ -69,8 +69,36 @@ spec:
       containers:
       - name: pem
         image: vizier-pem_image:latest
-        args: []
+        args:
+        - --disable_dwarf_parsing=1
+        - --stirling_socket_tracer_percpu_bw_scaling_factor=0
+        - --stirling_socket_tracer_max_total_data_bw=104857600 # 100MB
+        - --stirling_socket_tracer_max_total_control_bw=10485760 # 10MB
         env:
+        - name: PX_STIRLING_HTTP_BODY_LIMIT_BYTES
+          value: "4096"
+        - name: PL_STIRLING_SOURCES
+          value: "socket_tracer,process_stats"
+        - name: PX_STIRLING_ENABLE_MUX_TRACING
+          value: "0"
+        - name: PX_STIRLING_ENABLE_NATS_TRACING
+          value: "0"
+        - name: PX_STIRLING_ENABLE_CASS_TRACING
+          value: "0"
+        - name: PX_STIRLING_ENABLE_DNS_TRACING
+          value: "0"
+        - name: PX_STIRLING_ENABLE_MUX_TRACING
+          value: "0"
+        - name: PX_STIRLING_ENABLE_TLS_TRACING
+          value: "0"
+        - name: PL_TABLE_STORE_DATA_LIMIT_MB
+          value: "100"
+        - name: PL_DATASTREAM_BUFFER_SPIKE_SIZE
+          value: "20971520" # 20MB
+        - name: PL_DATASTREAM_BUFFER_MAX_GAP_SIZE
+          value: "5242880" # 5MB
+        - name: PX_TOTAL_CONN_TRACKER_MEM_USAGE
+          value: "104857600" # 100MB
         - name: TCMALLOC_SAMPLE_PARAMETER
           value: "1048576"
         - name: PL_CLIENT_TLS_CERT
@@ -114,7 +142,11 @@ spec:
               optional: true
         - name: PL_CLOCK_CONVERTER
           value: "default"
-        resources: {}
+        resources:
+          limits:
+            memory: 512Mi
+          requests:
+            memory: 512Mi
         securityContext:
           capabilities:
             add:

src/carnot/BUILD.bazel

@@ -46,7 +46,6 @@ pl_cc_library(
     ),
     deps = [
         "//src/carnot/exec:cc_library",
-        "//src/carnot/exec/ml:cc_library",
         "//src/carnot/funcs:cc_library",
         "//src/carnot/plan:cc_library",
         "//src/carnot/planner/compiler:cc_library",

src/carnot/exec/BUILD.bazel

@@ -38,7 +38,6 @@ pl_cc_library(
     ],
     deps = [
         "//src/carnot/carnotpb:carnot_pl_cc_proto",
-        "//src/carnot/exec/ml:cc_library",
         "//src/carnot/plan:cc_library",
         "//src/carnot/planpb:plan_pl_cc_proto",
         "//src/carnot/udf:cc_library",