fixing an error + adding right logs

Author: larakulkarni1

dd-java-agent/instrumentation/spark/spark-common/src/main/java/datadog/trace/instrumentation/spark/AbstractDatadogSparkListener.java

@@ -254,10 +254,7 @@ private void initApplicationSpanIfNotInitialized() {
     }
 
     log.debug("Starting tracer application span.");
-    log.debug(databricksClusterName);
     if (!isRunningOnDatabricks) {
-      String databricksClusterNameSpark = databricksClusterName + "creating a spark.app";
-      log.debug(databricksClusterNameSpark);
       AgentTracer.SpanBuilder builder = buildSparkSpan("spark.application", null);
 
       if (applicationStart != null) {
@@ -332,13 +329,28 @@ public synchronized void finishApplication(
     }
     applicationEnded = true;
 
+    log.info(
+        "finishApplication: isRunningOnDatabricks={}, databricksClusterName={}, applicationSpan={}, jobCount={}",
+        isRunningOnDatabricks,
+        databricksClusterName,
+        applicationSpan != null ? "exists" : "null",
+        jobCount);
+
     if (applicationSpan == null && jobCount > 0) {
       // If the application span is not initialized, but spark jobs have been executed, all those
       // spark jobs were databricks or streaming. In this case we don't send the application span
       return;
     }
     initApplicationSpanIfNotInitialized();
 
+    if (applicationSpan == null) {
+      // On Databricks or streaming environments, the application span is not created.
+      // Flush any remaining traces and return.
+      log.info("No application span created, skipping. isRunningOnDatabricks={}", isRunningOnDatabricks);
+      tracer.flush();
+      return;
+    }
+
     if (throwable != null) {
       applicationSpan.addThrowable(throwable);
     } else if (exitCode != 0) {