Skip to content

Commit 6520ae8

Browse files
[BugFix] fix grpc failure when tracing init before workers forked (PaddlePaddle#6732)
* [fix] fix grpc failure when tracing init before workers forked * [fix] change default exporter to http * [fix] fix test_trace
1 parent 812657b commit 6520ae8

3 files changed

Lines changed: 11 additions & 2 deletions

File tree

fastdeploy/entrypoints/openai/api_server.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@
8484
retrive_model_from_server,
8585
)
8686

87-
tracing.process_tracing_init()
87+
_tracing_inited = False
8888

8989
parser = make_arg_parser(FlexibleArgumentParser())
9090
args = parser.parse_args()
@@ -171,8 +171,15 @@ async def lifespan(app: FastAPI):
171171
async context manager for FastAPI lifespan
172172
"""
173173
global engine_args
174+
global _tracing_inited
174175
import logging
175176

177+
# Initialize tracing in worker lifecycle instead of module import time.
178+
# This avoids creating grpc/cygrpc state before gunicorn forks workers.
179+
if not _tracing_inited:
180+
tracing.process_tracing_init()
181+
_tracing_inited = True
182+
176183
uvicorn_access = logging.getLogger("uvicorn.access")
177184
uvicorn_access.handlers.clear()
178185

fastdeploy/metrics/trace.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ def get_otlp_span_exporter(endpoint, headers):
289289
OTLPSpanExporter as HTTPSpanExporter,
290290
)
291291

292-
protocol = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, "grpc")
292+
protocol = os.environ.get(OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, "http/protobuf")
293293
supported_protocols = {"grpc", "http/protobuf"}
294294

295295
if protocol not in supported_protocols:

tests/metrics/test_trace.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -908,6 +908,8 @@ def test_function():
908908

909909
def test_get_otlp_span_exporter_grpc(self):
910910
"""Test get_otlp_span_exporter with grpc protocol"""
911+
# Set environment variable for grpc protocol
912+
os.environ["OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"] = "grpc"
911913
exporter = trace.get_otlp_span_exporter("http://localhost:4317", None)
912914
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
913915
OTLPSpanExporter as GRPCSpanExporter,

0 commit comments

Comments
 (0)