time information in transcription?

[jlmxyz]

it is possible to create "subtitle" like output? eg when processing a file, have "timing information added" for example when an end of sentence is detected, or on "no speech detected", or maybe on "each word"

[KitaitiMakoto]

Does output of --output-json-full fit your need?

% ./build/bin/whisper-cli --output-json-full ./samples/jfk.wav
% cat ./samples/jfk.wav.json

./samples/jfk.wav.json

{
	"systeminfo": "WHISPER : COREML = 0 | OPENVINO = 0 | MTL : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | MATMUL_INT8 = 1 | DOTPROD = 1 | ACCELERATE = 1 | REPACK = 1 | ",
	"model": {
		"type": "base",
		"multilingual": false,
		"vocab": 51864,
		"audio": {
			"ctx": 1500,
			"state": 512,
			"head": 8,
			"layer": 6
		},
		"text": {
			"ctx": 448,
			"state": 512,
			"head": 8,
			"layer": 6
		},
		"mels": 80,
		"ftype": 1
	},
	"params": {
		"model": "models/ggml-base.en.bin",
		"language": "en",
		"translate": false
	},
	"result": {
		"language": "en"
	},
	"transcription": [
		{
			"timestamps": {
				"from": "00:00:00,000",
				"to": "00:00:11,000"
			},
			"offsets": {
				"from": 0,
				"to": 11000
			},
			"text": " And so my fellow Americans, ask not what your country can do for you, ask what you can do for your country.",
			"tokens": [
				{
					"text": "[_BEG_]",
					"timestamps": {
						"from": "00:00:00,000",
						"to": "00:00:00,000"
					},
					"offsets": {
						"from": 0,
						"to": 0
					},
					"id": 50363,
					"p": 0.842062,
					"t_dtw": -1
				},
				{
					"text": " And",
					"timestamps": {
						"from": "00:00:00,320",
						"to": "00:00:00,370"
					},
					"offsets": {
						"from": 320,
						"to": 370
					},
					"id": 843,
					"p": 0.711617,
					"t_dtw": -1
				},
				{
					"text": " so",
					"timestamps": {
						"from": "00:00:00,370",
						"to": "00:00:00,530"
					},
					"offsets": {
						"from": 370,
						"to": 530
					},
					"id": 523,
					"p": 0.985344,
					"t_dtw": -1
				},
				{
					"text": " my",
					"timestamps": {
						"from": "00:00:00,690",
						"to": "00:00:00,850"
					},
					"offsets": {
						"from": 690,
						"to": 850
					},
					"id": 616,
					"p": 0.707049,
					"t_dtw": -1
				},
				{
					"text": " fellow",
					"timestamps": {
						"from": "00:00:00,850",
						"to": "00:00:01,590"
					},
					"offsets": {
						"from": 850,
						"to": 1590
					},
					"id": 5891,
					"p": 0.994885,
					"t_dtw": -1
				},
				{
					"text": " Americans",
					"timestamps": {
						"from": "00:00:01,590",
						"to": "00:00:02,100"
					},
					"offsets": {
						"from": 1590,
						"to": 2100
					},
					"id": 3399,
					"p": 0.900905,
					"t_dtw": -1
				},
				{
					"text": ",",
					"timestamps": {
						"from": "00:00:02,850",
						"to": "00:00:03,300"
					},
					"offsets": {
						"from": 2850,
						"to": 3300
					},
					"id": 11,
					"p": 0.283596,
					"t_dtw": -1
				},
				{
					"text": " ask",
					"timestamps": {
						"from": "00:00:03,300",
						"to": "00:00:04,140"
					},
					"offsets": {
						"from": 3300,
						"to": 4140
					},
					"id": 1265,
					"p": 0.797562,
					"t_dtw": -1
				},
				{
					"text": " not",
					"timestamps": {
						"from": "00:00:04,140",
						"to": "00:00:04,280"
					},
					"offsets": {
						"from": 4140,
						"to": 4280
					},
					"id": 407,
					"p": 0.788518,
					"t_dtw": -1
				},
				{
					"text": " what",
					"timestamps": {
						"from": "00:00:05,030",
						"to": "00:00:05,350"
					},
					"offsets": {
						"from": 5030,
						"to": 5350
					},
					"id": 644,
					"p": 0.933237,
					"t_dtw": -1
				},
				{
					"text": " your",
					"timestamps": {
						"from": "00:00:05,410",
						"to": "00:00:05,740"
					},
					"offsets": {
						"from": 5410,
						"to": 5740
					},
					"id": 534,
					"p": 0.987863,
					"t_dtw": -1
				},
				{
					"text": " country",
					"timestamps": {
						"from": "00:00:05,740",
						"to": "00:00:06,410"
					},
					"offsets": {
						"from": 5740,
						"to": 6410
					},
					"id": 1499,
					"p": 0.995797,
					"t_dtw": -1
				},
				{
					"text": " can",
					"timestamps": {
						"from": "00:00:06,410",
						"to": "00:00:06,740"
					},
					"offsets": {
						"from": 6410,
						"to": 6740
					},
					"id": 460,
					"p": 0.977441,
					"t_dtw": -1
				},
				{
					"text": " do",
					"timestamps": {
						"from": "00:00:06,740",
						"to": "00:00:06,920"
					},
					"offsets": {
						"from": 6740,
						"to": 6920
					},
					"id": 466,
					"p": 0.990299,
					"t_dtw": -1
				},
				{
					"text": " for",
					"timestamps": {
						"from": "00:00:07,000",
						"to": "00:00:07,000"
					},
					"offsets": {
						"from": 7000,
						"to": 7000
					},
					"id": 329,
					"p": 0.958497,
					"t_dtw": -1
				},
				{
					"text": " you",
					"timestamps": {
						"from": "00:00:07,010",
						"to": "00:00:07,520"
					},
					"offsets": {
						"from": 7010,
						"to": 7520
					},
					"id": 345,
					"p": 0.985205,
					"t_dtw": -1
				},
				{
					"text": ",",
					"timestamps": {
						"from": "00:00:07,810",
						"to": "00:00:08,050"
					},
					"offsets": {
						"from": 7810,
						"to": 8050
					},
					"id": 11,
					"p": 0.493044,
					"t_dtw": -1
				},
				{
					"text": " ask",
					"timestamps": {
						"from": "00:00:08,190",
						"to": "00:00:08,370"
					},
					"offsets": {
						"from": 8190,
						"to": 8370
					},
					"id": 1265,
					"p": 0.65607,
					"t_dtw": -1
				},
				{
					"text": " what",
					"timestamps": {
						"from": "00:00:08,370",
						"to": "00:00:08,750"
					},
					"offsets": {
						"from": 8370,
						"to": 8750
					},
					"id": 644,
					"p": 0.988193,
					"t_dtw": -1
				},
				{
					"text": " you",
					"timestamps": {
						"from": "00:00:08,910",
						"to": "00:00:09,040"
					},
					"offsets": {
						"from": 8910,
						"to": 9040
					},
					"id": 345,
					"p": 0.981901,
					"t_dtw": -1
				},
				{
					"text": " can",
					"timestamps": {
						"from": "00:00:09,040",
						"to": "00:00:09,320"
					},
					"offsets": {
						"from": 9040,
						"to": 9320
					},
					"id": 460,
					"p": 0.96871,
					"t_dtw": -1
				},
				{
					"text": " do",
					"timestamps": {
						"from": "00:00:09,320",
						"to": "00:00:09,380"
					},
					"offsets": {
						"from": 9320,
						"to": 9380
					},
					"id": 466,
					"p": 0.903065,
					"t_dtw": -1
				},
				{
					"text": " for",
					"timestamps": {
						"from": "00:00:09,440",
						"to": "00:00:09,760"
					},
					"offsets": {
						"from": 9440,
						"to": 9760
					},
					"id": 329,
					"p": 0.918014,
					"t_dtw": -1
				},
				{
					"text": " your",
					"timestamps": {
						"from": "00:00:09,760",
						"to": "00:00:09,990"
					},
					"offsets": {
						"from": 9760,
						"to": 9990
					},
					"id": 534,
					"p": 0.982246,
					"t_dtw": -1
				},
				{
					"text": " country",
					"timestamps": {
						"from": "00:00:10,020",
						"to": "00:00:10,360"
					},
					"offsets": {
						"from": 10020,
						"to": 10360
					},
					"id": 1499,
					"p": 0.995706,
					"t_dtw": -1
				},
				{
					"text": ".",
					"timestamps": {
						"from": "00:00:10,510",
						"to": "00:00:10,990"
					},
					"offsets": {
						"from": 10510,
						"to": 10990
					},
					"id": 13,
					"p": 0.869686,
					"t_dtw": -1
				},
				{
					"text": "[_TT_550]",
					"timestamps": {
						"from": "00:00:11,000",
						"to": "00:00:11,000"
					},
					"offsets": {
						"from": 11000,
						"to": 11000
					},
					"id": 50913,
					"p": 0.0760255,
					"t_dtw": -1
				}
			]
		}
	]
}

It includes from and to of each token (word and period (".")). So, you can calculate "no speech detected" periods and ends of sentence (timing of "."s). You should be able to build output text using these result.

[jlmxyz]

thanks! will try it!