Tobii Gaze Overlay
Last modified by Maarten Struijk Wilbrink on 2026/08/25 15:08
Contents
Introduction
Tobii gazedata can be visualised with a gaze-dot. This is the default behaviour of programs like the Tobii Glasses Controller App, which uses the `.gz` file to map a gazedot onto the `.mp4` videofile.
Use-case
The gaze information is not hardcoded into the video, which is beneficial in cases where the raw video is needed in other situations. In cases where gaze information however should be visible in the video at all times (especially without having access to Tobii Pro Labs), the code below can be used.
Notes
This only works if the video and the `.gz` file are from the same recording, and have not been edited.
Code
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
#!/usr/bin/env python3
"""
Overlay Tobii Glasses 3 gaze data onto scene video.
Usage:
python3 gaze_overlay.py scenevideo.mp4 gazedata.gz output.mp4
Requires: opencv-python, ffmpeg (in PATH)
pip install opencv-python
"""
import bisect
import gzip
import json
import subprocess
import sys
from typing import Any
import cv2
from cv2.typing import MatLike
GazeSample = tuple[float, float, float] # (timestamp, x_norm, y_norm)
def load_gaze(gazedata_path: str) -> list[GazeSample]:
"""Parse gazedata.gz -> sorted list of (timestamp, x_norm, y_norm)."""
samples: list[GazeSample] = []
with gzip.open(gazedata_path, "rt") as f:
for line in f:
line = line.strip()
if not line:
continue
rec: dict[str, Any] = json.loads(line)
if rec.get("type") != "gaze":
continue
data: dict[str, Any] = rec.get("data", {})
g2d: list[float] | None = data.get("gaze2d")
if not g2d or len(g2d) != 2:
continue # missing/invalid sample (e.g. blink) - skip
ts: float = rec["timestamp"]
samples.append((ts, g2d[0], g2d[1]))
samples.sort(key=lambda s: s[0])
return samples
def nearest_gaze(
samples: list[GazeSample], ts_list: list[float], t: float
) -> GazeSample | None:
"""Find gaze sample nearest to time t. Returns None if too far (>0.15s)."""
idx = bisect.bisect_left(ts_list, t)
candidates: list[GazeSample] = []
if idx < len(samples):
candidates.append(samples[idx])
if idx > 0:
candidates.append(samples[idx - 1])
if not candidates:
return None
best = min(candidates, key=lambda s: abs(s[0] - t))
if abs(best[0] - t) > 0.15: # gap too large, don't draw stale dot
return None
return best
def main() -> None:
if len(sys.argv) != 4:
print("Usage: python gaze_overlay.py scenevideo.mp4 gazedata.gz output.mp4")
sys.exit(1)
video_path, gaze_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
silent_out = out_path + ".silent.mp4"
print("Loading gaze data...")
samples = load_gaze(gaze_path)
ts_list = [s[0] for s in samples]
print(f" {len(samples)} valid gaze samples, ")
print(f"span {ts_list[0]:.2f}s - {ts_list[-1]:.2f}s")
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
print(f"ERROR: could not open {video_path}")
sys.exit(1)
fps = cap.get(cv2.CAP_PROP_FPS)
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
n_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
print(f"Video: {width}x{height} @ {fps:.3f}fps, {n_frames} frames")
fourcc = cv2.VideoWriter.fourcc(*"mp4v")
writer = cv2.VideoWriter(silent_out, fourcc, fps, (width, height))
frame_idx = 0
missed = 0
while True:
ret, frame = cap.read()
if not ret:
break
frame: MatLike = frame
t = frame_idx / fps
gaze = nearest_gaze(samples, ts_list, t)
if gaze is not None:
_, gx, gy = gaze
px = int(gx * width)
py = int(gy * height)
# outer ring
_ = cv2.circle(frame, (px, py), 22, (0, 0, 255), 3)
# center dot
_ = cv2.circle(frame, (px, py), 6, (0, 0, 255), -1)
else:
missed += 1
_ = writer.write(frame)
frame_idx += 1
if frame_idx % 200 == 0:
print(f" frame {frame_idx}/{n_frames}")
cap.release()
writer.release()
print(f"Done. {missed} frames had no gaze sample within 0.15s.")
print("Muxing original audio back in via ffmpeg...")
cmd = [
"ffmpeg",
"-y",
"-i",
silent_out,
"-i",
video_path,
"-c:v",
"copy",
"-map",
"0:v:0",
"-map",
"1:a:0?",
"-shortest",
out_path,
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
print(
"ffmpeg mux failed, output has no audio. Silent video kept at:", silent_out
)
print(result.stderr[-1500:])
else:
print(f"Final output with audio: {out_path}")
if __name__ == "__main__":
main()
"""
Overlay Tobii Glasses 3 gaze data onto scene video.
Usage:
python3 gaze_overlay.py scenevideo.mp4 gazedata.gz output.mp4
Requires: opencv-python, ffmpeg (in PATH)
pip install opencv-python
"""
import bisect
import gzip
import json
import subprocess
import sys
from typing import Any
import cv2
from cv2.typing import MatLike
GazeSample = tuple[float, float, float] # (timestamp, x_norm, y_norm)
def load_gaze(gazedata_path: str) -> list[GazeSample]:
"""Parse gazedata.gz -> sorted list of (timestamp, x_norm, y_norm)."""
samples: list[GazeSample] = []
with gzip.open(gazedata_path, "rt") as f:
for line in f:
line = line.strip()
if not line:
continue
rec: dict[str, Any] = json.loads(line)
if rec.get("type") != "gaze":
continue
data: dict[str, Any] = rec.get("data", {})
g2d: list[float] | None = data.get("gaze2d")
if not g2d or len(g2d) != 2:
continue # missing/invalid sample (e.g. blink) - skip
ts: float = rec["timestamp"]
samples.append((ts, g2d[0], g2d[1]))
samples.sort(key=lambda s: s[0])
return samples
def nearest_gaze(
samples: list[GazeSample], ts_list: list[float], t: float
) -> GazeSample | None:
"""Find gaze sample nearest to time t. Returns None if too far (>0.15s)."""
idx = bisect.bisect_left(ts_list, t)
candidates: list[GazeSample] = []
if idx < len(samples):
candidates.append(samples[idx])
if idx > 0:
candidates.append(samples[idx - 1])
if not candidates:
return None
best = min(candidates, key=lambda s: abs(s[0] - t))
if abs(best[0] - t) > 0.15: # gap too large, don't draw stale dot
return None
return best
def main() -> None:
if len(sys.argv) != 4:
print("Usage: python gaze_overlay.py scenevideo.mp4 gazedata.gz output.mp4")
sys.exit(1)
video_path, gaze_path, out_path = sys.argv[1], sys.argv[2], sys.argv[3]
silent_out = out_path + ".silent.mp4"
print("Loading gaze data...")
samples = load_gaze(gaze_path)
ts_list = [s[0] for s in samples]
print(f" {len(samples)} valid gaze samples, ")
print(f"span {ts_list[0]:.2f}s - {ts_list[-1]:.2f}s")
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
print(f"ERROR: could not open {video_path}")
sys.exit(1)
fps = cap.get(cv2.CAP_PROP_FPS)
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
n_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
print(f"Video: {width}x{height} @ {fps:.3f}fps, {n_frames} frames")
fourcc = cv2.VideoWriter.fourcc(*"mp4v")
writer = cv2.VideoWriter(silent_out, fourcc, fps, (width, height))
frame_idx = 0
missed = 0
while True:
ret, frame = cap.read()
if not ret:
break
frame: MatLike = frame
t = frame_idx / fps
gaze = nearest_gaze(samples, ts_list, t)
if gaze is not None:
_, gx, gy = gaze
px = int(gx * width)
py = int(gy * height)
# outer ring
_ = cv2.circle(frame, (px, py), 22, (0, 0, 255), 3)
# center dot
_ = cv2.circle(frame, (px, py), 6, (0, 0, 255), -1)
else:
missed += 1
_ = writer.write(frame)
frame_idx += 1
if frame_idx % 200 == 0:
print(f" frame {frame_idx}/{n_frames}")
cap.release()
writer.release()
print(f"Done. {missed} frames had no gaze sample within 0.15s.")
print("Muxing original audio back in via ffmpeg...")
cmd = [
"ffmpeg",
"-y",
"-i",
silent_out,
"-i",
video_path,
"-c:v",
"copy",
"-map",
"0:v:0",
"-map",
"1:a:0?",
"-shortest",
out_path,
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
print(
"ffmpeg mux failed, output has no audio. Silent video kept at:", silent_out
)
print(result.stderr[-1500:])
else:
print(f"Final output with audio: {out_path}")
if __name__ == "__main__":
main()