🗄
Gyroflow protobuf
Gyroflow protobuf format is designed to contain even the most advanced and detailed data about the video capture pipeline, that is useful for post-stabilization.
You can download the latest protobuf definition from telemetry-parser's git repository.
TODO
- Camera and lens metadata: Brand, model, focal length, f-number, focus distance etc.
- Lens distortion model and coefficients
- Frame readout time for rolling shutter correction
- Frame capture metadata - ISO, shutter speed, white balance, etc.
- Raw IMU samples - gyroscope, accelerometer, magnetometer readings
- Quaternions - final camera orientation after sensor fusion
- Lens OIS data - detailed information about lens OIS movements so we can stabilization when Lens OIS was enabled
- IBIS data - detailed information about the in-body image stabilization, so we can support stabilization when IBIS was enabled
- EIS data - if the camera contains any form of electronic stabilization, the protobuf can contain what exactly it did to the image so we can account for it.
Typically the protobuf will be stored as binary data in a separate MP4 track of the video file. This makes it easy to read and write and it's the standard way to embed additional data in video files.
TODO: examples and screenshots
TODO
Most fields in the gyroflow.proto have comments so the file should be self-explanatory. However, here are some additional details about more advanced features:
TODO: diagram and description
TODO: timestamp_offset_us
TODO: diagram with the typical design and the values we can use
TODO: diagram with the typical design and the values we can use
TODO: possible types and diagrams with examples
TODO
TODO
TODO
gyroflow.proto
1
syntax = "proto3";
2
3
// Main entry point of the data
4
// The first message will contain the Header with CameraMetadata and ClipMetadata
5
// All subsequent per-frame samples will contain the FrameMetadata, without Header
6
message Main {
7
string magic_string = 1; // Magic string useful for format detection in binary data. Always "GyroflowProtobuf"
8
uint32 protocol_version = 2; // Version of the protocol, currently 1.
9
10
Header header = 3;
11
FrameMetadata frame = 4;
12
}
13
14
// One-time metadata containing information about the camera, lens and this particular video clip
15
message Header {
16
message CameraMetadata {
17
string camera_brand = 1; // Camera manufacturer
18
string camera_model = 2; // Camera model
19
optional string camera_serial_number = 3; // Camera serial number
20
optional string firmware_version = 4; // Camera firmware version
21
string lens_brand = 5; // Lens manufacturer
22
string lens_model = 6; // Lens model
23
float sensor_width_mm = 7; // Physical sensor width in millimeters
24
float sensor_height_mm = 8; // Physical sensor height in millimeters
25
uint32 sensor_pixel_width = 9; // Full sensor width in pixels
26
uint32 sensor_pixel_height = 10; // Full sensor height in pixels
27
optional float crop_factor = 11; // Crop factor in relation to full frame sensor size. e.g. 1.6x for APS-C
28
optional string lens_profile = 12; // The Gyroflow lens identifier, or a path to lens profile json file (relative to the `camera_presets` directory), or the json contents directly
29
optional string imu_orientation = 13; // IMU orientation used by Gyroflow as XYZ, Xyz, Zyx etc. Defaults to "XYZ". Read more in the Gyroflow documentation about this orientation convention.
30
optional Quaternion imu_rotation = 14; // Arbitrary IMU rotation. Applies to the raw IMU samples (FrameMetadata.imu field).
31
optional Quaternion quats_rotation = 15; // Arbitrary IMU rotation. Applies to the quaternions after sensor fusion (FrameMetadata.quaternions field).
32
optional string additional_data = 16; // Optional note or additional data. If it starts with {, it will be parsed as JSON
33
}
34
message ClipMetadata {
35
enum ReadoutDirection {
36
TopToBottom = 0; // Sensor reads pixels from top to bottom.
37
BottomToTop = 1; // Sensor reads pixels from bottom to top.
38
RightToLeft = 2; // Sensor reads pixels from right to left. !!! Not implemented yet !!!
39
LeftToRight = 3; // Sensor reads pixels from left to right. !!! Not implemented yet !!!
40
}
41
42
uint32 frame_width = 1; // Video frame width in pixels
43
uint32 frame_height = 2; // Video frame height in pixels
44
float duration_us = 3; // Clip duration in microseconds
45
float record_frame_rate = 4; // Recording frame rate
46
float sensor_frame_rate = 5; // Sensor frame rate. In most cases it will be equal to `record_frame_rate`
47
float file_frame_rate = 6; // File frame rate. May be different in VFR mode. e.g. 120 fps recorded as 30 fps file
48
int32 rotation_degrees = 7; // Video rotation in degrees. For example 180 degrees for upside-down, or 90 for vertical mode.
49
uint32 imu_sample_rate = 8; // Sampling rate of the IMU chip.
50
optional string color_profile = 9; // Shooting color profile, eg. Natural, Log, etc
51
float pixel_aspect_ratio = 10; // For anamorphic lenses
52
float frame_readout_time_us = 11; // Time it takes to read the video frame from the sensor, for rolling shutter correction. NOTE: It should be the time between first row of pixels to the last row of pixels, not for full sensor readout (if the crop is involved).
53
ReadoutDirection frame_readout_direction = 12; // Frame readout direction
54
}
55
56
CameraMetadata camera = 1;
57
ClipMetadata clip = 2;
58
}
59
60
message FrameMetadata {
61
double start_timestamp_us = 1; // Frame capture start - the timestamp when the first row of pixels was captured. Internal camera clock timestamp. Unit: microseconds
62
double end_timestamp_us = 2; // Frame capture end - the timestamp when the last row of pixels was captured. Internal camera clock timestamp. Unit: microseconds
63
uint32 frame_number = 3; // Frame number in sequence. The first frame of the video clip should have this set to 1.
64
65
optional uint32 iso = 4; // ISO Value
66
optional float exposure_time_us = 5; // Actual exposure time in microseconds
67
optional uint32 white_balance_kelvin = 6; // White balance in kelvins
68
optional float white_balance_tint = 7; // White balance tint value
69
optional float digital_zoom_ratio = 8; // Digital zoom ratio. If the video is zoomed in digitally, this value should indicate that. E.g. 0.9 for 10% digital crop
70
optional int32 shutter_speed_numerator = 9; // Shutter speed numerator. E.g. 1 in case of 1/240 shutter speed.
71
optional int32 shutter_speed_denumerator = 10; // Shutter speed denumerator. E.g. 240 in case of 1/240 shutter speed.
72
optional float shutter_angle_degrees = 11; // Shutter angle in degrees. E.g. 180
73
74
repeated LensData lens = 12; // Per-frame lens information, like focal length, distortion coefficients etc
75
repeated IMUData imu = 13; // Per-frame raw IMU data samples, will likely have multiple samples in one video frame
76
repeated QuaternionData quaternions = 14; // Per-frame quaternion data. Optional, can contain camera orientation after sensor fusion
77
repeated LensOISData ois = 15; // Per-frame Lens optical stabilization data. Not present when OIS is disabled. ??? Exact data and format to be determined ???
78
repeated IBISData ibis = 16; // Per-frame in-body image stabilization (IBIS) data. Not present when IBIS is disabled. ??? Exact data and format to be determined ???
79
repeated EISData eis = 17; // Per-frame electronic in-camera stabilization data. Not present when EIS is disabled. ??? Exact data and format to be determined ???
80
}
81
82
message LensData {
83
enum DistortionModel {
84
OpenCVFisheye = 0; // OpenCV's fisheye model. More details: https://docs.opencv.org/4.x/db/d58/group__calib3d__fisheye.html
85
OpenCVStandard = 1; // OpenCV's standard model. More details: https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html
86
Poly3 = 2; // LensFun's Poly3 model. More details: https://lensfun.github.io/manual/latest/group__Lens.html#gaa505e04666a189274ba66316697e308e
87
Poly5 = 3; // LensFun's Poly5 model. More details: https://lensfun.github.io/manual/latest/group__Lens.html#gaa505e04666a189274ba66316697e308e
88
PTLens = 4; // LensFun's PTLens model. More details: https://lensfun.github.io/manual/latest/group__Lens.html#gaa505e04666a189274ba66316697e308e
89
GenericPolynomial = 5; // ??? Not implemented yet. ???
90
}
91
DistortionModel distortion_model = 1;
92
repeated float distortion_coefficients = 2; // Distortion model coefficients as an array of float values.
93
repeated float camera_intrinsic_matrix = 3; // Row-major 3x3 camera intrinsic matrix. Usually [[fx, 0, cx], [0, fy, cy], [0, 0, 1]], where fx and fy are focal length values in pixels (f_mm = f_pixels * sensor_width_mm / image_width_px ; f_pixels = f_mm / sensor_width_mm * image_width_px), and cx and cy is the principal point in pixels (usually width/2, height/2).
94
float focal_length_mm = 4; // Native lens focal length in mm
95
float f_number = 5; // Lens aperture number. E.g. 2.8
96
float focus_distance_mm = 6; // Focal plane distance in millimeters
97
}
98
99
message IMUData {
100
double sample_timestamp_us = 1; // Exact timestamp of the sampling time from the internal camera clock. Unit: microseconds
101
float gyroscope_x = 2; // Gyroscope X reading. Unit: degrees/sec
102
float gyroscope_y = 3; // Gyroscope Y reading. Unit: degrees/sec
103
float gyroscope_z = 4; // Gyroscope Z reading. Unit: degrees/sec
104
float accelerometer_x = 5; // Accelerometer X reading. Unit: m/s²
105
float accelerometer_y = 6; // Accelerometer Y reading. Unit: m/s²
106
float accelerometer_z = 7; // Accelerometer Z reading. Unit: m/s²
107
optional float magnetometer_x = 8; // Magnetometer X reading. Unit: µT
108
optional float magnetometer_y = 9; // Magnetometer Y reading. Unit: µT
109
optional float magnetometer_z = 10; // Magnetometer Z reading. Unit: µT
110
}
111
112
message Quaternion {
113
float w = 1; // Quaternion component W (angle)
114
float x = 2; // Quaternion component X
115
float y = 3; // Quaternion component Y
116
float z = 4; // Quaternion component Z
117
}
118
119
message QuaternionData {
120
double sample_timestamp_us = 1; // Exact timestamp of the sampling time from the internal camera clock. Unit: microseconds
121
Quaternion quat = 2; // Quaternion
122
}
123
124
message LensOISData {
125
double sample_timestamp_us = 1; // Exact timestamp of the sampling time from the internal camera clock. Unit: microseconds
126
float x = 3; // Optical element shift value in the X axis. ??? Units to be determined ???
127
float y = 4; // Optical element shift value in the Y axis. ??? Units to be determined ???
128
}
129
130
message IBISData {
131
enum IBISDataType {
132
IBIS_3AXIS = 0; // IBIS mode 3-axis: X and Y shift + Roll angle.
133
IBIS_5AXIS = 1; // IBIS mode 5-axis: X and Y shift + Pitch/Roll/Yaw angles.
134
}
135
double sample_timestamp_us = 1; // Exact timestamp of the sampling time from the internal camera clock. Unit: microseconds
136
IBISDataType type = 2; // IBIS mode: 3-axis or 5-axis
137
float shift_x = 3; // X Sensor shift value. ??? Units to be determined ???
138
float shift_y = 4; // Y Sensor shift value. ??? Units to be determined ???
139
float roll_angle_degrees = 5; // Sensor roll rotation angle in degrees.
140
}
141
142
message EISData {
143
enum EISDataType {
144
QUATERNION = 0; // Rotation only, indicates how the frame was rotated internally by the camera EIS, from pixels read from the sensor to the final pixels in the encoded video file.
145
MESH_WARP = 1; // Mesh warp. Allows for arbitrary mapping of the video frame. Contains exact transform/deform of the video frame read from the sensor to the final pixels in the encoded video file.
146
MATRIX_4X4 = 2; // 4x4 matrix - rotation, translation and scaling. Indicates how the frame was transformed in the 3d space by the camera EIS, from pixels read from the sensor to the final pixels in the encoded video file.
147
}
148
optional double sample_timestamp_us = 1; // Exact timestamp of the sampling time from the internal camera clock. Unit: microseconds. Timestamp is ignored if there's only one entry of EISData per frame.
149
EISDataType type = 2; // Type of EIS. Can be quaternion, mesh warp or 4x4 transform matrix.
150
Quaternion quaternion = 3; // If type is QUATERNION, this field contains the quaternion data
151
MeshWarpData mesh_warp = 4; // If type is MESH_WARP, this field contains the mesh values
152
repeated float matrix_4x4 = 5; // If type is MATRIX_4x4, this field contains the 16 float matrix values (row-major order).
153
}
154
155
message MeshWarpData {
156
int32 grid_width = 1; // Number of video frame divisions in the horizontal direction.
157
int32 grid_height = 2; // Number of video frame divisions in the vertical direction.
158
repeated float values = 3; // grid_width * grid_height float numbers representing new position of a coordinate at X and Y grid position.
159
}
Last modified 9d ago