msm: vidc: Allow clients to specify/query the colorspaceandroid-6.0.0_r0.7

When encoder clients specify the color space, the resulting bitstream
will have the color space embedded in it.  This allows for the
decoder to display the picture accurately without the color being
skewed.

Similarly decoder clients can query the color space of the bitstream by
enabling MSM_VIDC_EXTRADATA_VUI_DISPLAY_INFO.  If the bitstream contains
the colorspace hint, the appropriate extradata will be emitted by the
decoder.

Bug: 24444518
Change-Id: I83a3e28b13c3aa306e7cc2d1552a6d39c0d3d49f
Signed-off-by: Deva Ramasubramanian <dramasub@codeaurora.org>
Signed-off-by: Praveen Chavan <pchavan@codeaurora.org>
Signed-off-by: Mekala Natarajan <mekalan@codeaurora.org>

9 files changed, 147 insertions, 8 deletions

diff --git a/drivers/media/platform/msm/vidc/hfi_packetization.c b/drivers/media/platform/msm/vidc/hfi_packetization.c
index 7a82e9772719..926c56183544 100644
--- a/drivers/media/platform/msm/vidc/hfi_packetization.c
+++ b/drivers/media/platform/msm/vidc/hfi_packetization.c
@@ -483,6 +483,9 @@ static int get_hfi_extradata_index(enum hal_extradata_id index)
 	case HAL_EXTRADATA_METADATA_MBI:
 		ret = HFI_PROPERTY_PARAM_VENC_MBI_DUMPING;
 		break;
+	case HAL_EXTRADATA_VUI_DISPLAY_INFO:
+		ret = HFI_PROPERTY_PARAM_VUI_DISPLAY_INFO_EXTRADATA;
+		break;
 	default:
 		dprintk(VIDC_WARN, "Extradata index not found: %d\n", index);
 		break;
@@ -1850,6 +1853,43 @@ int create_pkt_cmd_session_set_property(
 			sizeof(struct hfi_hybrid_hierp);
 		break;
 	}
+	case HAL_PARAM_VENC_VIDEO_SIGNAL_INFO:
+	{
+		u32 color_space, matrix_coeffs, transfer_chars;
+		struct hal_video_signal_info *hal = pdata;
+		struct hfi_video_signal_metadata *signal_info =
+			(struct hfi_video_signal_metadata *)
+			&pkt->rg_property_data[1];
+
+		switch (hal->color_space) {
+		/* See colour_primaries of ISO/IEC 14496 for significance */
+		case HAL_VIDEO_COLOR_SPACE_601:
+			color_space = 5;
+			transfer_chars = 6;
+			matrix_coeffs = 5;
+			break;
+		case HAL_VIDEO_COLOR_SPACE_709:
+			color_space = 1;
+			transfer_chars = 1;
+			matrix_coeffs = 1;
+			break;
+		default:
+			return -ENOTSUPP;
+		}
+
+		signal_info->enable = true;
+		signal_info->video_format = 5;
+		signal_info->video_full_range = !hal->clamped;
+		signal_info->color_description = 1;
+		signal_info->color_primaries = color_space;
+		signal_info->transfer_characteristics = transfer_chars;
+		signal_info->matrix_coeffs = matrix_coeffs;
+
+		pkt->rg_property_data[0] =
+			HFI_PROPERTY_PARAM_VENC_VIDEO_SIGNAL_INFO;
+		pkt->size += sizeof(u32) + sizeof(*signal_info);
+		break;
+	}
 	/* FOLLOWING PROPERTIES ARE NOT IMPLEMENTED IN CORE YET */
 	case HAL_CONFIG_BUFFER_REQUIREMENTS:
 	case HAL_CONFIG_PRIORITY:
diff --git a/drivers/media/platform/msm/vidc/msm_vdec.c b/drivers/media/platform/msm/vidc/msm_vdec.c
index 4a9adeeeba58..0b3be2b3156e 100644
--- a/drivers/media/platform/msm/vidc/msm_vdec.c
+++ b/drivers/media/platform/msm/vidc/msm_vdec.c
@@ -62,8 +62,6 @@ static const char *const mpeg_video_vidc_extradata[] = {
 	"Extradata Frame Rate",
 	"Extradata Panscan Window",
 	"Extradata Recovery point SEI",
-	"Extradata Closed Caption UD",
-	"Extradata AFD UD",
 	"Extradata Multislice info",
 	"Extradata number of concealed MB",
 	"Extradata metadata filler",
@@ -71,6 +69,10 @@ static const char *const mpeg_video_vidc_extradata[] = {
 	"Extradata digital zoom",
 	"Extradata aspect ratio",
 	"Extradata mpeg2 seqdisp",
+	"Extradata UD",
+	"Extradata frame QP",
+	"Extradata frame bits info",
+	"Extradata display VUI",
 };
 static const char *const mpeg_vidc_video_alloc_mode_type[] = {
 	"Buffer Allocation Static",
@@ -247,7 +249,7 @@ static struct msm_vidc_ctrl msm_vdec_ctrls[] = {
 		.name = "Extradata Type",
 		.type = V4L2_CTRL_TYPE_MENU,
 		.minimum = V4L2_MPEG_VIDC_EXTRADATA_NONE,
-		.maximum = V4L2_MPEG_VIDC_EXTRADATA_FRAME_BITS_INFO,
+		.maximum = V4L2_MPEG_VIDC_EXTRADATA_VUI_DISPLAY,
 		.default_value = V4L2_MPEG_VIDC_EXTRADATA_NONE,
 		.menu_skip_mask = ~(
 			(1 << V4L2_MPEG_VIDC_EXTRADATA_NONE) |
@@ -269,7 +271,8 @@ static struct msm_vidc_ctrl msm_vdec_ctrls[] = {
 			(1 << V4L2_MPEG_VIDC_EXTRADATA_MPEG2_SEQDISP) |
 			(1 << V4L2_MPEG_VIDC_EXTRADATA_STREAM_USERDATA) |
 			(1 << V4L2_MPEG_VIDC_EXTRADATA_FRAME_QP) |
-			(1 << V4L2_MPEG_VIDC_EXTRADATA_FRAME_BITS_INFO)
+			(1 << V4L2_MPEG_VIDC_EXTRADATA_FRAME_BITS_INFO) |
+			(1 << V4L2_MPEG_VIDC_EXTRADATA_VUI_DISPLAY)
 			),
 		.qmenu = mpeg_video_vidc_extradata,
 		.step = 0,
diff --git a/drivers/media/platform/msm/vidc/msm_venc.c b/drivers/media/platform/msm/vidc/msm_venc.c
index 8a7daa1b21a3..3e8512036f0d 100644
--- a/drivers/media/platform/msm/vidc/msm_venc.c
+++ b/drivers/media/platform/msm/vidc/msm_venc.c
@@ -3180,6 +3180,7 @@ int msm_venc_s_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f)
 	} else if (f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
 		struct hal_uncompressed_format_select hal_fmt = {0};
 		struct hal_frame_size frame_sz;
+		struct hal_video_signal_info signal_info = {0};
 
 		inst->prop.width[OUTPUT_PORT] = f->fmt.pix_mp.width;
 		inst->prop.height[OUTPUT_PORT] = f->fmt.pix_mp.height;
@@ -3191,6 +3192,7 @@ int msm_venc_s_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f)
 			goto exit;
 		}
 
+		/* Configure frame dimensions */
 		frame_sz.buffer_type = HAL_BUFFER_INPUT;
 		frame_sz.width = inst->prop.width[OUTPUT_PORT];
 		frame_sz.height = inst->prop.height[OUTPUT_PORT];
@@ -3213,6 +3215,7 @@ int msm_venc_s_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f)
 			goto exit;
 		}
 
+		/* Configure frame color format */
 		fmt = msm_comm_get_pixel_fmt_fourcc(venc_formats,
 			ARRAY_SIZE(venc_formats), f->fmt.pix_mp.pixelformat,
 			OUTPUT_PORT);
@@ -3246,6 +3249,54 @@ int msm_venc_s_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f)
 				"Failed to set input color format\n");
 			goto exit;
 		}
+
+		/* Configure frame color format characteristics */
+		if (f->fmt.pix_mp.colorspace) {
+			switch (f->fmt.pix_mp.colorspace) {
+				case V4L2_COLORSPACE_REC709:
+					signal_info.color_space =
+						HAL_VIDEO_COLOR_SPACE_709;
+					signal_info.clamped = true;
+					break;
+				case V4L2_COLORSPACE_BT878:
+					/* equiv to ITU-R BT.601 clamped */
+					signal_info.clamped = true;
+					/* fall thru */
+				case V4L2_COLORSPACE_470_SYSTEM_BG:
+					/* equiv to ITU-R BT.601 */
+					signal_info.color_space =
+						HAL_VIDEO_COLOR_SPACE_601;
+					break;
+				default:
+					dprintk(VIDC_ERR, "Colorspace %d not supported\n",
+							f->fmt.pix_mp.colorspace);
+					rc = -ENOTSUPP;
+					goto exit;
+			}
+
+			switch (f->fmt.pix_mp.pixelformat) {
+				case V4L2_PIX_FMT_NV12:
+				case V4L2_PIX_FMT_NV21:
+					rc = call_hfi_op(hdev, session_set_property,
+							inst->session,
+							HAL_PARAM_VENC_VIDEO_SIGNAL_INFO,
+							&signal_info);
+					if (rc) {
+						dprintk(VIDC_ERR,
+								"Failed to set the colorspace: %d\n",
+								rc);
+						goto exit;
+					}
+					break;
+				default:
+					dprintk(VIDC_ERR,
+							"Colorspace %d not supported for format %d\n",
+							f->fmt.pix_mp.colorspace,
+							f->fmt.pix_mp.pixelformat);
+					rc = -ENOTSUPP;
+					break;
+			}
+		}
 	}
 
 	if (!fmt) {
@@ -3277,9 +3328,8 @@ int msm_venc_s_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f)
 		frame_sz.width = inst->prop.width[CAPTURE_PORT];
 		frame_sz.height = inst->prop.height[CAPTURE_PORT];
 		frame_sz.buffer_type = HAL_BUFFER_OUTPUT;
-		rc = call_hfi_op(hdev, session_set_property, (void *)
-				inst->session, HAL_PARAM_FRAME_SIZE,
-				&frame_sz);
+		rc = call_hfi_op(hdev, session_set_property, inst->session,
+				HAL_PARAM_FRAME_SIZE, &frame_sz);
 		if (rc) {
 			dprintk(VIDC_ERR,
 					"Failed to set OUTPUT framesize\n");
diff --git a/drivers/media/platform/msm/vidc/msm_vidc_common.c b/drivers/media/platform/msm/vidc/msm_vidc_common.c
index 32f78826547b..6360b89396a2 100644
--- a/drivers/media/platform/msm/vidc/msm_vidc_common.c
+++ b/drivers/media/platform/msm/vidc/msm_vidc_common.c
@@ -3946,6 +3946,9 @@ enum hal_extradata_id msm_comm_get_hal_extradata_index(
 	case V4L2_MPEG_VIDC_EXTRADATA_METADATA_MBI:
 		ret = HAL_EXTRADATA_METADATA_MBI;
 		break;
+	case V4L2_MPEG_VIDC_EXTRADATA_VUI_DISPLAY:
+		ret = HAL_EXTRADATA_VUI_DISPLAY_INFO;
+		break;
 	default:
 		dprintk(VIDC_WARN, "Extradata not found: %d\n", index);
 		break;
diff --git a/drivers/media/platform/msm/vidc/vidc_hfi.h b/drivers/media/platform/msm/vidc/vidc_hfi.h
index 8eefc7cee616..5c9c4b2a9bfa 100644
--- a/drivers/media/platform/msm/vidc/vidc_hfi.h
+++ b/drivers/media/platform/msm/vidc/vidc_hfi.h
@@ -212,6 +212,8 @@ struct hfi_extradata_header {
 	(HFI_PROPERTY_PARAM_VDEC_OX_START + 0x019)
 #define HFI_PROPERTY_PARAM_VDEC_SCS_THRESHOLD \
 	(HFI_PROPERTY_PARAM_VDEC_OX_START + 0x01A)
+#define HFI_PROPERTY_PARAM_VUI_DISPLAY_INFO_EXTRADATA \
+	(HFI_PROPERTY_PARAM_VDEC_OX_START + 0x01B)
 
 #define HFI_PROPERTY_CONFIG_VDEC_OX_START				\
 	(HFI_DOMAIN_BASE_VDEC + HFI_ARCH_OX_OFFSET + 0x0000)
diff --git a/drivers/media/platform/msm/vidc/vidc_hfi_api.h b/drivers/media/platform/msm/vidc/vidc_hfi_api.h
index 154f06e8d5b9..5e82547e5c2d 100644
--- a/drivers/media/platform/msm/vidc/vidc_hfi_api.h
+++ b/drivers/media/platform/msm/vidc/vidc_hfi_api.h
@@ -114,6 +114,7 @@ enum hal_extradata_id {
 	HAL_EXTRADATA_DIGITAL_ZOOM,
 	HAL_EXTRADATA_LTR_INFO,
 	HAL_EXTRADATA_METADATA_MBI,
+	HAL_EXTRADATA_VUI_DISPLAY_INFO,
 };
 
 enum hal_property {
@@ -214,6 +215,7 @@ enum hal_property {
 	HAL_PARAM_VENC_HIER_B_MAX_ENH_LAYERS,
 	HAL_PARAM_VDEC_NON_SECURE_OUTPUT2,
 	HAL_PARAM_VENC_HIER_P_HYBRID_MODE,
+	HAL_PARAM_VENC_VIDEO_SIGNAL_INFO,
 };
 
 enum hal_domain {
@@ -924,6 +926,16 @@ struct hal_vpe_color_space_conversion {
 	u32 csc_limit[HAL_MAX_LIMIT_COEFFS];
 };
 
+enum hal_video_color_space {
+	HAL_VIDEO_COLOR_SPACE_601,
+	HAL_VIDEO_COLOR_SPACE_709,
+};
+
+struct hal_video_signal_info {
+	enum hal_video_color_space color_space;
+	bool clamped;
+};
+
 enum vidc_resource_id {
 	VIDC_RESOURCE_OCMEM = 0x00000001,
 	VIDC_UNUSED_RESORUCE = 0x10000000,
diff --git a/drivers/media/platform/msm/vidc/vidc_hfi_helper.h b/drivers/media/platform/msm/vidc/vidc_hfi_helper.h
index e6b58a098673..8b981bd1578f 100644
--- a/drivers/media/platform/msm/vidc/vidc_hfi_helper.h
+++ b/drivers/media/platform/msm/vidc/vidc_hfi_helper.h
@@ -338,7 +338,7 @@ struct hfi_buffer_info {
 	(HFI_PROPERTY_PARAM_VENC_COMMON_START + 0x01B)
 #define HFI_PROPERTY_PARAM_VENC_LTRMODE		\
 	 (HFI_PROPERTY_PARAM_VENC_COMMON_START + 0x01C)
-#define HFI_PROPERTY_PARAM_VENC_VIDEO_FULL_RANGE	\
+#define HFI_PROPERTY_PARAM_VENC_VIDEO_SIGNAL_INFO	\
 	(HFI_PROPERTY_PARAM_VENC_COMMON_START + 0x01D)
 #define HFI_PROPERTY_PARAM_VENC_H264_VUI_TIMING_INFO	\
 	(HFI_PROPERTY_PARAM_VENC_COMMON_START + 0x01E)
@@ -626,6 +626,16 @@ struct hfi_frame_size {
 	u32 height;
 };
 
+struct hfi_video_signal_metadata {
+	u32 enable;
+	u32 video_format;
+	u32 video_full_range;
+	u32 color_description;
+	u32 color_primaries;
+	u32 transfer_characteristics;
+	u32 matrix_coeffs;
+};
+
 struct hfi_h264_vui_timing_info {
 	u32 enable;
 	u32 fixed_frame_rate;
diff --git a/include/uapi/linux/v4l2-controls.h b/include/uapi/linux/v4l2-controls.h
index 4f944f5ba66e..cedad8eb1475 100644
--- a/include/uapi/linux/v4l2-controls.h
+++ b/include/uapi/linux/v4l2-controls.h
@@ -739,6 +739,7 @@ enum v4l2_mpeg_vidc_extradata {
 	V4L2_MPEG_VIDC_EXTRADATA_FRAME_BITS_INFO = 19,
 	V4L2_MPEG_VIDC_EXTRADATA_LTR = 20,
 	V4L2_MPEG_VIDC_EXTRADATA_METADATA_MBI = 21,
+	V4L2_MPEG_VIDC_EXTRADATA_VUI_DISPLAY = 22,
 };
 
 #define V4L2_CID_MPEG_VIDC_SET_PERF_LEVEL (V4L2_CID_MPEG_MSM_VIDC_BASE + 26)
diff --git a/include/uapi/media/msm_vidc.h b/include/uapi/media/msm_vidc.h
index 6fa53ad7997b..26fd3ad6947e 100644
--- a/include/uapi/media/msm_vidc.h
+++ b/include/uapi/media/msm_vidc.h
@@ -109,6 +109,23 @@ struct msm_vidc_s3d_frame_packing_payload {
 	unsigned int fpa_extension_flag;
 };
 
+struct msm_vidc_vui_display_info_payload {
+	unsigned int video_signal_present_flag;
+	unsigned int video_format;
+	unsigned int bit_depth_y;
+	unsigned int bit_depth_c;
+	unsigned int video_full_range_flag;
+	unsigned int color_description_present_flag;
+	unsigned int color_primaries;
+	unsigned int transfer_characteristics;
+	unsigned int matrix_coefficients;
+	unsigned int chroma_location_info_present_flag;
+	unsigned int chroma_format_idc;
+	unsigned int separate_color_plane_flag;
+	unsigned int chroma_sample_loc_type_top_field;
+	unsigned int chroma_sample_loc_type_bottom_field;
+};
+
 enum msm_vidc_extradata_type {
 	MSM_VIDC_EXTRADATA_NONE = 0x00000000,
 	MSM_VIDC_EXTRADATA_MB_QUANTIZATION = 0x00000001,
@@ -133,6 +150,7 @@ enum msm_vidc_extradata_type {
 	MSM_VIDC_EXTRADATA_METADATA_LTR = 0x7F100004,
 	MSM_VIDC_EXTRADATA_METADATA_FILLER = 0x7FE00002,
 	MSM_VIDC_EXTRADATA_METADATA_MBI = 0x7F100005,
+	MSM_VIDC_EXTRADATA_VUI_DISPLAY_INFO = 0x7F100006,
 };
 enum msm_vidc_interlace_type {
 	MSM_VIDC_INTERLACE_FRAME_PROGRESSIVE = 0x01,