V4L/DVB (8920): cx18/ivtv: fix check of window boundaries for VIDIOC_S_FMT

It was possible to set out-of-bounds windows sizes, this is now
fixed.

Signed-off-by: Hans Verkuil <hverkuil@xs4all.nl>
Signed-off-by: Mauro Carvalho Chehab <mchehab@redhat.com>

drivers/media/video/cx18/cx18-ioctl.c

@@ -171,7 +171,6 @@ static int cx18_try_fmt_vid_cap(struct file *file, void *fh,
 {
 	struct cx18_open_id *id = fh;
 	struct cx18 *cx = id->cx;
-
 	int w = fmt->fmt.pix.width;
 	int h = fmt->fmt.pix.height;
 
@@ -203,8 +202,7 @@ static int cx18_s_fmt_vid_cap(struct file *file, void *fh,
 	struct cx18_open_id *id = fh;
 	struct cx18 *cx = id->cx;
 	int ret;
-	int w = fmt->fmt.pix.width;
-	int h = fmt->fmt.pix.height;
+	int w, h;
 
 	ret = v4l2_prio_check(&cx->prio, &id->prio);
 	if (ret)
@@ -213,6 +211,8 @@ static int cx18_s_fmt_vid_cap(struct file *file, void *fh,
 	ret = cx18_try_fmt_vid_cap(file, fh, fmt);
 	if (ret)
 		return ret;
+	w = fmt->fmt.pix.width;
+	h = fmt->fmt.pix.height;
 
 	if (cx->params.width == w && cx->params.height == h)
 		return 0;

drivers/media/video/ivtv/ivtv-ioctl.c

@@ -512,27 +512,20 @@ static int ivtv_try_fmt_sliced_vbi_cap(struct file *file, void *fh, struct v4l2_
 static int ivtv_try_fmt_vid_out(struct file *file, void *fh, struct v4l2_format *fmt)
 {
 	struct ivtv_open_id *id = fh;
-	s32 w, h;
-	int field;
-	int ret;
+	s32 w = fmt->fmt.pix.width;
+	s32 h = fmt->fmt.pix.height;
+	int field = fmt->fmt.pix.field;
+	int ret = ivtv_g_fmt_vid_out(file, fh, fmt);
 
-	w = fmt->fmt.pix.width;
-	h = fmt->fmt.pix.height;
-	field = fmt->fmt.pix.field;
-	ret = ivtv_g_fmt_vid_out(file, fh, fmt);
-	fmt->fmt.pix.width = w;
-	fmt->fmt.pix.height = h;
 	if (!ret && id->type == IVTV_DEC_STREAM_TYPE_YUV) {
 		fmt->fmt.pix.field = field;
-		if (fmt->fmt.pix.width < 2)
-			fmt->fmt.pix.width = 2;
-		if (fmt->fmt.pix.width > 720)
-			fmt->fmt.pix.width = 720;
-		if (fmt->fmt.pix.height < 2)
-			fmt->fmt.pix.height = 2;
-		if (fmt->fmt.pix.height > 576)
-			fmt->fmt.pix.height = 576;
+		w = min(w, 720);
+		w = max(w, 2);
+		h = min(h, 576);
+		h = max(h, 2);
 	}
+	fmt->fmt.pix.width = w;
+	fmt->fmt.pix.height = h;
 	return ret;
 }
 
@@ -560,9 +553,9 @@ static int ivtv_s_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f
 	struct ivtv_open_id *id = fh;
 	struct ivtv *itv = id->itv;
 	struct cx2341x_mpeg_params *p = &itv->params;
+	int ret = ivtv_try_fmt_vid_cap(file, fh, fmt);
 	int w = fmt->fmt.pix.width;
 	int h = fmt->fmt.pix.height;
-	int ret = ivtv_try_fmt_vid_cap(file, fh, fmt);
 
 	if (ret)
 		return ret;