V4L/DVB (5571): V4l1-compat: Make VIDIOCSPICT return errors in a useful way

Among other things, VIDIOCSPICT sets the pixel format.  Some drivers
don't support all formats, e.g. cx88 doesn't support the planar formats.
The compat code that translates VIDIOCSPICT into V4L2 ioctls doesn't pass
on any errors, so a userspace program doesn't know if it has selected an
unsupported pixel format.

VIDIOCSPICT sets both the memory capture and overlay formats, and it's
possible that one will be set while the other will fail, e.g. cx88
doesn't even support overlay.  Also, trying to set the overlay format
will fail for non-root users.

Signed-off-by: Trent Piepho <xyzzy@speakeasy.org>
Signed-off-by: Mauro Carvalho Chehab <mchehab@infradead.org>

drivers/media/video/v4l1-compat.c

@@ -616,6 +616,8 @@ v4l_compat_translate_ioctl(struct inode         *inode,
 	case VIDIOCSPICT: /*  set tone controls & partial capture format  */
 	{
 		struct video_picture	*pict = arg;
+		int mem_err = 0, ovl_err = 0;
+
 		memset(&fbuf2, 0, sizeof(fbuf2));
 
 		set_v4l_control(inode, file,
@@ -628,33 +630,59 @@ v4l_compat_translate_ioctl(struct inode         *inode,
 				V4L2_CID_SATURATION, pict->colour, drv);
 		set_v4l_control(inode, file,
 				V4L2_CID_WHITENESS, pict->whiteness, drv);
+		/*
+		 * V4L1 uses this ioctl to set both memory capture and overlay
+		 * pixel format, while V4L2 has two different ioctls for this.
+		 * Some cards may not support one or the other, and may support
+		 * different pixel formats for memory vs overlay.
+		 */
 
 		fmt2 = kzalloc(sizeof(*fmt2),GFP_KERNEL);
 		fmt2->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
 		err = drv(inode, file, VIDIOC_G_FMT, fmt2);
-		if (err < 0)
+		/* If VIDIOC_G_FMT failed, then the driver likely doesn't
+		   support memory capture.  Trying to set the memory capture
+		   parameters would be pointless.  */
+		if (err < 0) {
 			dprintk("VIDIOCSPICT / VIDIOC_G_FMT: %d\n",err);
-		if (fmt2->fmt.pix.pixelformat !=
-		    palette_to_pixelformat(pict->palette)) {
+			mem_err = -1000;  /* didn't even try */
+		} else if (fmt2->fmt.pix.pixelformat !=
+			 palette_to_pixelformat(pict->palette)) {
 			fmt2->fmt.pix.pixelformat = palette_to_pixelformat(
 				pict->palette);
-			err = drv(inode, file, VIDIOC_S_FMT, fmt2);
-			if (err < 0)
-				dprintk("VIDIOCSPICT / VIDIOC_S_FMT: %d\n",err);
+			mem_err = drv(inode, file, VIDIOC_S_FMT, fmt2);
+			if (mem_err < 0)
+				dprintk("VIDIOCSPICT / VIDIOC_S_FMT: %d\n",
+					mem_err);
 		}
 
 		err = drv(inode, file, VIDIOC_G_FBUF, &fbuf2);
-		if (err < 0)
+		/* If VIDIOC_G_FBUF failed, then the driver likely doesn't
+		   support overlay.  Trying to set the overlay parameters
+		   would be quite pointless.  */
+		if (err < 0) {
 			dprintk("VIDIOCSPICT / VIDIOC_G_FBUF: %d\n",err);
-		if (fbuf2.fmt.pixelformat !=
-		    palette_to_pixelformat(pict->palette)) {
+			ovl_err = -1000;  /* didn't even try */
+		} else if (fbuf2.fmt.pixelformat !=
+			 palette_to_pixelformat(pict->palette)) {
 			fbuf2.fmt.pixelformat = palette_to_pixelformat(
 				pict->palette);
-			err = drv(inode, file, VIDIOC_S_FBUF, &fbuf2);
-			if (err < 0)
-				dprintk("VIDIOCSPICT / VIDIOC_S_FBUF: %d\n",err);
-			err = 0; /* likely fails for non-root */
+			ovl_err = drv(inode, file, VIDIOC_S_FBUF, &fbuf2);
+			if (ovl_err < 0)
+				dprintk("VIDIOCSPICT / VIDIOC_S_FBUF: %d\n",
+					ovl_err);
 		}
+		if (ovl_err < 0 && mem_err < 0)
+			/* ioctl failed, couldn't set either parameter */
+			if (mem_err != -1000) {
+			    err = mem_err;
+			} else if (ovl_err == -EPERM) {
+			    err = 0;
+			} else {
+			    err = ovl_err;
+			}
+		else
+			err = 0;
 		break;
 	}
 	case VIDIOCGTUNER: /*  get tuner information  */