Honor minimum encoder resolution (livepeer#326)

Fix for mistaking landscape for portrait

ffmpeg/api_test.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/require"
 )
 
 func TestAPI_SkippedSegment(t *testing.T) {
@@ -540,9 +542,7 @@ func shortSegments(t *testing.T, accel Acceleration, fc int) {
 			},
 		}
 		res, err := tc.Transcode(in, out)
-		if err != nil {
-			t.Error(err)
-		}
+		require.NoError(t, err)
 		if res.Encoded[0].Frames != 0 {
 			t.Error("Unexpected frame counts from stream copy")
 			t.Error(res)
@@ -573,9 +573,7 @@ func shortSegments(t *testing.T, accel Acceleration, fc int) {
 			},
 		}
 		res, err := tc.Transcode(in, out)
-		if err != nil {
-			t.Error(err)
-		}
+		require.NoError(t, err)
 		if res.Decoded.Frames != 0 || res.Encoded[0].Frames != 0 {
 			t.Error("Unexpected count of decoded frames ", res.Decoded.Frames, res.Decoded.Pixels)
 		}
@@ -1530,6 +1528,7 @@ func detectionFreq(t *testing.T, accel Acceleration, deviceid string) {
 
 	InitFFmpeg()
 	tc, err := NewTranscoderWithDetector(&DSceneAdultSoccer, deviceid)
+	require.NotNil(t, tc, "look for `Failed to load native model` logs above")
 	if err != nil {
 		t.Error(err)
 	}

ffmpeg/extras.c

@@ -167,6 +167,8 @@ int lpms_get_codec_info(char *fname, pcodec_info out)
       if(audio_present && pixel_format_missing && no_picture_height) {
         ret = GET_CODEC_NEEDS_BYPASS;
       }
+      out->width  = ic->streams[vstream]->codecpar->width;
+      out->height = ic->streams[vstream]->codecpar->height;
   } else {
       // Indicate failure to extract video codec from given container
       out->video_codec[0] = 0;

ffmpeg/extras.h

@@ -5,6 +5,8 @@ typedef struct s_codec_info {
   char * video_codec;
   char * audio_codec;
   int    pixel_format;
+  int    width;
+  int    height;
 } codec_info, *pcodec_info;
 
 int lpms_rtmp2hls(char *listen, char *outf, char *ts_tmpl, char *seg_time, char *seg_start);

ffmpeg/ffmpeg.go

@@ -241,8 +241,22 @@ const (
 	CodecStatusMissing       CodecStatus = 2
 )
 
-func GetCodecInfo(fname string) (CodecStatus, string, string, PixelFormat, error) {
-	var acodec, vcodec string
+type MediaFormatInfo struct {
+	Acodec, Vcodec string
+	PixFormat      PixelFormat
+	Width, Height  int
+}
+
+func (f *MediaFormatInfo) ScaledHeight(width int) int {
+	return int(float32(width) * float32(f.Height) / float32(f.Width))
+}
+
+func (f *MediaFormatInfo) ScaledWidth(height int) int {
+	return int(float32(height) * float32(f.Width) / float32(f.Height))
+}
+
+func GetCodecInfo(fname string) (CodecStatus, MediaFormatInfo, error) {
+	format := MediaFormatInfo{}
 	cfname := C.CString(fname)
 	defer C.free(unsafe.Pointer(cfname))
 	acodec_c := C.CString(strings.Repeat("0", 255))
@@ -255,20 +269,21 @@ func GetCodecInfo(fname string) (CodecStatus, string, string, PixelFormat, error
 	params_c.pixel_format = C.AV_PIX_FMT_NONE
 	status := CodecStatus(C.lpms_get_codec_info(cfname, &params_c))
 	if C.strlen(acodec_c) < 255 {
-		acodec = C.GoString(acodec_c)
+		format.Acodec = C.GoString(acodec_c)
 	}
 	if C.strlen(vcodec_c) < 255 {
-		vcodec = C.GoString(vcodec_c)
+		format.Vcodec = C.GoString(vcodec_c)
 	}
-	pixelFormat := PixelFormat{int(params_c.pixel_format)}
-	return status, acodec, vcodec, pixelFormat, nil
+	format.PixFormat = PixelFormat{int(params_c.pixel_format)}
+	format.Width = int(params_c.width)
+	format.Height = int(params_c.height)
+	return status, format, nil
 }
 
 // GetCodecInfo opens the segment and attempts to get video and audio codec names. Additionally, first return value
 // indicates whether the segment has zero video frames
-func GetCodecInfoBytes(data []byte) (CodecStatus, string, string, PixelFormat, error) {
-	var acodec, vcodec string
-	var pixelFormat PixelFormat
+func GetCodecInfoBytes(data []byte) (CodecStatus, MediaFormatInfo, error) {
+	format := MediaFormatInfo{}
 	status := CodecStatusInternalError
 	or, ow, err := os.Pipe()
 	go func() {
@@ -277,11 +292,11 @@ func GetCodecInfoBytes(data []byte) (CodecStatus, string, string, PixelFormat, e
 		ow.Close()
 	}()
 	if err != nil {
-		return status, acodec, vcodec, pixelFormat, ErrEmptyData
+		return status, format, ErrEmptyData
 	}
 	fname := fmt.Sprintf("pipe:%d", or.Fd())
-	status, acodec, vcodec, pixelFormat, err = GetCodecInfo(fname)
-	return status, acodec, vcodec, pixelFormat, err
+	status, format, err = GetCodecInfo(fname)
+	return status, format, err
 }
 
 // HasZeroVideoFrameBytes  opens video and returns true if it has video stream with 0-frame
@@ -299,7 +314,7 @@ func HasZeroVideoFrameBytes(data []byte) (bool, error) {
 		io.Copy(ow, br)
 		ow.Close()
 	}()
-	status, _, _, _, err := GetCodecInfo(fname)
+	status, _, err := GetCodecInfo(fname)
 	ow.Close()
 	return status == CodecStatusNeedsBypass, err
 }
@@ -492,6 +507,89 @@ func Transcode3(input *TranscodeOptionsIn, ps []TranscodeOptions) (*TranscodeRes
 	return t.Transcode(input, ps)
 }
 
+type CodingSizeLimit struct {
+	WidthMin, HeightMin int
+	WidthMax, HeightMax int
+}
+
+type Size struct {
+	W, H int
+}
+
+func (s *Size) Valid(l *CodingSizeLimit) bool {
+	if s.W < l.WidthMin || s.W > l.WidthMax || s.H < l.HeightMin || s.H > l.HeightMax {
+		return false
+	}
+	return true
+}
+
+func clamp(val, min, max int) int {
+	if val <= min {
+		return min
+	}
+	if val >= max {
+		return max
+	}
+	return val
+}
+
+func (l *CodingSizeLimit) Clamp(p *VideoProfile, format MediaFormatInfo) error {
+	w, h, err := VideoProfileResolution(*p)
+	if err != nil {
+		return err
+	}
+	// detect correct rotation
+	outputAr := float32(w) / float32(h)
+	inputAr := float32(format.Width) / float32(format.Height)
+	if (inputAr > 1.0) != (outputAr > 1.0) {
+		// comparing landscape to portrait, apply rotate on chosen resolution
+		w, h = h, w
+	}
+	// Adjust to minimal encode dimensions keeping aspect ratio
+
+	var adjustedWidth, adjustedHeight Size
+	adjustedWidth.W = clamp(w, l.WidthMin, l.WidthMax)
+	adjustedWidth.H = format.ScaledHeight(adjustedWidth.W)
+	adjustedHeight.H = clamp(h, l.HeightMin, l.HeightMax)
+	adjustedHeight.W = format.ScaledWidth(adjustedHeight.H)
+	if adjustedWidth.Valid(l) {
+		p.Resolution = fmt.Sprintf("%dx%d", adjustedWidth.W, adjustedWidth.H)
+		return nil
+	}
+	if adjustedHeight.Valid(l) {
+		p.Resolution = fmt.Sprintf("%dx%d", adjustedHeight.W, adjustedHeight.H)
+		return nil
+	}
+	return fmt.Errorf("profile %dx%d size out of bounds %dx%d-%dx%d", w, h, l.WidthMin, l.WidthMin, l.WidthMax, l.HeightMax)
+}
+
+// 7th Gen NVENC limits:
+var nvidiaCodecSizeLimts = map[VideoCodec]CodingSizeLimit{
+	H264: {146, 50, 4096, 4096},
+	H265: {132, 40, 8192, 8192},
+}
+
+func ensureEncoderLimits(outputs []TranscodeOptions, format MediaFormatInfo) error {
+	// not using range to be able to make inplace modifications to outputs elements
+	for i := 0; i < len(outputs); i++ {
+		if outputs[i].Accel == Nvidia {
+			limits, haveLimits := nvidiaCodecSizeLimts[outputs[i].Profile.Encoder]
+			resolutionSpecified := outputs[i].Profile.Resolution != ""
+			// Sometimes rendition Resolution is not specified. We skip this rendition.
+			if haveLimits && resolutionSpecified {
+				err := limits.Clamp(&outputs[i].Profile, format)
+				if err != nil {
+					// if err == ErrTranscoderRes {
+					// 	return fmt.Errorf("Found profile [%d] without resolution %v", i, outputs[i])
+					// }
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
 // create C output params array and return it along with corresponding finalizer
 // function that makes sure there are no C memory leaks
 func createCOutputParams(input *TranscodeOptionsIn, ps []TranscodeOptions) ([]C.output_params, func(), error) {
@@ -733,6 +831,20 @@ func destroyCOutputParams(params []C.output_params) {
 }
 
 func (t *Transcoder) Transcode(input *TranscodeOptionsIn, ps []TranscodeOptions) (*TranscodeResults, error) {
+	// here we require input size and aspect ratio
+	status, format, err := GetCodecInfo(input.Fname)
+	if err != nil {
+		return nil, err
+	}
+	if status == CodecStatusOk {
+		// We dont return error in case status != CodecStatusOk because proper error would be returned later in the logic.
+		// Like 'TranscoderInvalidVideo' or `No such file or directory` would be replaced by error we specify here.
+		err = ensureEncoderLimits(ps, format)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	if t.stopped || t.handle == nil {
@@ -761,9 +873,9 @@ func (t *Transcoder) Transcode(input *TranscodeOptionsIn, ps []TranscodeOptions)
 		t.started = true
 	}
 	if !t.started {
-		status, _, vcodec, _, _ := GetCodecInfo(input.Fname)
+		status, format, _ := GetCodecInfo(input.Fname)
 		// NeedsBypass is state where video is present in container & vithout any frames
-		videoMissing := status == CodecStatusNeedsBypass || vcodec == ""
+		videoMissing := status == CodecStatusNeedsBypass || format.Vcodec == ""
 		if videoMissing {
 			// Audio-only segment, fail fast right here as we cannot handle them nicely
 			return nil, ErrTranscoderVid

ffmpeg/ffmpeg_test.go

@@ -1711,19 +1711,19 @@ func TestTranscoder_GetCodecInfo(t *testing.T) {
 		t.Fatal(err)
 	}
 	fname := path.Join(wd, "..", "data", "zero-frame.ts")
-	status, acodec, vcodec, pixelFormat, err := GetCodecInfo(fname)
+	status, format, err := GetCodecInfo(fname)
 	isZeroFrame := status == CodecStatusNeedsBypass
-	fmt.Printf("zero-frame.ts %t %s %s %d %v\n", isZeroFrame, acodec, vcodec, pixelFormat, err)
+	fmt.Printf("zero-frame.ts %t %s %s %d %v\n", isZeroFrame, format.Acodec, format.Vcodec, format.PixFormat, err)
 	if isZeroFrame != true {
 		t.Errorf("Expecting true, got %v fname=%s", isZeroFrame, fname)
 	}
 	data, err := ioutil.ReadFile(fname)
 	if err != nil {
 		t.Error(err)
 	}
-	status, acodec, vcodec, pixelFormat, err = GetCodecInfoBytes(data)
+	status, format, err = GetCodecInfoBytes(data)
 	isZeroFrame = status == CodecStatusNeedsBypass
-	fmt.Printf("zero-frame.ts %t %s %s %d %v\n", isZeroFrame, acodec, vcodec, pixelFormat, err)
+	fmt.Printf("zero-frame.ts %t %s %s %d %v\n", isZeroFrame, format.Acodec, format.Vcodec, format.PixFormat, err)
 	if err != nil {
 		t.Error(err)
 	}
@@ -1735,14 +1735,14 @@ func TestTranscoder_GetCodecInfo(t *testing.T) {
 		t.Errorf("Unexpected error %v", err)
 	}
 	fname = path.Join(wd, "..", "data", "bunny.mp4")
-	status, acodec, vcodec, pixelFormat, err = GetCodecInfo(fname)
+	status, format, err = GetCodecInfo(fname)
 	isZeroFrame = status == CodecStatusNeedsBypass
-	fmt.Printf("bunny.mp4 %t %s %s %d %v\n", isZeroFrame, acodec, vcodec, pixelFormat, err)
+	fmt.Printf("bunny.mp4 %t %s %s %d %v\n", isZeroFrame, format.Acodec, format.Vcodec, format.PixFormat, err)
 	if isZeroFrame != false {
 		t.Errorf("Expecting false, got %v fname=%s", isZeroFrame, fname)
 	}
-	assert.Equal(t, "h264", vcodec)
-	assert.Equal(t, "aac", acodec)
+	assert.Equal(t, "h264", format.Vcodec)
+	assert.Equal(t, "aac", format.Acodec)
 }
 
 func TestTranscoder_ZeroFrameLongBadSegment(t *testing.T) {
@@ -1808,3 +1808,62 @@ func TestTranscoder_Clip2(t *testing.T) {
 	assert.Equal(t, 601, res.Encoded[0].Frames)
 	assert.Equal(t, int64(22155264), res.Encoded[0].Pixels)
 }
+
+func TestResolution_Clamp(t *testing.T) {
+	// expect no error
+	checkError := require.NoError
+	test := func(limit CodingSizeLimit, profile, input, expected Size) {
+		p := &VideoProfile{Resolution: fmt.Sprintf("%dx%d", profile.W, profile.H)}
+		m := MediaFormatInfo{Width: input.W, Height: input.H}
+		// call function we are testing:
+		err := limit.Clamp(p, m)
+		checkError(t, err)
+		var resultW, resultH int
+		_, err = fmt.Sscanf(p.Resolution, "%dx%d", &resultW, &resultH)
+		require.NoError(t, err)
+		assert.Equal(t, expected, Size{resultW, resultH})
+	}
+
+	l := CodingSizeLimit{
+		WidthMin:  70,
+		HeightMin: 50,
+		WidthMax:  700,
+		HeightMax: 500,
+	}
+	// use aspect ratio == 2 to easily check calculation
+	portrait := Size{900, 1800}
+	landscape := Size{1800, 900}
+
+	// no change, fits within hw limits
+	test(l, Size{120, 60}, landscape, Size{120, 60})
+	// h=40 too small must be 50
+	test(l, Size{80, 40}, landscape, Size{100, 50})
+	// In portrait mode our profile 80x40 is interpreted as 40x80, increasing h=70
+	test(l, Size{80, 40}, portrait, Size{70, 140})
+	// portrait 60x120 used with landscape source, rotated to 120x60, within limits
+	test(l, Size{60, 120}, landscape, Size{120, 60})
+	// portrait 60x120 profile on portrait source does not rotate, increasing w=70
+	test(l, Size{60, 120}, portrait, Size{70, 140})
+
+	// test choice between adjustedWidth adjustedHeight variants:
+	test(l, Size{30, 60}, portrait, Size{70, 140})
+	test(l, Size{30, 60}, landscape, Size{100, 50})
+
+	// Test max values:
+	test(l, Size{1000, 500}, landscape, Size{700, 350})
+	test(l, Size{1000, 500}, portrait, Size{250, 500})
+	test(l, Size{500, 1000}, landscape, Size{700, 350})
+	test(l, Size{600, 300}, portrait, Size{250, 500})
+	test(l, Size{300, 600}, portrait, Size{250, 500})
+
+	// Test impossible limits for aspect ratio == 2
+	l = CodingSizeLimit{
+		WidthMin:  500,
+		HeightMin: 500,
+		WidthMax:  600,
+		HeightMax: 600,
+	}
+	// expect error
+	checkError = require.Error
+	test(l, Size{300, 600}, portrait, Size{300, 600})
+}