diff --git a/data/portrait.ts b/data/portrait.ts new file mode 100644 index 0000000000..3acc3a3ab6 Binary files /dev/null and b/data/portrait.ts differ diff --git a/ffmpeg/api_test.go b/ffmpeg/api_test.go index 2073faf073..a731ddfa26 100644 --- a/ffmpeg/api_test.go +++ b/ffmpeg/api_test.go @@ -6,6 +6,8 @@ import ( "os" "testing" "time" + + "github.com/stretchr/testify/require" ) func TestAPI_SkippedSegment(t *testing.T) { @@ -540,9 +542,7 @@ func shortSegments(t *testing.T, accel Acceleration, fc int) { }, } res, err := tc.Transcode(in, out) - if err != nil { - t.Error(err) - } + require.NoError(t, err) if res.Encoded[0].Frames != 0 { t.Error("Unexpected frame counts from stream copy") t.Error(res) @@ -573,9 +573,7 @@ func shortSegments(t *testing.T, accel Acceleration, fc int) { }, } res, err := tc.Transcode(in, out) - if err != nil { - t.Error(err) - } + require.NoError(t, err) if res.Decoded.Frames != 0 || res.Encoded[0].Frames != 0 { t.Error("Unexpected count of decoded frames ", res.Decoded.Frames, res.Decoded.Pixels) } @@ -1530,6 +1528,7 @@ func detectionFreq(t *testing.T, accel Acceleration, deviceid string) { InitFFmpeg() tc, err := NewTranscoderWithDetector(&DSceneAdultSoccer, deviceid) + require.NotNil(t, tc, "look for `Failed to load native model` logs above") if err != nil { t.Error(err) } diff --git a/ffmpeg/extras.c b/ffmpeg/extras.c index ff221b22fa..05bfb2678f 100644 --- a/ffmpeg/extras.c +++ b/ffmpeg/extras.c @@ -167,6 +167,8 @@ int lpms_get_codec_info(char *fname, pcodec_info out) if(audio_present && pixel_format_missing && no_picture_height) { ret = GET_CODEC_NEEDS_BYPASS; } + out->width = ic->streams[vstream]->codecpar->width; + out->height = ic->streams[vstream]->codecpar->height; } else { // Indicate failure to extract video codec from given container out->video_codec[0] = 0; diff --git a/ffmpeg/extras.h b/ffmpeg/extras.h index 87b5cbc6bb..96f172a1e6 100644 --- a/ffmpeg/extras.h +++ b/ffmpeg/extras.h @@ -5,6 +5,8 @@ typedef struct s_codec_info { char * video_codec; char * audio_codec; int pixel_format; + int width; + int height; } codec_info, *pcodec_info; int lpms_rtmp2hls(char *listen, char *outf, char *ts_tmpl, char *seg_time, char *seg_start); diff --git a/ffmpeg/ffmpeg.go b/ffmpeg/ffmpeg.go index c78ee0d466..35a604865e 100755 --- a/ffmpeg/ffmpeg.go +++ b/ffmpeg/ffmpeg.go @@ -241,8 +241,22 @@ const ( CodecStatusMissing CodecStatus = 2 ) -func GetCodecInfo(fname string) (CodecStatus, string, string, PixelFormat, error) { - var acodec, vcodec string +type MediaFormatInfo struct { + Acodec, Vcodec string + PixFormat PixelFormat + Width, Height int +} + +func (f *MediaFormatInfo) ScaledHeight(width int) int { + return int(float32(width) * float32(f.Height) / float32(f.Width)) +} + +func (f *MediaFormatInfo) ScaledWidth(height int) int { + return int(float32(height) * float32(f.Width) / float32(f.Height)) +} + +func GetCodecInfo(fname string) (CodecStatus, MediaFormatInfo, error) { + format := MediaFormatInfo{} cfname := C.CString(fname) defer C.free(unsafe.Pointer(cfname)) acodec_c := C.CString(strings.Repeat("0", 255)) @@ -255,20 +269,21 @@ func GetCodecInfo(fname string) (CodecStatus, string, string, PixelFormat, error params_c.pixel_format = C.AV_PIX_FMT_NONE status := CodecStatus(C.lpms_get_codec_info(cfname, ¶ms_c)) if C.strlen(acodec_c) < 255 { - acodec = C.GoString(acodec_c) + format.Acodec = C.GoString(acodec_c) } if C.strlen(vcodec_c) < 255 { - vcodec = C.GoString(vcodec_c) + format.Vcodec = C.GoString(vcodec_c) } - pixelFormat := PixelFormat{int(params_c.pixel_format)} - return status, acodec, vcodec, pixelFormat, nil + format.PixFormat = PixelFormat{int(params_c.pixel_format)} + format.Width = int(params_c.width) + format.Height = int(params_c.height) + return status, format, nil } // GetCodecInfo opens the segment and attempts to get video and audio codec names. Additionally, first return value // indicates whether the segment has zero video frames -func GetCodecInfoBytes(data []byte) (CodecStatus, string, string, PixelFormat, error) { - var acodec, vcodec string - var pixelFormat PixelFormat +func GetCodecInfoBytes(data []byte) (CodecStatus, MediaFormatInfo, error) { + format := MediaFormatInfo{} status := CodecStatusInternalError or, ow, err := os.Pipe() go func() { @@ -277,11 +292,11 @@ func GetCodecInfoBytes(data []byte) (CodecStatus, string, string, PixelFormat, e ow.Close() }() if err != nil { - return status, acodec, vcodec, pixelFormat, ErrEmptyData + return status, format, ErrEmptyData } fname := fmt.Sprintf("pipe:%d", or.Fd()) - status, acodec, vcodec, pixelFormat, err = GetCodecInfo(fname) - return status, acodec, vcodec, pixelFormat, err + status, format, err = GetCodecInfo(fname) + return status, format, err } // HasZeroVideoFrameBytes opens video and returns true if it has video stream with 0-frame @@ -299,7 +314,7 @@ func HasZeroVideoFrameBytes(data []byte) (bool, error) { io.Copy(ow, br) ow.Close() }() - status, _, _, _, err := GetCodecInfo(fname) + status, _, err := GetCodecInfo(fname) ow.Close() return status == CodecStatusNeedsBypass, err } @@ -492,6 +507,89 @@ func Transcode3(input *TranscodeOptionsIn, ps []TranscodeOptions) (*TranscodeRes return t.Transcode(input, ps) } +type CodingSizeLimit struct { + WidthMin, HeightMin int + WidthMax, HeightMax int +} + +type Size struct { + W, H int +} + +func (s *Size) Valid(l *CodingSizeLimit) bool { + if s.W < l.WidthMin || s.W > l.WidthMax || s.H < l.HeightMin || s.H > l.HeightMax { + return false + } + return true +} + +func clamp(val, min, max int) int { + if val <= min { + return min + } + if val >= max { + return max + } + return val +} + +func (l *CodingSizeLimit) Clamp(p *VideoProfile, format MediaFormatInfo) error { + w, h, err := VideoProfileResolution(*p) + if err != nil { + return err + } + // detect correct rotation + outputAr := float32(w) / float32(h) + inputAr := float32(format.Width) / float32(format.Height) + if (inputAr > 1.0) != (outputAr > 1.0) { + // comparing landscape to portrait, apply rotate on chosen resolution + w, h = h, w + } + // Adjust to minimal encode dimensions keeping aspect ratio + + var adjustedWidth, adjustedHeight Size + adjustedWidth.W = clamp(w, l.WidthMin, l.WidthMax) + adjustedWidth.H = format.ScaledHeight(adjustedWidth.W) + adjustedHeight.H = clamp(h, l.HeightMin, l.HeightMax) + adjustedHeight.W = format.ScaledWidth(adjustedHeight.H) + if adjustedWidth.Valid(l) { + p.Resolution = fmt.Sprintf("%dx%d", adjustedWidth.W, adjustedWidth.H) + return nil + } + if adjustedHeight.Valid(l) { + p.Resolution = fmt.Sprintf("%dx%d", adjustedHeight.W, adjustedHeight.H) + return nil + } + return fmt.Errorf("profile %dx%d size out of bounds %dx%d-%dx%d", w, h, l.WidthMin, l.WidthMin, l.WidthMax, l.HeightMax) +} + +// 7th Gen NVENC limits: +var nvidiaCodecSizeLimts = map[VideoCodec]CodingSizeLimit{ + H264: {146, 50, 4096, 4096}, + H265: {132, 40, 8192, 8192}, +} + +func ensureEncoderLimits(outputs []TranscodeOptions, format MediaFormatInfo) error { + // not using range to be able to make inplace modifications to outputs elements + for i := 0; i < len(outputs); i++ { + if outputs[i].Accel == Nvidia { + limits, haveLimits := nvidiaCodecSizeLimts[outputs[i].Profile.Encoder] + resolutionSpecified := outputs[i].Profile.Resolution != "" + // Sometimes rendition Resolution is not specified. We skip this rendition. + if haveLimits && resolutionSpecified { + err := limits.Clamp(&outputs[i].Profile, format) + if err != nil { + // if err == ErrTranscoderRes { + // return fmt.Errorf("Found profile [%d] without resolution %v", i, outputs[i]) + // } + return err + } + } + } + } + return nil +} + // create C output params array and return it along with corresponding finalizer // function that makes sure there are no C memory leaks func createCOutputParams(input *TranscodeOptionsIn, ps []TranscodeOptions) ([]C.output_params, func(), error) { @@ -733,6 +831,20 @@ func destroyCOutputParams(params []C.output_params) { } func (t *Transcoder) Transcode(input *TranscodeOptionsIn, ps []TranscodeOptions) (*TranscodeResults, error) { + // here we require input size and aspect ratio + status, format, err := GetCodecInfo(input.Fname) + if err != nil { + return nil, err + } + if status == CodecStatusOk { + // We dont return error in case status != CodecStatusOk because proper error would be returned later in the logic. + // Like 'TranscoderInvalidVideo' or `No such file or directory` would be replaced by error we specify here. + err = ensureEncoderLimits(ps, format) + if err != nil { + return nil, err + } + } + t.mu.Lock() defer t.mu.Unlock() if t.stopped || t.handle == nil { @@ -761,9 +873,9 @@ func (t *Transcoder) Transcode(input *TranscodeOptionsIn, ps []TranscodeOptions) t.started = true } if !t.started { - status, _, vcodec, _, _ := GetCodecInfo(input.Fname) + status, format, _ := GetCodecInfo(input.Fname) // NeedsBypass is state where video is present in container & vithout any frames - videoMissing := status == CodecStatusNeedsBypass || vcodec == "" + videoMissing := status == CodecStatusNeedsBypass || format.Vcodec == "" if videoMissing { // Audio-only segment, fail fast right here as we cannot handle them nicely return nil, ErrTranscoderVid diff --git a/ffmpeg/ffmpeg_test.go b/ffmpeg/ffmpeg_test.go index a5d5d352dd..c6cbc05401 100644 --- a/ffmpeg/ffmpeg_test.go +++ b/ffmpeg/ffmpeg_test.go @@ -1711,9 +1711,9 @@ func TestTranscoder_GetCodecInfo(t *testing.T) { t.Fatal(err) } fname := path.Join(wd, "..", "data", "zero-frame.ts") - status, acodec, vcodec, pixelFormat, err := GetCodecInfo(fname) + status, format, err := GetCodecInfo(fname) isZeroFrame := status == CodecStatusNeedsBypass - fmt.Printf("zero-frame.ts %t %s %s %d %v\n", isZeroFrame, acodec, vcodec, pixelFormat, err) + fmt.Printf("zero-frame.ts %t %s %s %d %v\n", isZeroFrame, format.Acodec, format.Vcodec, format.PixFormat, err) if isZeroFrame != true { t.Errorf("Expecting true, got %v fname=%s", isZeroFrame, fname) } @@ -1721,9 +1721,9 @@ func TestTranscoder_GetCodecInfo(t *testing.T) { if err != nil { t.Error(err) } - status, acodec, vcodec, pixelFormat, err = GetCodecInfoBytes(data) + status, format, err = GetCodecInfoBytes(data) isZeroFrame = status == CodecStatusNeedsBypass - fmt.Printf("zero-frame.ts %t %s %s %d %v\n", isZeroFrame, acodec, vcodec, pixelFormat, err) + fmt.Printf("zero-frame.ts %t %s %s %d %v\n", isZeroFrame, format.Acodec, format.Vcodec, format.PixFormat, err) if err != nil { t.Error(err) } @@ -1735,14 +1735,14 @@ func TestTranscoder_GetCodecInfo(t *testing.T) { t.Errorf("Unexpected error %v", err) } fname = path.Join(wd, "..", "data", "bunny.mp4") - status, acodec, vcodec, pixelFormat, err = GetCodecInfo(fname) + status, format, err = GetCodecInfo(fname) isZeroFrame = status == CodecStatusNeedsBypass - fmt.Printf("bunny.mp4 %t %s %s %d %v\n", isZeroFrame, acodec, vcodec, pixelFormat, err) + fmt.Printf("bunny.mp4 %t %s %s %d %v\n", isZeroFrame, format.Acodec, format.Vcodec, format.PixFormat, err) if isZeroFrame != false { t.Errorf("Expecting false, got %v fname=%s", isZeroFrame, fname) } - assert.Equal(t, "h264", vcodec) - assert.Equal(t, "aac", acodec) + assert.Equal(t, "h264", format.Vcodec) + assert.Equal(t, "aac", format.Acodec) } func TestTranscoder_ZeroFrameLongBadSegment(t *testing.T) { @@ -1808,3 +1808,62 @@ func TestTranscoder_Clip2(t *testing.T) { assert.Equal(t, 601, res.Encoded[0].Frames) assert.Equal(t, int64(22155264), res.Encoded[0].Pixels) } + +func TestResolution_Clamp(t *testing.T) { + // expect no error + checkError := require.NoError + test := func(limit CodingSizeLimit, profile, input, expected Size) { + p := &VideoProfile{Resolution: fmt.Sprintf("%dx%d", profile.W, profile.H)} + m := MediaFormatInfo{Width: input.W, Height: input.H} + // call function we are testing: + err := limit.Clamp(p, m) + checkError(t, err) + var resultW, resultH int + _, err = fmt.Sscanf(p.Resolution, "%dx%d", &resultW, &resultH) + require.NoError(t, err) + assert.Equal(t, expected, Size{resultW, resultH}) + } + + l := CodingSizeLimit{ + WidthMin: 70, + HeightMin: 50, + WidthMax: 700, + HeightMax: 500, + } + // use aspect ratio == 2 to easily check calculation + portrait := Size{900, 1800} + landscape := Size{1800, 900} + + // no change, fits within hw limits + test(l, Size{120, 60}, landscape, Size{120, 60}) + // h=40 too small must be 50 + test(l, Size{80, 40}, landscape, Size{100, 50}) + // In portrait mode our profile 80x40 is interpreted as 40x80, increasing h=70 + test(l, Size{80, 40}, portrait, Size{70, 140}) + // portrait 60x120 used with landscape source, rotated to 120x60, within limits + test(l, Size{60, 120}, landscape, Size{120, 60}) + // portrait 60x120 profile on portrait source does not rotate, increasing w=70 + test(l, Size{60, 120}, portrait, Size{70, 140}) + + // test choice between adjustedWidth adjustedHeight variants: + test(l, Size{30, 60}, portrait, Size{70, 140}) + test(l, Size{30, 60}, landscape, Size{100, 50}) + + // Test max values: + test(l, Size{1000, 500}, landscape, Size{700, 350}) + test(l, Size{1000, 500}, portrait, Size{250, 500}) + test(l, Size{500, 1000}, landscape, Size{700, 350}) + test(l, Size{600, 300}, portrait, Size{250, 500}) + test(l, Size{300, 600}, portrait, Size{250, 500}) + + // Test impossible limits for aspect ratio == 2 + l = CodingSizeLimit{ + WidthMin: 500, + HeightMin: 500, + WidthMax: 600, + HeightMax: 600, + } + // expect error + checkError = require.Error + test(l, Size{300, 600}, portrait, Size{300, 600}) +} diff --git a/ffmpeg/nvidia_test.go b/ffmpeg/nvidia_test.go index 5a6c85d657..666557890a 100644 --- a/ffmpeg/nvidia_test.go +++ b/ffmpeg/nvidia_test.go @@ -6,7 +6,10 @@ package ffmpeg import ( "fmt" "os" + "path" "testing" + + "github.com/stretchr/testify/require" ) func TestNvidia_Transcoding(t *testing.T) { @@ -728,4 +731,35 @@ func TestNvidia_DetectionFreq(t *testing.T) { detectionFreq(t, Nvidia, "0") } +func TestTranscoder_Portrait(t *testing.T) { + wd, err := os.Getwd() + require.NoError(t, err) + fname := path.Join(wd, "..", "data", "portrait.ts") + outNames := []string{ + path.Join(wd, "..", "data", "singleframe-out-360.ts"), + path.Join(wd, "..", "data", "singleframe-out-240.ts"), + path.Join(wd, "..", "data", "singleframe-out-144.ts"), + } + // prof := P720p30fps16x9 + in := &TranscodeOptionsIn{Fname: fname, Accel: Nvidia} + hevc := VideoProfile{Name: "P240p30fps16x9", Bitrate: "600k", Framerate: 30, AspectRatio: "16:9", Resolution: "426x240", Encoder: H265} + out := []TranscodeOptions{ + {Oname: outNames[0], Profile: P360p30fps16x9, Accel: Nvidia}, + {Oname: outNames[1], Profile: hevc, Accel: Nvidia}, + {Oname: outNames[2], Profile: P144p30fps16x9, Accel: Nvidia}, + } + res, err := Transcode3(in, out) + require.NoError(t, err) + for i := 0; i < len(outNames); i++ { + outInfo, err := os.Stat(outNames[i]) + if os.IsNotExist(err) { + t.Error(err) + } else { + defer os.Remove(outNames[i]) + } + require.NotEqual(t, outInfo.Size(), 0, "must produce output %s", outNames[i]) + require.Equal(t, res.Encoded[i].Frames, 30, "must produce 30 frames in output %s", outNames[i]) + } +} + // XXX test bframes or delayed frames