Skip to content

Commit

Permalink
optimize: performance of desktop viewer on Windows
Browse files Browse the repository at this point in the history
  • Loading branch information
XZB-1248 committed Sep 20, 2022
1 parent 1a365ab commit bc63722
Show file tree
Hide file tree
Showing 8 changed files with 177 additions and 83 deletions.
14 changes: 12 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
## v0.1.5

* Optimize: performance of desktop viewer on Windows.
* Remove: deprecated ioutil package.

* 优化:Windows下的远程桌面性能表现。
* 移除:已经废弃的ioutil包。



## v0.1.4

* Add: desktop viewer (experimental).
Expand Down Expand Up @@ -46,8 +56,8 @@

## v0.1.0

* fix: don't refresh after file upload.
* fix: don't display error when screenshot fails.
* Fix: don't refresh after file upload.
* Fix: don't display error when screenshot fails.

* 修复:文件上传成功后文件管理器不会自动刷新。
* 修复:截图失败时不会显示错误提示。
Expand Down
67 changes: 41 additions & 26 deletions client/service/desktop/desktop.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"image"
"image/jpeg"
"reflect"
"runtime"
"sync"
"time"
"unsafe"
Expand Down Expand Up @@ -51,20 +52,24 @@ type message struct {
// 0: raw image
// 1: compressed image (jpeg)

const fpsLimit = 10
const compress = true
const blockSize = 64
const imgQuality = 70
const displayIndex = 0
const imageQuality = 70

var lock = &sync.Mutex{}
var working = false
var sessions = cmap.New()
var prevDesktop *image.RGBA
var ErrNoImage = errors.New("no image yet")

func init() {
go healthCheck()
}

func worker() {
runtime.LockOSThread()
lock.Lock()
if working {
lock.Unlock()
Expand All @@ -73,20 +78,27 @@ func worker() {
working = true
lock.Unlock()
var (
screen screen
bounds image.Rectangle
img *image.RGBA
err error
errors int
)
bounds = screenshot.GetDisplayBounds(displayIndex)
screen.init(displayIndex)
for working {
if sessions.Count() == 0 {
lock.Lock()
working = false
lock.Unlock()
break
}
<-time.After(70 * time.Millisecond)
img, err = screenshot.CaptureDisplay(0)
img = image.NewRGBA(bounds)
err = screen.capture(img, bounds)
if err != nil {
if err == ErrNoImage {
return
}
errors++
if errors > 10 {
break
Expand All @@ -106,6 +118,7 @@ func worker() {
return true
})
}
<-time.After(time.Second / fpsLimit)
}
}
prevDesktop = nil
Expand All @@ -115,6 +128,8 @@ func worker() {
lock.Lock()
working = false
lock.Unlock()
screen.release()
runtime.UnlockOSThread()
}

func quitAll(info string) {
Expand Down Expand Up @@ -210,16 +225,26 @@ func getImageBlock(img *image.RGBA, rect image.Rectangle, compress bool) []byte
Stride: width * 4,
Rect: image.Rect(0, 0, width, height),
}
writer := new(bytes.Buffer)
jpeg.Encode(writer, subImg, &jpeg.Options{Quality: imgQuality})
writer := &bytes.Buffer{}
jpeg.Encode(writer, subImg, &jpeg.Options{Quality: imageQuality})
return writer.Bytes()
}

func getDiff(img, prev *image.RGBA) []image.Rectangle {
imgWidth := img.Rect.Dx()
imgHeight := img.Rect.Dy()
result := make([]image.Rectangle, 0)
for y := 0; y < imgHeight; y += blockSize {
for y := 0; y < imgHeight; y += blockSize * 2 {
height := utils.If(y+blockSize > imgHeight, imgHeight-y, blockSize)
for x := 0; x < imgWidth; x += blockSize {
width := utils.If(x+blockSize > imgWidth, imgWidth-x, blockSize)
rect := image.Rect(x, y, x+width, y+height)
if isDiff(img, prev, rect) {
result = append(result, rect)
}
}
}
for y := blockSize; y < imgHeight; y += blockSize * 2 {
height := utils.If(y+blockSize > imgHeight, imgHeight-y, blockSize)
for x := 0; x < imgWidth; x += blockSize {
width := utils.If(x+blockSize > imgWidth, imgWidth-x, blockSize)
Expand Down Expand Up @@ -249,25 +274,13 @@ func isDiff(img, prev *image.RGBA, rect image.Rectangle) bool {
if imgHeader.Len < end || prevHeader.Len < end {
return true
}
if rectWidth%2 == 0 {
for y := rect.Min.Y; y < rect.Max.Y; y++ {
cursor := uintptr((y*imgWidth + rect.Min.X) * 4)
for x := 0; x < rectWidth; x += 2 {
if *(*uint64)(unsafe.Pointer(imgPtr + cursor)) != *(*uint64)(unsafe.Pointer(prevPtr + cursor)) {
return true
}
cursor += 8
}
}
} else {
for y := rect.Min.Y; y < rect.Max.Y; y++ {
cursor := uintptr((y*imgWidth + rect.Min.X) * 4)
for x := 0; x < rectWidth; x++ {
if *(*uint32)(unsafe.Pointer(imgPtr + cursor)) != *(*uint32)(unsafe.Pointer(prevPtr + cursor)) {
return true
}
cursor += 4
for y := rect.Min.Y; y < rect.Max.Y; y += 2 {
cursor := uintptr((y*imgWidth + rect.Min.X) * 4)
for x := 0; x < rectWidth; x += 4 {
if *(*uint64)(unsafe.Pointer(imgPtr + cursor)) != *(*uint64)(unsafe.Pointer(prevPtr + cursor)) {
return true
}
cursor += 16
}
}
return false
Expand All @@ -287,7 +300,7 @@ func InitDesktop(pack modules.Packet) error {
desktop := &session{
event: pack.Event,
rawEvent: rawEvent,
lastPack: time.Now().Unix(),
lastPack: common.Unix,
escape: false,
channel: make(chan message, 4),
lock: &sync.Mutex{},
Expand Down Expand Up @@ -332,7 +345,7 @@ func PingDesktop(pack modules.Packet) {
return
} else {
desktop = val.(*session)
desktop.lastPack = time.Now().Unix()
desktop.lastPack = common.Unix
}
}

Expand Down Expand Up @@ -371,7 +384,9 @@ func GetDesktop(pack modules.Packet) {
desktop = val.(*session)
}
if !desktop.escape {
lock.Lock()
img := splitFullImage(prevDesktop, compress)
lock.Unlock()
desktop.lock.Lock()
desktop.channel <- message{t: 0, data: &img}
desktop.lock.Unlock()
Expand Down
26 changes: 26 additions & 0 deletions client/service/desktop/screenshot_others.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
//go:build !windows
// +build !windows

package desktop

import (
"github.com/kbinani/screenshot"
"image"
)

type screen struct {
displayIndex int
}

func (s *screen) init(displayIndex int) {
s.displayIndex = displayIndex
}

func (s *screen) capture(img *image.RGBA, _ image.Rectangle) error {
var err error
img, err = screenshot.CaptureDisplay(displayIndex)
return err
}

func (s *screen) release() {
}
66 changes: 66 additions & 0 deletions client/service/desktop/screenshot_windows.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
//go:build windows
// +build windows

package desktop

import (
"github.com/kirides/screencapture/d3d"
"github.com/kirides/screencapture/screenshot"
"github.com/kirides/screencapture/win"
"image"
)

type screen struct {
dxgi bool
ddup *d3d.OutputDuplicator
device *d3d.ID3D11Device
deviceCtx *d3d.ID3D11DeviceContext
displayIndex int
}

func (s *screen) init(displayIndex int) {
var err error
s.displayIndex = displayIndex
s.dxgi = false
return
if win.IsValidDpiAwarenessContext(win.DpiAwarenessContextPerMonitorAwareV2) {
_, err = win.SetThreadDpiAwarenessContext(win.DpiAwarenessContextPerMonitorAwareV2)
s.dxgi = err == nil
}
if s.dxgi {
s.device, s.deviceCtx, err = d3d.NewD3D11Device()
s.ddup, err = d3d.NewIDXGIOutputDuplication(s.device, s.deviceCtx, uint(displayIndex))
if err != nil {
s.dxgi = false
s.device.Release()
s.deviceCtx.Release()
}
}
}

func (s *screen) capture(img *image.RGBA, bounds image.Rectangle) error {
var err error
if s.dxgi {
err = s.ddup.GetImage(img, 100)
if err != nil {
if err == d3d.ErrNoImageYet {
return ErrNoImage
}
return err
}
return nil
}
return screenshot.CaptureImg(img, 0, 0, bounds.Dx(), bounds.Dy())
}

func (s *screen) release() {
if s.ddup != nil {
s.ddup.Release()
}
if s.device != nil {
s.device.Release()
}
if s.deviceCtx != nil {
s.deviceCtx.Release()
}
}
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@ require (
github.com/shirou/gopsutil/v3 v3.22.2
)

require github.com/kirides/screencapture v0.0.0-20211101142135-282f3f7e0f33 // indirect

require (
github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 // indirect
github.com/gen2brain/shm v0.0.0-20210511105953-083dbc7d9d83 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-playground/locales v0.13.0 // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS
github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 h1:Y5Q2mEwfzjMt5+3u70Gtw93ZOu2UuPeeeTBDntF7FoY=
github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo=
github.com/gen2brain/shm v0.0.0-20210511105953-083dbc7d9d83/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo=
github.com/gin-contrib/pprof v1.3.0 h1:G9eK6HnbkSqDZBYbzG4wrjCsA4e+cvYAHUZw6W+W9K0=
github.com/gin-contrib/pprof v1.3.0/go.mod h1:waMjT1H9b179t3CxuG1cV3DHpga6ybizwfBaM5OXaB0=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
Expand Down Expand Up @@ -50,6 +51,8 @@ github.com/kataras/pio v0.0.10 h1:b0qtPUqOpM2O+bqa5wr2O6dN4cQNwSmFd6HQqgVae0g=
github.com/kataras/pio v0.0.10/go.mod h1:gS3ui9xSD+lAUpbYnjOGiQyY7sUMJO+EHpiRzhtZ5no=
github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329 h1:qq2nCpSrXrmvDGRxW0ruW9BVEV1CN2a9YDOExdt+U0o=
github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329/go.mod h1:2VPVQDR4wO7KXHwP+DAypEy67rXf+okUx2zjgpCxZw4=
github.com/kirides/screencapture v0.0.0-20211101142135-282f3f7e0f33 h1:DXPbp2f7LBZzkWnpPGUG+H1+82++20vNIpalTamWv6E=
github.com/kirides/screencapture v0.0.0-20211101142135-282f3f7e0f33/go.mod h1:fMSGsolzmMhah/U24dXBHSf/6Ue/mXVSIb4wRU7U4Ts=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
Expand Down
46 changes: 13 additions & 33 deletions server/handler/desktop/desktop.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,15 @@ func InitDesktop(ctx *gin.Context) {
// device need to send a packet to browser
func desktopEventWrapper(desktop *desktop) common.EventCallback {
return func(pack modules.Packet, device *melody.Session) {
if len(pack.Act) == 0 {
if pack.Data == nil {
return
}
if data, ok := pack.Data[`data`]; ok {
desktop.targetConn.WriteBinary(*data.(*[]byte))
}
return
}
if pack.Act == `initDesktop` {
if pack.Code != 0 {
msg := `${i18n|desktopSessionCreationFailed}`
Expand All @@ -91,14 +100,6 @@ func desktopEventWrapper(desktop *desktop) common.EventCallback {
desktop.targetConn.Close()
return
}
if len(pack.Act) == 0 {
if pack.Data == nil {
return
}
if data, ok := pack.Data[`data`]; ok {
desktop.targetConn.WriteBinary(*data.(*[]byte))
}
}
}
}

Expand Down Expand Up @@ -157,45 +158,24 @@ func onDesktopMessage(session *melody.Session, data []byte) {
}
desktop := val.(*desktop)
session.Set(`LastPack`, common.Unix)
if pack.Act == `inputDesktop` {
if pack.Data == nil {
return
}
if input, ok := pack.Data[`input`]; ok {
common.SendPack(modules.Packet{Act: `inputDesktop`, Data: gin.H{
`input`: input,
`desktop`: desktop.uuid,
}, Event: desktop.event}, desktop.deviceConn)
}
if pack.Act == `pingDesktop` {
common.SendPack(modules.Packet{Act: `pingDesktop`, Data: gin.H{
`desktop`: desktop.uuid,
}, Event: desktop.event}, desktop.deviceConn)
return
}
if pack.Act == `killDesktop` {
if pack.Data == nil {
return
}
common.SendPack(modules.Packet{Act: `killDesktop`, Data: gin.H{
`desktop`: desktop.uuid,
}, Event: desktop.event}, desktop.deviceConn)
return
}
if pack.Act == `getDesktop` {
if pack.Data == nil {
return
}
common.SendPack(modules.Packet{Act: `getDesktop`, Data: gin.H{
`desktop`: desktop.uuid,
}, Event: desktop.event}, desktop.deviceConn)
return
}
if pack.Act == `ping` {
if pack.Data == nil {
return
}
common.SendPack(modules.Packet{Act: `pingDesktop`, Data: gin.H{
`desktop`: desktop.uuid,
}, Event: desktop.event}, desktop.deviceConn)
return
}
session.Close()
}

Expand Down
Loading

0 comments on commit bc63722

Please sign in to comment.