From 101937793bf58c74d3174f5395cab09a894b2d84 Mon Sep 17 00:00:00 2001 From: Boris Glimcher Date: Mon, 18 Sep 2023 21:30:38 +0300 Subject: [PATCH] feat: add grpc gw http listener Signed-off-by: Boris Glimcher --- Dockerfile | 4 ++-- README.md | 12 +++++++++-- cmd/main.go | 52 ++++++++++++++++++++++++++++++++++++++++++---- docker-compose.yml | 12 ++++++++++- 4 files changed, 71 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index d9b087ba..1ccf6d0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,6 @@ RUN go build -v -o /opi-spdk-bridge ./cmd/... FROM alpine:3.18 COPY --from=builder /opi-spdk-bridge / COPY --from=docker.io/fullstorydev/grpcurl:v1.8.7-alpine /bin/grpcurl /usr/local/bin/ -EXPOSE 50051 -CMD [ "/opi-spdk-bridge", "-port=50051" ] +EXPOSE 50051 8082 +CMD [ "/opi-spdk-bridge", "-grpc_port=50051", "-http_port=8082" ] HEALTHCHECK CMD grpcurl -plaintext localhost:50051 list || exit 1 diff --git a/README.md b/README.md index dfab0995..776b876e 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,10 @@ It is just an example and implies SPDK just as example, not mandated by OPI. on DPU/IPU (i.e. with IP=10.10.10.10) run ```bash -$ docker run --rm -it -v /var/tmp/:/var/tmp/ -p 50051:50051 ghcr.io/opiproject/opi-spdk-bridge:main -2022/09/21 21:39:49 server listening at [::]:50051 +$ docker run --rm -it -v /var/tmp/:/var/tmp/ -p 50051:50051 -p 8082:8082 ghcr.io/opiproject/opi-spdk-bridge:main +2023/09/12 20:29:05 Connection to SPDK will be via: unix detected from /var/tmp/spdk.sock +2023/09/12 20:29:05 gRPC server listening at [::]:50051 +2023/09/12 20:29:05 HTTP Server listening at 8082 ``` on X86 management VM run @@ -208,6 +210,12 @@ opi-spdk-server_1 | 2022/08/05 14:39:40 Received from SPDK: false opi-spdk-server_1 | 2022/08/05 14:39:40 Could not delete: id:8 ``` +In addition HTTP is supported via grpc gateway, for example: + +```bash +curl -kL http://10.10.10.10:8082/v1/inventory/1/inventory/2 +``` + Another remote call example ```bash diff --git a/cmd/main.go b/cmd/main.go index 4ecac424..efad25e2 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -4,11 +4,14 @@ package main import ( + "context" "flag" "fmt" "log" "net" + "net/http" "strings" + "time" "github.com/opiproject/gospdk/spdk" @@ -18,9 +21,14 @@ import ( "github.com/opiproject/opi-spdk-bridge/pkg/middleend" "github.com/opiproject/opi-spdk-bridge/pkg/server" + pc "github.com/opiproject/opi-api/common/v1/gen/go" pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/reflection" + + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" ) func splitBusesBySeparator(str string) []string { @@ -31,8 +39,11 @@ func splitBusesBySeparator(str string) []string { } func main() { - var port int - flag.IntVar(&port, "port", 50051, "The Server port") + var grpcPort int + flag.IntVar(&grpcPort, "grpc_port", 50051, "The gRPC server port") + + var httpPort int + flag.IntVar(&httpPort, "http_port", 8082, "The HTTP server port") var spdkAddress string flag.StringVar(&spdkAddress, "spdk_addr", "/var/tmp/spdk.sock", "Points to SPDK unix socket/tcp socket to interact with") @@ -56,9 +67,14 @@ func main() { var tlsFiles string flag.StringVar(&tlsFiles, "tls", "", "TLS files in server_cert:server_key:ca_cert format.") + go runGatewayServer(grpcPort, httpPort) + runGrpcServer(grpcPort, useKvm, spdkAddress, qmpAddress, ctrlrDir, busesStr, tcpTransportListenAddr, tlsFiles) +} + +func runGrpcServer(grpcPort int, useKvm bool, spdkAddress, qmpAddress, ctrlrDir, busesStr, tcpTransportListenAddr, tlsFiles string) { buses := splitBusesBySeparator(busesStr) - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", grpcPort)) if err != nil { log.Fatalf("failed to listen: %v", err) } @@ -114,8 +130,36 @@ func main() { reflection.Register(s) - log.Printf("Server listening at %v", lis.Addr()) + log.Printf("gRPC server listening at %v", lis.Addr()) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) } } + +func runGatewayServer(grpcPort int, httpPort int) { + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Register gRPC server endpoint + // Note: Make sure the gRPC server is running properly and accessible + mux := runtime.NewServeMux() + opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + err := pc.RegisterInventorySvcHandlerFromEndpoint(ctx, mux, fmt.Sprintf(":%d", grpcPort), opts) + if err != nil { + log.Panic("cannot register handler server") + } + + // Start HTTP server (and proxy calls to gRPC server endpoint) + log.Printf("HTTP Server listening at %v", httpPort) + server := &http.Server{ + Addr: fmt.Sprintf(":%d", httpPort), + Handler: mux, + ReadTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + } + err = server.ListenAndServe() + if err != nil { + log.Panic("cannot start HTTP gateway server") + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 749b4aae..929ec55d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,6 +57,7 @@ services: volumes_from: - spdk ports: + - "8082:8082" - "50051:50051" networks: - opi @@ -65,7 +66,7 @@ services: condition: service_healthy redis: condition: service_healthy - command: sh -c "/opi-spdk-bridge -port=50051 -spdk_addr=/var/tmp/spdk.sock -tcp_trid=$$(getent hosts spdk | awk '{ print $$1 }'):7777" + command: sh -c "/opi-spdk-bridge -grpc_port=50051 -http_port=8082 -spdk_addr=/var/tmp/spdk.sock -tcp_trid=$$(getent hosts spdk | awk '{ print $$1 }'):7777" healthcheck: test: grpcurl -plaintext localhost:50051 list || exit 1 @@ -76,6 +77,15 @@ services: healthcheck: test: ["CMD", "redis-cli", "--raw", "incr", "ping"] + opi-gw-test: + image: curlimages/curl:8.2.1 + networks: + - opi + depends_on: + opi-spdk-server: + condition: service_healthy + command: curl -qkL http://opi-spdk-server:8082/v1/inventory/1/inventory/2 + opi-spdk-client: image: ghcr.io/opiproject/godpu:main@sha256:a6fd61121fc87d887bd58264606a55c36ae0dcea0da9e76856499e2f84a961b0 networks: