diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index c40776045..39902ae9d 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -23,13 +23,14 @@ jobs: matrix: sanitizer: ['asan', 'msan'] env: - CFLAGS: -g -O0 -fsanitize=${{ matrix.sanitizer == 'asan' && 'address' || 'memory' }} -DZEND_TRACK_ARENA_ALLOC + CFLAGS: -g -O0 -fsanitize=${{ matrix.sanitizer == 'asan' && 'address' || 'memory' }} -DZEND_TRACK_ARENA_ALLOC LDFLAGS: -fsanitize=${{ matrix.sanitizer == 'asan' && 'address' || 'memory' }} CC: clang CXX: clang++ USE_ZEND_ALLOC: 0 LIBRARY_PATH: ${{ github.workspace }}/php/target/lib LD_LIBRARY_PATH: ${{ github.workspace }}/php/target/lib + EDANT_WATCHER_VERSION: next steps: - name: Remove local PHP @@ -95,6 +96,20 @@ jobs: - name: Add PHP to the PATH run: echo "$(pwd)/php/target/bin" >> "$GITHUB_PATH" + - + uses: actions/checkout@v4 + name: Checkout watcher + with: + repository: e-dant/watcher + ref: ${{ env.EDANT_WATCHER_VERSION }} + path: 'edant/watcher' + - + name: Compile edant/watcher + run: | + cd edant/watcher/watcher-c/ + clang -o libwatcher.so ./src/watcher-c.cpp -I ./include -I ../include -std=c++17 -Wall -Wextra -fPIC -shared ${{ matrix.sanitizer == 'msan' && '-fsanitize=memory -fno-omit-frame-pointer -fno-optimize-sibling-calls' || '' }} + sudo cp libwatcher.so /usr/local/lib/libwatcher.so + sudo ldconfig - name: Set Set CGO flags run: | diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 364f1f5f1..6f2db6e84 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,6 +23,7 @@ jobs: env: GOEXPERIMENT: cgocheck2 GOMAXPROCS: 10 + EDANT_WATCHER_VERSION: next steps: - uses: actions/checkout@v4 @@ -43,6 +44,20 @@ jobs: env: phpts: ts debug: true + - + uses: actions/checkout@v4 + name: Checkout watcher + with: + repository: e-dant/watcher + ref: ${{ env.EDANT_WATCHER_VERSION }} + path: 'edant/watcher' + - + name: Compile edant/watcher + run: | + cd edant/watcher/watcher-c/ + gcc -o libwatcher.so ./src/watcher-c.cpp -I ./include -I ../include -std=c++17 -O3 -Wall -Wextra -fPIC -shared + sudo cp libwatcher.so /usr/local/lib/libwatcher.so + sudo ldconfig - name: Set CGO flags run: | diff --git a/Dockerfile b/Dockerfile index 2e70cb9d2..a3ff89b71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,6 +85,16 @@ COPY --link *.* ./ COPY --link caddy caddy COPY --link internal internal COPY --link testdata testdata +COPY --link watcher watcher + +# install edant/watcher (necessary for file watching) +ARG EDANT_WATCHER_VERSION=next +WORKDIR /usr/local/src/watcher +RUN curl -L https://github.com/e-dant/watcher/archive/refs/heads/$EDANT_WATCHER_VERSION.tar.gz | tar xz +WORKDIR /usr/local/src/watcher/watcher-$EDANT_WATCHER_VERSION/watcher-c +RUN gcc -o libwatcher.so ./src/watcher-c.cpp -I ./include -I ../include -std=c++17 -O3 -Wall -Wextra -fPIC -shared && \ + cp libwatcher.so /usr/local/lib/libwatcher.so && \ + ldconfig /usr/local/lib # See https://github.com/docker-library/php/blob/master/8.3/bookworm/zts/Dockerfile#L57-L59 for PHP values ENV CGO_CFLAGS="-DFRANKENPHP_VERSION=$FRANKENPHP_VERSION $PHP_CFLAGS" @@ -104,6 +114,13 @@ FROM common AS runner ENV GODEBUG=cgocheck=0 +# copy watcher shared library +COPY --from=builder /usr/local/lib/libwatcher* /usr/local/lib/ +# fix for the file watcher on arm +RUN apt-get install -y --no-install-recommends libstdc++6 && \ + apt-get clean && \ + ldconfig + COPY --from=builder /usr/local/bin/frankenphp /usr/local/bin/frankenphp RUN setcap cap_net_bind_service=+ep /usr/local/bin/frankenphp && \ frankenphp version diff --git a/alpine.Dockerfile b/alpine.Dockerfile index d9d4f02b2..2422eb6ac 100644 --- a/alpine.Dockerfile +++ b/alpine.Dockerfile @@ -58,21 +58,24 @@ ENV PATH=/usr/local/go/bin:$PATH RUN apk add --no-cache --virtual .build-deps \ $PHPIZE_DEPS \ argon2-dev \ + # Needed for the custom Go build + bash \ brotli-dev \ coreutils \ curl-dev \ + # Needed for the custom Go build + git \ gnu-libiconv-dev \ libsodium-dev \ + # Needed for the file watcher + libstdc++ \ libxml2-dev \ linux-headers \ oniguruma-dev \ openssl-dev \ readline-dev \ sqlite-dev \ - upx \ - # Needed for the custom Go build - git \ - bash + upx # FIXME: temporary workaround for https://github.com/golang/go/issues/68285 WORKDIR / @@ -103,6 +106,16 @@ COPY --link *.* ./ COPY --link caddy caddy COPY --link internal internal COPY --link testdata testdata +COPY --link watcher watcher + +# install edant/watcher (necessary for file watching) +ARG EDANT_WATCHER_VERSION=next +WORKDIR /usr/local/src/watcher +RUN curl -L https://github.com/e-dant/watcher/archive/refs/heads/$EDANT_WATCHER_VERSION.tar.gz | tar xz +WORKDIR /usr/local/src/watcher/watcher-$EDANT_WATCHER_VERSION/watcher-c +RUN gcc -o libwatcher.so ./src/watcher-c.cpp -I ./include -I ../include -std=c++17 -O3 -Wall -Wextra -fPIC -shared && \ + cp libwatcher.so /usr/local/lib/libwatcher.so && \ + ldconfig /usr/local/lib # See https://github.com/docker-library/php/blob/master/8.3/alpine3.20/zts/Dockerfile#L53-L55 ENV CGO_CFLAGS="-DFRANKENPHP_VERSION=$FRANKENPHP_VERSION $PHP_CFLAGS" @@ -122,6 +135,11 @@ FROM common AS runner ENV GODEBUG=cgocheck=0 +# copy watcher shared library (libgcc and libstdc++ are needed for the watcher) +COPY --from=builder /usr/local/lib/libwatcher* /usr/local/lib/ +RUN apk add --no-cache libstdc++ && \ + ldconfig /usr/local/lib + COPY --from=builder /usr/local/bin/frankenphp /usr/local/bin/frankenphp RUN setcap cap_net_bind_service=+ep /usr/local/bin/frankenphp && \ frankenphp version diff --git a/build-static.sh b/build-static.sh index 0965b9bbf..5b4ec1d38 100755 --- a/build-static.sh +++ b/build-static.sh @@ -142,6 +142,16 @@ if [ "${os}" = "linux" ]; then CGO_LDFLAGS="${CGO_LDFLAGS} -lstdc++" fi fi + +# install edant/watcher for file watching (static version) +git clone --branch="${EDANT_WATCHER_VERSION:-next}" https://github.com/e-dant/watcher watcher +cd watcher/watcher-c +gcc -c -o libwatcher.o ./src/watcher-c.cpp -I ./include -I ../include -std=c++17 -Wall -Wextra -fPIC +ar rcs libwatcher.a libwatcher.o +cp libwatcher.a "../../buildroot/lib/libwatcher.a" +cd ../../ +CGO_LDFLAGS="${CGO_LDFLAGS} -lstdc++ ${PWD}/buildroot/lib/libwatcher.a" + export CGO_LDFLAGS LIBPHP_VERSION="$(./buildroot/bin/php-config --version)" diff --git a/caddy/caddy.go b/caddy/caddy.go index d689769d3..13ada1746 100644 --- a/caddy/caddy.go +++ b/caddy/caddy.go @@ -62,6 +62,8 @@ type workerConfig struct { Num int `json:"num,omitempty"` // Env sets an extra environment variable to the given value. Can be specified more than once for multiple environment variables. Env map[string]string `json:"env,omitempty"` + // Directories to watch for file changes + Watch []string `json:"watch,omitempty"` } type FrankenPHPApp struct { @@ -85,7 +87,7 @@ func (f *FrankenPHPApp) Start() error { opts := []frankenphp.Option{frankenphp.WithNumThreads(f.NumThreads), frankenphp.WithLogger(logger), frankenphp.WithMetrics(metrics)} for _, w := range f.Workers { - opts = append(opts, frankenphp.WithWorkers(repl.ReplaceKnown(w.FileName, ""), w.Num, w.Env)) + opts = append(opts, frankenphp.WithWorkers(repl.ReplaceKnown(w.FileName, ""), w.Num, w.Env, w.Watch)) } _, loaded, err := phpInterpreter.LoadOrNew(mainPHPInterpreterKey, func() (caddy.Destructor, error) { @@ -134,7 +136,6 @@ func (f *FrankenPHPApp) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { } f.NumThreads = v - case "worker": wc := workerConfig{} if d.NextArg() { @@ -178,6 +179,13 @@ func (f *FrankenPHPApp) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { wc.Env = make(map[string]string) } wc.Env[args[0]] = args[1] + case "watch": + if !d.NextArg() { + // the default if the watch directory is left empty: + wc.Watch = append(wc.Watch, "./**/*.{php,yaml,yml,twig,env}") + } else { + wc.Watch = append(wc.Watch, d.Val()) + } } if wc.FileName == "" { diff --git a/caddy/caddy_test.go b/caddy/caddy_test.go index 5dbafe490..51bd86a10 100644 --- a/caddy/caddy_test.go +++ b/caddy/caddy_test.go @@ -574,3 +574,31 @@ func TestAutoWorkerConfig(t *testing.T) { "frankenphp_testdata_index_php_ready_workers", )) } + +func TestWorkerWithInactiveWatcher(t *testing.T) { + tester := caddytest.NewTester(t) + tester.InitServer(` + { + skip_install_trust + admin localhost:2999 + http_port 9080 + + frankenphp { + worker { + file ../testdata/worker-with-watcher.php + num 1 + watch ./**/*.php + } + } + } + + localhost:9080 { + root ../testdata + rewrite worker-with-watcher.php + php + } + `, "caddyfile") + + tester.AssertGetResponse("http://localhost:9080", http.StatusOK, "requests:1") + tester.AssertGetResponse("http://localhost:9080", http.StatusOK, "requests:2") +} diff --git a/caddy/php-server.go b/caddy/php-server.go index 1f2ff41c3..bc9b3c1fa 100644 --- a/caddy/php-server.go +++ b/caddy/php-server.go @@ -29,7 +29,7 @@ import ( func init() { caddycmd.RegisterCommand(caddycmd.Command{ Name: "php-server", - Usage: "[--domain ] [--root ] [--listen ] [--worker /path/to/worker.php<,nb-workers>] [--access-log] [--debug] [--no-compress] [--mercure]", + Usage: "[--domain ] [--root ] [--listen ] [--worker /path/to/worker.php<,nb-workers>] [--watch ] [--access-log] [--debug] [--no-compress] [--mercure]", Short: "Spins up a production-ready PHP server", Long: ` A simple but production-ready PHP server. Useful for quick deployments, @@ -48,6 +48,7 @@ For more advanced use cases, see https://github.com/dunglas/frankenphp/blob/main cmd.Flags().StringP("root", "r", "", "The path to the root of the site") cmd.Flags().StringP("listen", "l", "", "The address to which to bind the listener") cmd.Flags().StringArrayP("worker", "w", []string{}, "Worker script") + cmd.Flags().StringArrayP("watch", "", []string{}, "Directory to watch for file changes") cmd.Flags().BoolP("access-log", "a", false, "Enable the access log") cmd.Flags().BoolP("debug", "v", false, "Enable verbose debug logs") cmd.Flags().BoolP("mercure", "m", false, "Enable the built-in Mercure.rocks hub") @@ -73,6 +74,10 @@ func cmdPHPServer(fs caddycmd.Flags) (int, error) { if err != nil { panic(err) } + watch, err := fs.GetStringArray("watch") + if err != nil { + panic(err) + } var workersOption []workerConfig if len(workers) != 0 { @@ -90,6 +95,7 @@ func cmdPHPServer(fs caddycmd.Flags) (int, error) { workersOption = append(workersOption, workerConfig{FileName: parts[0], Num: num}) } + workersOption[0].Watch = watch } if frankenphp.EmbeddedAppPath != "" { diff --git a/dev-alpine.Dockerfile b/dev-alpine.Dockerfile index f630ef68f..7cb8e02de 100644 --- a/dev-alpine.Dockerfile +++ b/dev-alpine.Dockerfile @@ -15,6 +15,8 @@ ENV PHPIZE_DEPS="\ pkgconfig \ re2c" +SHELL ["/bin/ash", "-eo", "pipefail", "-c"] + RUN apk add --no-cache \ $PHPIZE_DEPS \ argon2-dev \ @@ -29,6 +31,9 @@ RUN apk add --no-cache \ zlib-dev \ bison \ nss-tools \ + # file watcher + libstdc++ \ + linux-headers \ # Dev tools \ git \ clang \ @@ -58,6 +63,15 @@ RUN git clone --branch=PHP-8.3 https://github.com/php/php-src.git . && \ echo "opcache.enable=1" >> /usr/local/lib/php.ini && \ php --version +# install edant/watcher (necessary for file watching) +ARG EDANT_WATCHER_VERSION=next +WORKDIR /usr/local/src/watcher +RUN git clone --branch=$EDANT_WATCHER_VERSION https://github.com/e-dant/watcher . +WORKDIR /usr/local/src/watcher/watcher-c +RUN gcc -o libwatcher.so ./src/watcher-c.cpp -I ./include -I ../include -std=c++17 -O3 -Wall -Wextra -fPIC -shared && \ + cp libwatcher.so /usr/local/lib/libwatcher.so && \ + ldconfig /usr/local/lib + WORKDIR /go/src/app COPY . . diff --git a/dev.Dockerfile b/dev.Dockerfile index 3e7142fed..26254a60d 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -15,6 +15,8 @@ ENV PHPIZE_DEPS="\ pkg-config \ re2c" +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + # hadolint ignore=DL3009 RUN apt-get update && \ apt-get -y --no-install-recommends install \ @@ -63,6 +65,15 @@ RUN git clone --branch=PHP-8.3 https://github.com/php/php-src.git . && \ echo "opcache.enable=1" >> /usr/local/lib/php.ini && \ php --version +# install edant/watcher (necessary for file watching) +ARG EDANT_WATCHER_VERSION=next +WORKDIR /usr/local/src/watcher +RUN git clone --branch=$EDANT_WATCHER_VERSION https://github.com/e-dant/watcher . +WORKDIR /usr/local/src/watcher/watcher-c +RUN gcc -o libwatcher.so ./src/watcher-c.cpp -I ./include -I ../include -std=c++17 -O3 -Wall -Wextra -fPIC -shared && \ + cp libwatcher.so /usr/local/lib/libwatcher.so && \ + ldconfig /usr/local/lib + WORKDIR /go/src/app COPY . . diff --git a/docker-bake.hcl b/docker-bake.hcl index 693b45c24..9dcd70e7f 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -14,6 +14,10 @@ variable "GO_VERSION" { default = "1.22" } +variable EDANT_WATCHER_VERSION { + default = "next" +} + variable "SHA" {} variable "LATEST" { @@ -115,6 +119,7 @@ target "default" { } args = { FRANKENPHP_VERSION = VERSION + EDANT_WATCHER_VERSION = EDANT_WATCHER_VERSION } } @@ -140,6 +145,7 @@ target "static-builder" { } args = { FRANKENPHP_VERSION = VERSION + EDANT_WATCHER_VERSION = EDANT_WATCHER_VERSION } secret = ["id=github-token,env=GITHUB_TOKEN"] } diff --git a/docs/config.md b/docs/config.md index 46bf9d037..380dc62fd 100644 --- a/docs/config.md +++ b/docs/config.md @@ -51,6 +51,7 @@ Optionally, the number of threads to create and [worker scripts](worker.md) to s file # Sets the path to the worker script. num # Sets the number of PHP threads to start, defaults to 2x the number of available CPUs. env # Sets an extra environment variable to the given value. Can be specified more than once for multiple environment variables. + watch # Sets the path to watch for file changes. Can be specified more than once for multiple paths. } } } @@ -131,6 +132,51 @@ php_server [] { } ``` +### Watching for File Changes + +Since workers only boot your application once and keep it in memory, any changes +to your PHP files will not be reflected immediately. + +Workers can instead be restarted on file changes via the `watch` directive. +This is useful for development environments. + +```caddyfile +{ + frankenphp { + worker { + file /path/to/app/public/worker.php + watch + } + } +} +``` + +If the `watch` directory is not specified, it will fall back to `./**/*.{php,yaml,yml,twig,env}`, +which watches all `.php`, `.yaml`, `.yml`, `.twig` and `.env` files in the directory and subdirectories +where the FrankenPHP process was started. You can instead also specify one or more directories via a +[shell filename pattern](https://pkg.go.dev/path/filepath#Match): + +```caddyfile +{ + frankenphp { + worker { + file /path/to/app/public/worker.php + watch /path/to/app # watches all files in all subdirectories of /path/to/app + watch /path/to/app/*.php # watches files ending in .php in /path/to/app + watch /path/to/app/**/*.php # watches PHP files in /path/to/app and subdirectories + watch /path/to/app/**/*.{php,twig} # watches PHP and Twig files in /path/to/app and subdirectories + } + } +} +``` + +* The `**` pattern signifies recursive watching +* Directories can also be relative (to where the FrankenPHP process is started from) +* If you have multiple workers defined, all of them will be restarted when a file changes +* Be wary about watching files that are created at runtime (like logs) since they might cause unwanted worker restarts. + +The file watcher is based on [e-dant/watcher](https://github.com/e-dant/watcher). + ### Full Duplex (HTTP/1) When using HTTP/1.x, it may be desirable to enable full-duplex mode to allow writing a response before the entire body @@ -149,7 +195,7 @@ This is an opt-in configuration that needs to be added to the global options in > [!CAUTION] > > Enabling this option may cause old HTTP/1.x clients that don't support full-duplex to deadlock. -This can also be configured using the `CADDY_GLOBAL_OPTIONS` environment config: +> This can also be configured using the `CADDY_GLOBAL_OPTIONS` environment config: ```sh CADDY_GLOBAL_OPTIONS="servers { enable_full_duplex }" diff --git a/docs/worker.md b/docs/worker.md index 0ede0dbd4..25d6b55f8 100644 --- a/docs/worker.md +++ b/docs/worker.md @@ -25,9 +25,16 @@ Use the `--worker` option of the `php-server` command to serve the content of th ./frankenphp php-server --worker /path/to/your/worker/script.php ``` -If your PHP app is [embeded in the binary](embed.md), you can add a custom `Caddyfile` in the root directory of the app. +If your PHP app is [embedded in the binary](embed.md), you can add a custom `Caddyfile` in the root directory of the app. It will be used automatically. +It's also possible to [restart the worker on file changes](config.md#watching-for-file-changes) with the `--watch` option. +The following command will trigger a restart if any file ending in `.php` in the `/path/to/your/app/` directory or subdirectories is modified: + +```console +./frankenphp php-server --worker /path/to/your/worker/script.php --watch "/path/to/your/app/**/*.php" +``` + ## Symfony Runtime The worker mode of FrankenPHP is supported by the [Symfony Runtime Component](https://symfony.com/doc/current/components/runtime.html). diff --git a/frankenphp.c b/frankenphp.c index 7bcc476a8..9d8f9269b 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -1056,3 +1056,22 @@ int frankenphp_execute_script_cli(char *script, int argc, char **argv) { return (intptr_t)exit_status; } + +int frankenphp_execute_php_function(const char *php_function) { + zval retval = {0}; + zend_fcall_info fci = {0}; + zend_fcall_info_cache fci_cache = {0}; + zend_string *func_name = + zend_string_init(php_function, strlen(php_function), 0); + ZVAL_STR(&fci.function_name, func_name); + fci.size = sizeof fci; + fci.retval = &retval; + int success = 0; + + zend_try { success = zend_call_function(&fci, &fci_cache) == SUCCESS; } + zend_end_try(); + + zend_string_release(func_name); + + return success; +} diff --git a/frankenphp.go b/frankenphp.go index 6b6c2acf7..7cd83d051 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -347,6 +347,10 @@ func Init(options ...Option) error { return err } + if err := restartWorkersOnFileChanges(opt.workers); err != nil { + return err + } + if c := logger.Check(zapcore.InfoLevel, "FrankenPHP started 🐘"); c != nil { c.Write(zap.String("php_version", Version().Version), zap.Int("num_threads", opt.numThreads)) } @@ -361,15 +365,11 @@ func Init(options ...Option) error { // Shutdown stops the workers and the PHP runtime. func Shutdown() { - stopWorkers() - close(done) - shutdownWG.Wait() + drainWorkers() + drainThreads() metrics.Shutdown() requestChan = nil - // Always reset the WaitGroup to ensure we're in a clean state - workersReadyWG = sync.WaitGroup{} - // Remove the installed app if EmbeddedAppPath != "" { os.RemoveAll(EmbeddedAppPath) @@ -383,6 +383,11 @@ func go_shutdown() { shutdownWG.Done() } +func drainThreads() { + close(done) + shutdownWG.Wait() +} + func getLogger() *zap.Logger { loggerMu.RLock() defer loggerMu.RUnlock() @@ -862,3 +867,20 @@ func freeArgs(argv []*C.char) { C.free(unsafe.Pointer(arg)) } } + +func executePHPFunction(functionName string) { + cFunctionName := C.CString(functionName) + defer C.free(unsafe.Pointer(cFunctionName)) + + success := C.frankenphp_execute_php_function(cFunctionName) + + if success == 1 { + if c := logger.Check(zapcore.DebugLevel, "php function call successful"); c != nil { + c.Write(zap.String("function", functionName)) + } + } else { + if c := logger.Check(zapcore.ErrorLevel, "php function call failed"); c != nil { + c.Write(zap.String("function", functionName)) + } + } +} diff --git a/frankenphp.h b/frankenphp.h index 8cb3761b0..5feb71bba 100644 --- a/frankenphp.h +++ b/frankenphp.h @@ -56,4 +56,6 @@ void frankenphp_register_bulk_variables(go_string known_variables[27], int frankenphp_execute_script_cli(char *script, int argc, char **argv); +int frankenphp_execute_php_function(const char *php_function); + #endif diff --git a/frankenphp_test.go b/frankenphp_test.go index 302b37f9f..d6ca541ab 100644 --- a/frankenphp_test.go +++ b/frankenphp_test.go @@ -35,6 +35,7 @@ import ( type testOptions struct { workerScript string + watch []string nbWorkers int env map[string]string nbParrallelRequests int @@ -60,7 +61,7 @@ func runTest(t *testing.T, test func(func(http.ResponseWriter, *http.Request), * initOpts := []frankenphp.Option{frankenphp.WithLogger(opts.logger)} if opts.workerScript != "" { - initOpts = append(initOpts, frankenphp.WithWorkers(testDataDir+opts.workerScript, opts.nbWorkers, opts.env)) + initOpts = append(initOpts, frankenphp.WithWorkers(testDataDir+opts.workerScript, opts.nbWorkers, opts.env, opts.watch)) } initOpts = append(initOpts, opts.initOpts...) diff --git a/frankenphp_with_watcher_test.go b/frankenphp_with_watcher_test.go new file mode 100644 index 000000000..0829e22e3 --- /dev/null +++ b/frankenphp_with_watcher_test.go @@ -0,0 +1,94 @@ +package frankenphp_test + +import ( + "github.com/stretchr/testify/assert" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// we have to wait a few milliseconds for the watcher debounce to take effect +const pollingTime = 250 + +// in tests checking for no reload: we will poll 3x250ms = 0.75s +const minTimesToPollForChanges = 3 + +// in tests checking for a reload: we will poll a maximum of 60x250ms = 15s +const maxTimesToPollForChanges = 60 + +func TestWorkersShouldReloadOnMatchingPattern(t *testing.T) { + if isRunningInMsanMode() { + t.Skip("Skipping watcher tests in memory sanitizer mode") + return + } + watch := []string{"./testdata/**/*.txt"} + + runTest(t, func(handler func(http.ResponseWriter, *http.Request), _ *httptest.Server, i int) { + requestBodyHasReset := pollForWorkerReset(t, handler, maxTimesToPollForChanges) + assert.True(t, requestBodyHasReset) + }, &testOptions{nbParrallelRequests: 1, nbWorkers: 1, workerScript: "worker-with-watcher.php", watch: watch}) +} + +func TestWorkersShouldNotReloadOnExcludingPattern(t *testing.T) { + if isRunningInMsanMode() { + t.Skip("Skipping watcher tests in memory sanitizer mode") + + return + } + watch := []string{"./testdata/**/*.php"} + + runTest(t, func(handler func(http.ResponseWriter, *http.Request), _ *httptest.Server, i int) { + requestBodyHasReset := pollForWorkerReset(t, handler, minTimesToPollForChanges) + assert.False(t, requestBodyHasReset) + }, &testOptions{nbParrallelRequests: 1, nbWorkers: 1, workerScript: "worker-with-watcher.php", watch: watch}) +} + +func fetchBody(method string, url string, handler func(http.ResponseWriter, *http.Request)) string { + req := httptest.NewRequest(method, url, nil) + w := httptest.NewRecorder() + handler(w, req) + resp := w.Result() + body, _ := io.ReadAll(resp.Body) + + return string(body) +} + +func isRunningInMsanMode() bool { + cflags := os.Getenv("CFLAGS") + return strings.Contains(cflags, "-fsanitize=memory") +} + +func pollForWorkerReset(t *testing.T, handler func(http.ResponseWriter, *http.Request), limit int) bool { + // first we make an initial request to start the request counter + body := fetchBody("GET", "http://example.com/worker-with-watcher.php", handler) + assert.Equal(t, "requests:1", body) + + // now we spam file updates and check if the request counter resets + for i := 0; i < limit; i++ { + updateTestFile("./testdata/files/test.txt", "updated", t) + time.Sleep(pollingTime * time.Millisecond) + body := fetchBody("GET", "http://example.com/worker-with-watcher.php", handler) + if body == "requests:1" { + return true + } + } + return false +} + +func updateTestFile(fileName string, content string, t *testing.T) { + absFileName, err := filepath.Abs(fileName) + assert.NoError(t, err) + dirName := filepath.Dir(absFileName) + if _, err := os.Stat(dirName); os.IsNotExist(err) { + err = os.MkdirAll(dirName, 0700) + assert.NoError(t, err) + } + bytes := []byte(content) + err = os.WriteFile(absFileName, bytes, 0644) + assert.NoError(t, err) +} diff --git a/go.sum b/go.sum index 938803948..c15d88ef2 100644 --- a/go.sum +++ b/go.sum @@ -2,23 +2,20 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ= github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4= github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/maypok86/otter v1.2.2 h1:jJi0y8ruR/ZcKmJ4FbQj3QQTqKwV+LNrSOo2S1zbF5M= github.com/maypok86/otter v1.2.2/go.mod h1:mKLfoI7v1HOmQMwFgX4QkRk23mX6ge3RDvjdHOWG4R4= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= @@ -29,7 +26,6 @@ github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSz github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= diff --git a/options.go b/options.go index 70b90c6ac..724e75c8b 100644 --- a/options.go +++ b/options.go @@ -21,6 +21,7 @@ type workerOpt struct { fileName string num int env PreparedEnv + watch []string } // WithNumThreads configures the number of PHP threads to start. @@ -41,9 +42,9 @@ func WithMetrics(m Metrics) Option { } // WithWorkers configures the PHP workers to start. -func WithWorkers(fileName string, num int, env map[string]string) Option { +func WithWorkers(fileName string, num int, env map[string]string, watch []string) Option { return func(o *opt) error { - o.workers = append(o.workers, workerOpt{fileName, num, PrepareEnv(env)}) + o.workers = append(o.workers, workerOpt{fileName, num, PrepareEnv(env), watch}) return nil } diff --git a/static-builder.Dockerfile b/static-builder.Dockerfile index 79c9b5e42..42120ff09 100644 --- a/static-builder.Dockerfile +++ b/static-builder.Dockerfile @@ -12,6 +12,9 @@ ENV FRANKENPHP_VERSION=${FRANKENPHP_VERSION} ARG PHP_VERSION='' ENV PHP_VERSION=${PHP_VERSION} +ARG EDANT_WATCHER_VERSION='' +ENV EDANT_WATCHER_VERSION=${EDANT_WATCHER_VERSION} + ARG PHP_EXTENSIONS='' ARG PHP_EXTENSION_LIBS='' ARG CLEAN='' @@ -103,6 +106,7 @@ RUN go mod graph | awk '{if ($1 !~ "@") print $2}' | xargs go get WORKDIR /go/src/app COPY *.* ./ COPY caddy caddy +COPY watcher watcher RUN --mount=type=secret,id=github-token GITHUB_TOKEN=$(cat /run/secrets/github-token) ./build-static.sh && \ rm -Rf dist/static-php-cli/source/* diff --git a/testdata/files/.gitignore b/testdata/files/.gitignore new file mode 100644 index 000000000..2211df63d --- /dev/null +++ b/testdata/files/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/testdata/worker-with-watcher.php b/testdata/worker-with-watcher.php new file mode 100644 index 000000000..248cf469f --- /dev/null +++ b/testdata/worker-with-watcher.php @@ -0,0 +1,11 @@ + len(partsToMatch)-patternSize-1 { + return false + } + } + } + + return true +} + +// we also check for the following bracket syntax: /path/*.{php,twig,yaml} +func matchBracketPattern(pattern string, fileName string) bool { + openingBracket := strings.Index(pattern, "{") + closingBracket := strings.Index(pattern, "}") + + // if there are no brackets we can match regularly + if openingBracket == -1 || closingBracket == -1 { + return matchPattern(pattern, fileName) + } + + beforeTheBrackets := pattern[:openingBracket] + betweenTheBrackets := pattern[openingBracket+1 : closingBracket] + afterTheBrackets := pattern[closingBracket+1:] + + // all bracket entries are checked individually, only one needs to match + // *.{php,twig,yaml} -> *.php, *.twig, *.yaml + for _, pattern := range strings.Split(betweenTheBrackets, ",") { + if matchPattern(beforeTheBrackets+pattern+afterTheBrackets, fileName) { + return true + } + } + + return false +} + +func matchPattern(pattern string, fileName string) bool { + if pattern == "" { + return true + } + patternMatches, err := filepath.Match(pattern, fileName) + if err != nil { + logger.Error("failed to match filename", zap.String("file", fileName), zap.Error(err)) + return false + } + + return patternMatches +} diff --git a/watcher/watch_pattern_test.go b/watcher/watch_pattern_test.go new file mode 100644 index 000000000..8b15afea4 --- /dev/null +++ b/watcher/watch_pattern_test.go @@ -0,0 +1,162 @@ +package watcher + +import ( + "github.com/stretchr/testify/assert" + "path/filepath" + "testing" +) + +func TestDisallowOnEventTypeBiggerThan3(t *testing.T) { + const fileName = "/some/path/watch-me.php" + const eventType = 4 + + watchPattern, err := parseFilePattern("/some/path") + + assert.NoError(t, err) + assert.False(t, watchPattern.allowReload(fileName, eventType, 0)) +} + +func TestDisallowOnPathTypeBiggerThan2(t *testing.T) { + const fileName = "/some/path/watch-me.php" + const pathType = 3 + + watchPattern, err := parseFilePattern("/some/path") + + assert.NoError(t, err) + assert.False(t, watchPattern.allowReload(fileName, 0, pathType)) +} + +func TestWatchesCorrectDir(t *testing.T) { + hasDir(t, "/path", "/path") + hasDir(t, "/path/", "/path") + hasDir(t, "/path/**/*.php", "/path") + hasDir(t, "/path/*.php", "/path") + hasDir(t, "/path/*/*.php", "/path") + hasDir(t, "/path/?dir/*.php", "/path") + hasDir(t, ".", relativeDir(t, "")) + hasDir(t, "./", relativeDir(t, "")) + hasDir(t, "./**", relativeDir(t, "")) + hasDir(t, "..", relativeDir(t, "/..")) +} + +func TestValidRecursiveDirectories(t *testing.T) { + shouldMatch(t, "/path", "/path/file.php") + shouldMatch(t, "/path", "/path/subpath/file.php") + shouldMatch(t, "/path/", "/path/subpath/file.php") + shouldMatch(t, "/path**", "/path/subpath/file.php") + shouldMatch(t, "/path/**", "/path/subpath/file.php") + shouldMatch(t, "/path/**/", "/path/subpath/file.php") + shouldMatch(t, ".", relativeDir(t, "file.php")) + shouldMatch(t, ".", relativeDir(t, "subpath/file.php")) + shouldMatch(t, "./**", relativeDir(t, "subpath/file.php")) + shouldMatch(t, "..", relativeDir(t, "subpath/file.php")) +} + +func TestInvalidRecursiveDirectories(t *testing.T) { + shouldNotMatch(t, "/path", "/other/file.php") + shouldNotMatch(t, "/path/**", "/other/file.php") + shouldNotMatch(t, ".", "/other/file.php") +} + +func TestValidNonRecursiveFilePatterns(t *testing.T) { + shouldMatch(t, "/*.php", "/file.php") + shouldMatch(t, "/path/*.php", "/path/file.php") + shouldMatch(t, "/path/?ile.php", "/path/file.php") + shouldMatch(t, "/path/file.php", "/path/file.php") + shouldMatch(t, "*.php", relativeDir(t, "file.php")) + shouldMatch(t, "./*.php", relativeDir(t, "file.php")) +} + +func TestInValidNonRecursiveFilePatterns(t *testing.T) { + shouldNotMatch(t, "/path/*.txt", "/path/file.php") + shouldNotMatch(t, "/path/*.php", "/path/subpath/file.php") + shouldNotMatch(t, "/*.php", "/path/file.php") + shouldNotMatch(t, "*.txt", relativeDir(t, "file.php")) + shouldNotMatch(t, "*.php", relativeDir(t, "subpath/file.php")) +} + +func TestValidRecursiveFilePatterns(t *testing.T) { + shouldMatch(t, "/path/**/*.php", "/path/file.php") + shouldMatch(t, "/path/**/*.php", "/path/subpath/file.php") + shouldMatch(t, "/path/**/?ile.php", "/path/subpath/file.php") + shouldMatch(t, "/path/**/file.php", "/path/subpath/file.php") + shouldMatch(t, "**/*.php", relativeDir(t, "file.php")) + shouldMatch(t, "**/*.php", relativeDir(t, "subpath/file.php")) + shouldMatch(t, "./**/*.php", relativeDir(t, "subpath/file.php")) +} + +func TestInvalidRecursiveFilePatterns(t *testing.T) { + shouldNotMatch(t, "/path/**/*.txt", "/path/file.php") + shouldNotMatch(t, "/path/**/*.txt", "/other/file.php") + shouldNotMatch(t, "/path/**/*.txt", "/path/subpath/file.php") + shouldNotMatch(t, "/path/**/?ilm.php", "/path/subpath/file.php") + shouldNotMatch(t, "**/*.php", "/other/file.php") + shouldNotMatch(t, ".**/*.php", "/other/file.php") + shouldNotMatch(t, "./**/*.php", "/other/file.php") +} + +func TestValidDirectoryPatterns(t *testing.T) { + shouldMatch(t, "/path/*/*.php", "/path/subpath/file.php") + shouldMatch(t, "/path/*/*/*.php", "/path/subpath/subpath/file.php") + shouldMatch(t, "/path/?/*.php", "/path/1/file.php") + shouldMatch(t, "/path/**/vendor/*.php", "/path/vendor/file.php") + shouldMatch(t, "/path/**/vendor/*.php", "/path/subpath/vendor/file.php") + shouldMatch(t, "/path/**/vendor/**/*.php", "/path/vendor/file.php") + shouldMatch(t, "/path/**/vendor/**/*.php", "/path/subpath/subpath/vendor/subpath/subpath/file.php") + shouldMatch(t, "/path/**/vendor/*/*.php", "/path/subpath/subpath/vendor/subpath/file.php") + shouldMatch(t, "/path*/path*/*", "/path1/path2/file.php") +} + +func TestInvalidDirectoryPatterns(t *testing.T) { + shouldNotMatch(t, "/path/subpath/*.php", "/path/other/file.php") + shouldNotMatch(t, "/path/*/*.php", "/path/subpath/subpath/file.php") + shouldNotMatch(t, "/path/?/*.php", "/path/subpath/file.php") + shouldNotMatch(t, "/path/*/*/*.php", "/path/subpath/file.php") + shouldNotMatch(t, "/path/*/*/*.php", "/path/subpath/subpath/subpath/file.php") + shouldNotMatch(t, "/path/**/vendor/*.php", "/path/subpath/vendor/subpath/file.php") + shouldNotMatch(t, "/path/**/vendor/*.php", "/path/subpath/file.php") + shouldNotMatch(t, "/path/**/vendor/**/*.php", "/path/subpath/file.php") + shouldNotMatch(t, "/path/**/vendor/**/*.txt", "/path/subpath/vendor/subpath/file.php") + shouldNotMatch(t, "/path/**/vendor/**/*.php", "/path/subpath/subpath/subpath/file.php") + shouldNotMatch(t, "/path/**/vendor/*/*.php", "/path/subpath/vendor/subpath/subpath/file.php") + shouldNotMatch(t, "/path*/path*", "/path1/path1/file.php") +} + +func TestValidExtendedPatterns(t *testing.T) { + shouldMatch(t, "/path/*.{php}", "/path/file.php") + shouldMatch(t, "/path/*.{php,twig}", "/path/file.php") + shouldMatch(t, "/path/*.{php,twig}", "/path/file.twig") + shouldMatch(t, "/path/**/{file.php,file.twig}", "/path/subpath/file.twig") + shouldMatch(t, "/path/{folder1,folder2}/file.php", "/path/folder1/file.php") +} + +func TestInValidExtendedPatterns(t *testing.T) { + shouldNotMatch(t, "/path/*.{php}", "/path/file.txt") + shouldNotMatch(t, "/path/*.{php,twig}", "/path/file.txt") + shouldNotMatch(t, "/path/{file.php,file.twig}", "/path/file.txt") + shouldNotMatch(t, "/path/{folder1,folder2}/file.php", "/path/folder3/file.php") +} + +func relativeDir(t *testing.T, relativePath string) string { + dir, err := filepath.Abs("./" + relativePath) + assert.NoError(t, err) + return dir +} + +func hasDir(t *testing.T, pattern string, dir string) { + watchPattern, err := parseFilePattern(pattern) + assert.NoError(t, err) + assert.Equal(t, dir, watchPattern.dir) +} + +func shouldMatch(t *testing.T, pattern string, fileName string) { + watchPattern, err := parseFilePattern(pattern) + assert.NoError(t, err) + assert.True(t, watchPattern.allowReload(fileName, 0, 0)) +} + +func shouldNotMatch(t *testing.T, pattern string, fileName string) { + watchPattern, err := parseFilePattern(pattern) + assert.NoError(t, err) + assert.False(t, watchPattern.allowReload(fileName, 0, 0)) +} diff --git a/watcher/watcher-c.h b/watcher/watcher-c.h new file mode 100644 index 000000000..5d6163c4f --- /dev/null +++ b/watcher/watcher-c.h @@ -0,0 +1,79 @@ +#pragma once + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Represents "what happened" to a path. */ +static const int8_t WTR_WATCHER_EFFECT_RENAME = 0; +static const int8_t WTR_WATCHER_EFFECT_MODIFY = 1; +static const int8_t WTR_WATCHER_EFFECT_CREATE = 2; +static const int8_t WTR_WATCHER_EFFECT_DESTROY = 3; +static const int8_t WTR_WATCHER_EFFECT_OWNER = 4; +static const int8_t WTR_WATCHER_EFFECT_OTHER = 5; + +/* Represents "what kind" of path it is. */ +static const int8_t WTR_WATCHER_PATH_DIR = 0; +static const int8_t WTR_WATCHER_PATH_FILE = 1; +static const int8_t WTR_WATCHER_PATH_HARD_LINK = 2; +static const int8_t WTR_WATCHER_PATH_SYM_LINK = 3; +static const int8_t WTR_WATCHER_PATH_WATCHER = 4; +static const int8_t WTR_WATCHER_PATH_OTHER = 5; + +/* The `event` object is used to carry information about + filesystem events to the user through the (user-supplied) + callback given to `watch`. + The `event` object will contain the: + - `path_name`: The path to the event. + - `path_type`: One of: + - dir + - file + - hard_link + - sym_link + - watcher + - other + - `effect_type`: One of: + - rename + - modify + - create + - destroy + - owner + - other + - `effect_time`: + The time of the event in nanoseconds since epoch. +*/ +struct wtr_watcher_event { + int64_t effect_time; + char const *path_name; + char const *associated_path_name; + int8_t effect_type; + int8_t path_type; +}; + +/* Ensure the user's callback can receive + events and will return nothing. */ +typedef void (*wtr_watcher_callback)(struct wtr_watcher_event event, + void *context); + +void *wtr_watcher_open(char const *const path, wtr_watcher_callback callback, + void *context); + +bool wtr_watcher_close(void *watcher); + +/* The user, or the language we're working with, + might not prefer a callback-style API. + We provide a pipe-based API for these cases. + Instead of forwarding events to a callback, + we write json-serialized events to a pipe. */ +void *wtr_watcher_open_pipe(char const *const path, int *read_fd, + int *write_fd); + +bool wtr_watcher_close_pipe(void *watcher, int read_fd, int write_fd); + +#ifdef __cplusplus +} +#endif diff --git a/watcher/watcher.c b/watcher/watcher.c new file mode 100644 index 000000000..3d2156121 --- /dev/null +++ b/watcher/watcher.c @@ -0,0 +1,22 @@ +#include "_cgo_export.h" +#include "watcher-c.h" + +void handle_event(struct wtr_watcher_event event, void *data) { + go_handle_file_watcher_event((char *)event.path_name, event.effect_type, + event.path_type, (uintptr_t)data); +} + +uintptr_t start_new_watcher(char const *const path, uintptr_t data) { + void *watcher = wtr_watcher_open(path, handle_event, (void *)data); + if (watcher == NULL) { + return 0; + } + return (uintptr_t)watcher; +} + +int stop_watcher(uintptr_t watcher) { + if (!wtr_watcher_close((void *)watcher)) { + return 0; + } + return 1; +} diff --git a/watcher/watcher.go b/watcher/watcher.go new file mode 100644 index 000000000..937370361 --- /dev/null +++ b/watcher/watcher.go @@ -0,0 +1,145 @@ +package watcher + +// #cgo LDFLAGS: -lwatcher -lstdc++ +// #cgo CFLAGS: -Wall -Werror +// #include +// #include +// #include "watcher.h" +import "C" +import ( + "errors" + "go.uber.org/zap" + "runtime/cgo" + "sync" + "time" + "unsafe" +) + +type watcher struct { + sessions []C.uintptr_t + callback func() + trigger chan struct{} + stop chan struct{} +} + +// duration to wait before triggering a reload after a file change +const debounceDuration = 150 * time.Millisecond + +var ( + // the currently active file watcher + activeWatcher *watcher + // after stopping the watcher we will wait for eventual reloads to finish + reloadWaitGroup sync.WaitGroup + // we are passing the logger from the main package to the watcher + logger *zap.Logger + AlreadyStartedError = errors.New("The watcher is already running") + UnableToStartWatching = errors.New("Unable to start the watcher") +) + +func InitWatcher(filePatterns []string, callback func(), zapLogger *zap.Logger) error { + if len(filePatterns) == 0 { + return nil + } + if activeWatcher != nil { + return AlreadyStartedError + } + logger = zapLogger + activeWatcher = &watcher{callback: callback} + err := activeWatcher.startWatching(filePatterns) + if err != nil { + return err + } + reloadWaitGroup = sync.WaitGroup{} + + return nil +} + +func DrainWatcher() { + if activeWatcher == nil { + return + } + logger.Debug("stopping watcher") + activeWatcher.stopWatching() + reloadWaitGroup.Wait() + activeWatcher = nil +} + +func (w *watcher) startWatching(filePatterns []string) error { + w.trigger = make(chan struct{}) + w.stop = make(chan struct{}) + w.sessions = make([]C.uintptr_t, len(filePatterns)) + watchPatterns, err := parseFilePatterns(filePatterns) + if err != nil { + return err + } + for i, watchPattern := range watchPatterns { + watchPattern.trigger = w.trigger + session, err := startSession(watchPattern) + if err != nil { + return err + } + w.sessions[i] = session + } + go listenForFileEvents(w.trigger, w.stop) + return nil +} + +func (w *watcher) stopWatching() { + close(w.stop) + for _, session := range w.sessions { + stopSession(session) + } +} + +func startSession(w *watchPattern) (C.uintptr_t, error) { + handle := cgo.NewHandle(w) + cDir := C.CString(w.dir) + defer C.free(unsafe.Pointer(cDir)) + watchSession := C.start_new_watcher(cDir, C.uintptr_t(handle)) + if watchSession != 0 { + logger.Debug("watching", zap.String("dir", w.dir), zap.Strings("patterns", w.patterns)) + return watchSession, nil + } + logger.Error("couldn't start watching", zap.String("dir", w.dir)) + + return watchSession, UnableToStartWatching +} + +func stopSession(session C.uintptr_t) { + success := C.stop_watcher(session) + if success == 0 { + logger.Warn("couldn't close the watcher") + } +} + +//export go_handle_file_watcher_event +func go_handle_file_watcher_event(path *C.char, eventType C.int, pathType C.int, handle C.uintptr_t) { + watchPattern := cgo.Handle(handle).Value().(*watchPattern) + if watchPattern.allowReload(C.GoString(path), int(eventType), int(pathType)) { + watchPattern.trigger <- struct{}{} + } +} + +func listenForFileEvents(triggerWatcher chan struct{}, stopWatcher chan struct{}) { + timer := time.NewTimer(debounceDuration) + timer.Stop() + defer timer.Stop() + for { + select { + case <-stopWatcher: + break + case <-triggerWatcher: + timer.Reset(debounceDuration) + case <-timer.C: + timer.Stop() + scheduleReload() + } + } +} + +func scheduleReload() { + logger.Info("filesystem change detected") + reloadWaitGroup.Add(1) + activeWatcher.callback() + reloadWaitGroup.Done() +} diff --git a/watcher/watcher.h b/watcher/watcher.h new file mode 100644 index 000000000..d492cb812 --- /dev/null +++ b/watcher/watcher.h @@ -0,0 +1,6 @@ +#include +#include + +uintptr_t start_new_watcher(char const *const path, uintptr_t data); + +int stop_watcher(uintptr_t watcher); diff --git a/worker.go b/worker.go index 7a65b55b0..bce902d0c 100644 --- a/worker.go +++ b/worker.go @@ -10,8 +10,10 @@ import ( "path/filepath" "runtime/cgo" "sync" + "sync/atomic" "time" + "github.com/dunglas/frankenphp/watcher" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -19,10 +21,18 @@ import ( var ( workersRequestChans sync.Map // map[fileName]chan *http.Request workersReadyWG sync.WaitGroup + workerShutdownWG sync.WaitGroup + workersAreReady atomic.Bool + workersAreDone atomic.Bool + workersDone chan interface{} ) // TODO: start all the worker in parallel to reduce the boot time func initWorkers(opt []workerOpt) error { + workersDone = make(chan interface{}) + workersAreReady.Store(false) + workersAreDone.Store(false) + for _, w := range opt { if err := startWorkers(w.fileName, w.num, w.env); err != nil { return err @@ -38,12 +48,12 @@ func startWorkers(fileName string, nbWorkers int, env PreparedEnv) error { return fmt.Errorf("workers %q: %w", fileName, err) } - if _, ok := workersRequestChans.Load(absFileName); ok { - return fmt.Errorf("workers %q: already started", absFileName) + if _, ok := workersRequestChans.Load(absFileName); !ok { + workersRequestChans.Store(absFileName, make(chan *http.Request)) } - workersRequestChans.Store(absFileName, make(chan *http.Request)) shutdownWG.Add(nbWorkers) + workerShutdownWG.Add(nbWorkers) workersReadyWG.Add(nbWorkers) var ( @@ -59,13 +69,14 @@ func startWorkers(fileName string, nbWorkers int, env PreparedEnv) error { l := getLogger() - const maxBackoff = 16 * time.Second - const minBackoff = 100 * time.Millisecond - const maxConsecutiveFailures = 3 + const maxBackoff = 1 * time.Second + const minBackoff = 10 * time.Millisecond + const maxConsecutiveFailures = 60 for i := 0; i < nbWorkers; i++ { go func() { defer shutdownWG.Done() + defer workerShutdownWG.Done() backoff := minBackoff failureCount := 0 backingOffLock := sync.RWMutex{} @@ -126,13 +137,11 @@ func startWorkers(fileName string, nbWorkers int, env PreparedEnv) error { } // TODO: make the max restart configurable - if _, ok := workersRequestChans.Load(absFileName); ok { + if !workersAreDone.Load() { if fc.ready { fc.ready = false - workersReadyWG.Add(1) } - workersReadyWG.Add(1) if fc.exitStatus == 0 { if c := l.Check(zapcore.InfoLevel, "restarting"); c != nil { c.Write(zap.String("worker", absFileName)) @@ -145,6 +154,7 @@ func startWorkers(fileName string, nbWorkers int, env PreparedEnv) error { backingOffLock.Unlock() metrics.StopWorker(absFileName, StopReasonRestart) } else { + // we will wait a few milliseconds to not overwhelm the logger in case of repeated unexpected terminations if c := l.Check(zapcore.ErrorLevel, "unexpected termination, restarting"); c != nil { backingOffLock.RLock() c.Write(zap.String("worker", absFileName), zap.Int("failure_count", failureCount), zap.Int("exit_status", int(fc.exitStatus)), zap.Duration("waiting", backoff)) @@ -187,6 +197,7 @@ func startWorkers(fileName string, nbWorkers int, env PreparedEnv) error { } workersReadyWG.Wait() + workersAreReady.Store(true) m.Lock() defer m.Unlock() @@ -198,11 +209,41 @@ func startWorkers(fileName string, nbWorkers int, env PreparedEnv) error { } func stopWorkers() { - workersRequestChans.Range(func(k, v any) bool { - workersRequestChans.Delete(k) + workersAreDone.Store(true) + close(workersDone) +} + +func drainWorkers() { + watcher.DrainWatcher() + stopWorkers() + workerShutdownWG.Wait() + workersRequestChans = sync.Map{} +} + +func restartWorkersOnFileChanges(workerOpts []workerOpt) error { + directoriesToWatch := []string{} + for _, w := range workerOpts { + directoriesToWatch = append(directoriesToWatch, w.watch...) + } + restartWorkers := func() { + restartWorkers(workerOpts) + } + if err := watcher.InitWatcher(directoriesToWatch, restartWorkers, getLogger()); err != nil { - return true - }) + return err + } + + return nil +} + +func restartWorkers(workerOpts []workerOpt) { + stopWorkers() + workerShutdownWG.Wait() + if err := initWorkers(workerOpts); err != nil { + logger.Error("failed to restart workers when watching files") + panic(err) + } + logger.Info("workers restarted successfully") } //export go_frankenphp_worker_ready @@ -211,7 +252,9 @@ func go_frankenphp_worker_ready(mrh C.uintptr_t) { fc := mainRequest.Context().Value(contextKey).(*FrankenPHPContext) fc.ready = true metrics.ReadyWorker(fc.scriptFilename) - workersReadyWG.Done() + if !workersAreReady.Load() { + workersReadyWG.Done() + } } //export go_frankenphp_worker_handle_request_start @@ -234,10 +277,11 @@ func go_frankenphp_worker_handle_request_start(mrh C.uintptr_t) C.uintptr_t { var r *http.Request select { - case <-done: + case <-workersDone: if c := l.Check(zapcore.DebugLevel, "shutting down"); c != nil { c.Write(zap.String("worker", fc.scriptFilename)) } + executePHPFunction("opcache_reset") return 0 case r = <-rc: diff --git a/worker_test.go b/worker_test.go index 27f9bbca4..25d25f993 100644 --- a/worker_test.go +++ b/worker_test.go @@ -117,8 +117,8 @@ func TestWorkerGetOpt(t *testing.T) { func ExampleServeHTTP_workers() { if err := frankenphp.Init( - frankenphp.WithWorkers("worker1.php", 4, map[string]string{"ENV1": "foo"}), - frankenphp.WithWorkers("worker2.php", 2, map[string]string{"ENV2": "bar"}), + frankenphp.WithWorkers("worker1.php", 4, map[string]string{"ENV1": "foo"}, []string{}), + frankenphp.WithWorkers("worker2.php", 2, map[string]string{"ENV2": "bar"}, []string{}), ); err != nil { panic(err) }