diff --git a/.github/workflows/axosyslog-docker.yml b/.github/workflows/axosyslog-docker.yml index 500c965685..44ba18c08f 100644 --- a/.github/workflows/axosyslog-docker.yml +++ b/.github/workflows/axosyslog-docker.yml @@ -9,9 +9,6 @@ on: tarball-artifact: required: false type: string - axosyslog-modules-artifact: - required: true - type: string snapshot-version: required: false type: string @@ -37,12 +34,6 @@ jobs: name: ${{ inputs.tarball-artifact }} path: docker/apkbuild/axoflow/axosyslog - - name: Download axosyslog-modules tarball artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.axosyslog-modules-artifact }} - path: docker/apkbuild/axoflow/axosyslog - - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index c32cf4c088..f1bc387142 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -69,29 +69,9 @@ jobs: # with: # pkg-type: nightly - axosyslog-modules: - name: axosyslog-modules - runs-on: ubuntu-latest - steps: - - name: Checkout source - uses: actions/checkout@v4 - - - name: Create AxoSyslog modules tarball - working-directory: docker - run: | - tar -czvf python-modules.tar.gz python-modules/ - - - name: Store axosyslog-modules tarball as artifact - uses: actions/upload-artifact@v4 - with: - name: axosyslog-modules-tarball - path: docker/python-modules.tar.gz - build-and-test: runs-on: ubuntu-latest - needs: - - tarball - - axosyslog-modules + needs: tarball steps: - name: Checkout source uses: actions/checkout@v4 @@ -102,12 +82,6 @@ jobs: name: source-tarball path: docker/apkbuild/axoflow/axosyslog - - name: Download axosyslog-modules tarball artifact - uses: actions/download-artifact@v4 - with: - name: axosyslog-modules-tarball - path: docker/apkbuild/axoflow/axosyslog - - name: Build Docker image uses: docker/build-push-action@v5 with: @@ -125,15 +99,13 @@ jobs: export SYSLOG_NG_VERSION="${{ needs.tarball.outputs.snapshot-version }}" docker/tests/smoke.sh - publish-image: if: github.ref == 'refs/heads/main' uses: ./.github/workflows/axosyslog-docker.yml - needs: [tarball, build-and-test, axosyslog-modules] + needs: [tarball, build-and-test] with: pkg-type: nightly tarball-artifact: source-tarball - axosyslog-modules-artifact: axosyslog-modules-tarball snapshot-version: ${{ needs.tarball.outputs.snapshot-version }} # https://github.com/actions/delete-package-versions/issues/90 diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 24a4697c43..fce55b7120 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -64,29 +64,8 @@ jobs: # with: # pkg-type: stable - # TODO integrate axosyslog-modules with the source tree and remove this - axosyslog-modules: - name: axosyslog-modules - needs: find-draft-release-run - runs-on: ubuntu-latest - steps: - - name: Checkout source - uses: actions/checkout@v4 - - - name: Create AxoSyslog modules tarball - working-directory: docker - run: | - tar -czvf python-modules.tar.gz python-modules/ - - - name: Store axosyslog-modules tarball as artifact - uses: actions/upload-artifact@v4 - with: - name: axosyslog-modules-tarball - path: docker/python-modules.tar.gz - publish-image: uses: ./.github/workflows/axosyslog-docker.yml - needs: axosyslog-modules + needs: find-draft-release-run with: pkg-type: stable - axosyslog-modules-artifact: axosyslog-modules-tarball diff --git a/docker/Makefile.am b/docker/Makefile.am index 717148e803..dd55a95729 100644 --- a/docker/Makefile.am +++ b/docker/Makefile.am @@ -1,8 +1,4 @@ EXTRA_DIST += \ - docker/python-modules/webhook/__init__.py \ - docker/python-modules/webhook/scl/webhook.conf \ - docker/python-modules/webhook/source.py \ - docker/python-modules/requirements.txt \ docker/tests/smoke.sh \ docker/alpine.dockerfile \ docker/apkbuild/axoflow/rootbld-repositories \ diff --git a/docker/apkbuild/axoflow/axosyslog/APKBUILD b/docker/apkbuild/axoflow/axosyslog/APKBUILD index 7ac90a59bf..357fe513fe 100644 --- a/docker/apkbuild/axoflow/axosyslog/APKBUILD +++ b/docker/apkbuild/axoflow/axosyslog/APKBUILD @@ -52,7 +52,6 @@ subpackages=" $pkgname-grpc:_grpc " source="https://github.com/axoflow/axosyslog/releases/download/axosyslog-$pkgver/axosyslog-$pkgver.tar.gz - python-modules.tar.gz " builddir="$srcdir/axosyslog-$pkgver" @@ -147,9 +146,6 @@ _python3() { depends="$pkgname=$pkgver-r$pkgrel python3" install="$subpkgname.post-install" - cat $srcdir/python-modules/requirements.txt >> $pkgdir/usr/lib/syslog-ng/python/requirements.txt - rm $srcdir/python-modules/requirements.txt - mv $srcdir/python-modules/* $pkgdir/usr/lib/syslog-ng/python/syslogng/modules _submv usr/lib/syslog-ng/libmod-python.so usr/lib/syslog-ng/python usr/bin/syslog-ng-update-virtualenv } @@ -178,6 +174,5 @@ _submv() { } sha512sums=" -2f1e0dea4c0ecfc3c77df7e6ac231ee8436c9c78fcb4df8ccdc417fea7d56791fdeb0844ac35f0342ce7c2bea5618d8723b6b54319c556120099eb809873082e axosyslog-4.7.1.tar.gz -SKIP python-modules.tar.gz +SKIP axosyslog-4.7.1.tar.gz " diff --git a/docker/apkbuild/axoflow/axosyslog/APKBUILD-debug.patch b/docker/apkbuild/axoflow/axosyslog/APKBUILD-debug.patch index fe8696a3f4..1741a5a0e9 100644 --- a/docker/apkbuild/axoflow/axosyslog/APKBUILD-debug.patch +++ b/docker/apkbuild/axoflow/axosyslog/APKBUILD-debug.patch @@ -9,7 +9,7 @@ makedepends=" bison bpftool -@@ -86,6 +86,7 @@ +@@ -85,6 +85,7 @@ build() { CFLAGS="$CFLAGS -flto=auto" \ ./configure \ diff --git a/docker/python-modules/requirements.txt b/docker/python-modules/requirements.txt deleted file mode 100644 index 2295c8f4a3..0000000000 --- a/docker/python-modules/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -tornado==6.4 diff --git a/docker/python-modules/webhook/__init__.py b/docker/python-modules/webhook/__init__.py deleted file mode 100644 index 8ccf273a6a..0000000000 --- a/docker/python-modules/webhook/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .source import HTTPSource - -__all__ = [ - "HTTPSource" -] diff --git a/modules/python-modules/Makefile.am b/modules/python-modules/Makefile.am index 788cbd3547..0e7490dd93 100644 --- a/modules/python-modules/Makefile.am +++ b/modules/python-modules/Makefile.am @@ -1,67 +1,74 @@ EXTRA_DIST += \ - modules/python-modules/syslogng/__init__.py \ - modules/python-modules/syslogng/confgen.py \ - modules/python-modules/syslogng/dest.py \ - modules/python-modules/syslogng/logger.py \ - modules/python-modules/syslogng/message.py \ - modules/python-modules/syslogng/parser.py \ - modules/python-modules/syslogng/persist.py \ - modules/python-modules/syslogng/reloc.py \ - modules/python-modules/syslogng/source.py \ - modules/python-modules/syslogng/template.py \ - modules/python-modules/syslogng/debuggercli/__init__.py \ - modules/python-modules/syslogng/debuggercli/choicecompleter.py \ - modules/python-modules/syslogng/debuggercli/commandlinelexer.py \ - modules/python-modules/syslogng/debuggercli/completer.py \ - modules/python-modules/syslogng/debuggercli/completerlang.py \ - modules/python-modules/syslogng/debuggercli/debuglang.py \ - modules/python-modules/syslogng/debuggercli/debuggercli.py \ - modules/python-modules/syslogng/debuggercli/editline.py \ - modules/python-modules/syslogng/debuggercli/getoptlexer.py \ - modules/python-modules/syslogng/debuggercli/langcompleter.py \ - modules/python-modules/syslogng/debuggercli/lexer.py \ - modules/python-modules/syslogng/debuggercli/lexertoken.py \ - modules/python-modules/syslogng/debuggercli/macrocompleter.py \ - modules/python-modules/syslogng/debuggercli/readline.py \ - modules/python-modules/syslogng/debuggercli/syslognginternals.py \ - modules/python-modules/syslogng/debuggercli/tablexer.py \ - modules/python-modules/syslogng/debuggercli/templatelang.py \ - modules/python-modules/syslogng/debuggercli/templatelexer.py \ - modules/python-modules/syslogng/debuggercli/tflang.py \ - modules/python-modules/syslogng/debuggercli/tests/__init__.py \ - modules/python-modules/syslogng/debuggercli/tests/test_completer.py \ - modules/python-modules/syslogng/debuggercli/tests/test_commandlinelexer.py\ - modules/python-modules/syslogng/debuggercli/tests/test_completerlang.py \ - modules/python-modules/syslogng/debuggercli/tests/test_choicecompleter.py\ - modules/python-modules/syslogng/debuggercli/tests/test_debuglang.py \ - modules/python-modules/syslogng/debuggercli/tests/test_debuggercli.py \ - modules/python-modules/syslogng/debuggercli/tests/test_getoptlexer.py \ - modules/python-modules/syslogng/debuggercli/tests/test_langcompleter.py \ - modules/python-modules/syslogng/debuggercli/tests/test_lexer.py \ - modules/python-modules/syslogng/debuggercli/tests/test_macrocompleter.py\ - modules/python-modules/syslogng/debuggercli/tests/test_tablexer.py \ - modules/python-modules/syslogng/debuggercli/tests/test_templatelexer.py \ - modules/python-modules/syslogng/debuggercli/tests/test_templatelang.py \ - modules/python-modules/syslogng/debuggercli/tests/test_tflang.py \ - modules/python-modules/pylintrc \ - modules/python-modules/test_pymodules.sh \ - modules/python-modules/CMakeLists.txt \ - \ - modules/python-modules/syslogng/modules/example/scl/example.conf \ - modules/python-modules/syslogng/modules/example/__init__.py \ - modules/python-modules/syslogng/modules/kubernetes/scl/kubernetes.conf \ - modules/python-modules/syslogng/modules/kubernetes/__init__.py \ - modules/python-modules/syslogng/modules/hypr/tests/test_confgen.py \ - modules/python-modules/syslogng/modules/hypr/tests/test_hypr_audit_source.py \ - modules/python-modules/syslogng/modules/hypr/scl/hypr.conf \ - modules/python-modules/syslogng/modules/hypr/__init__.py \ - modules/python-modules/syslogng/modules/s3/__init__.py \ - modules/python-modules/syslogng/modules/s3/compressable_file_buffer.py \ - modules/python-modules/syslogng/modules/s3/s3_object.py \ - modules/python-modules/syslogng/modules/s3/s3_destination.py \ - modules/python-modules/syslogng/modules/s3/scl/s3.conf \ - modules/python-modules/setup.py \ - modules/python-modules/README.md + modules/python-modules/.gitignore \ + modules/python-modules/CMakeLists.txt \ + modules/python-modules/Makefile.am \ + modules/python-modules/pylintrc \ + modules/python-modules/README.md \ + modules/python-modules/setup.py \ + modules/python-modules/syslogng/__init__.py \ + modules/python-modules/syslogng/confgen.py \ + modules/python-modules/syslogng/debuggercli/__init__.py \ + modules/python-modules/syslogng/debuggercli/choicecompleter.py \ + modules/python-modules/syslogng/debuggercli/commandlinelexer.py \ + modules/python-modules/syslogng/debuggercli/completer.py \ + modules/python-modules/syslogng/debuggercli/completerlang.py \ + modules/python-modules/syslogng/debuggercli/debuggercli.py \ + modules/python-modules/syslogng/debuggercli/debuglang.py \ + modules/python-modules/syslogng/debuggercli/editline.py \ + modules/python-modules/syslogng/debuggercli/getoptlexer.py \ + modules/python-modules/syslogng/debuggercli/langcompleter.py \ + modules/python-modules/syslogng/debuggercli/lexer.py \ + modules/python-modules/syslogng/debuggercli/lexertoken.py \ + modules/python-modules/syslogng/debuggercli/macrocompleter.py \ + modules/python-modules/syslogng/debuggercli/readline.py \ + modules/python-modules/syslogng/debuggercli/syslognginternals.py \ + modules/python-modules/syslogng/debuggercli/tablexer.py \ + modules/python-modules/syslogng/debuggercli/templatelang.py \ + modules/python-modules/syslogng/debuggercli/templatelexer.py \ + modules/python-modules/syslogng/debuggercli/tests/__init__.py \ + modules/python-modules/syslogng/debuggercli/tests/test_choicecompleter.py \ + modules/python-modules/syslogng/debuggercli/tests/test_commandlinelexer.py \ + modules/python-modules/syslogng/debuggercli/tests/test_completer.py \ + modules/python-modules/syslogng/debuggercli/tests/test_completerlang.py \ + modules/python-modules/syslogng/debuggercli/tests/test_debuggercli.py \ + modules/python-modules/syslogng/debuggercli/tests/test_debuglang.py \ + modules/python-modules/syslogng/debuggercli/tests/test_getoptlexer.py \ + modules/python-modules/syslogng/debuggercli/tests/test_langcompleter.py \ + modules/python-modules/syslogng/debuggercli/tests/test_lexer.py \ + modules/python-modules/syslogng/debuggercli/tests/test_macrocompleter.py \ + modules/python-modules/syslogng/debuggercli/tests/test_tablexer.py \ + modules/python-modules/syslogng/debuggercli/tests/test_templatelang.py \ + modules/python-modules/syslogng/debuggercli/tests/test_templatelexer.py \ + modules/python-modules/syslogng/debuggercli/tests/test_tflang.py \ + modules/python-modules/syslogng/debuggercli/tflang.py \ + modules/python-modules/syslogng/dest.py \ + modules/python-modules/syslogng/logger.py \ + modules/python-modules/syslogng/message.py \ + modules/python-modules/syslogng/modules/example/__init__.py \ + modules/python-modules/syslogng/modules/example/scl/example.conf \ + modules/python-modules/syslogng/modules/hypr/__init__.py \ + modules/python-modules/syslogng/modules/hypr/README.md \ + modules/python-modules/syslogng/modules/hypr/scl/hypr.conf \ + modules/python-modules/syslogng/modules/hypr/tests/test_confgen.py \ + modules/python-modules/syslogng/modules/hypr/tests/test_hypr_audit_source.py \ + modules/python-modules/syslogng/modules/kubernetes/__init__.py \ + modules/python-modules/syslogng/modules/kubernetes/scl/kubernetes.conf \ + modules/python-modules/syslogng/modules/kubernetes/tests/test_kubernetes_api_enrichment.py \ + modules/python-modules/syslogng/modules/s3/__init__.py \ + modules/python-modules/syslogng/modules/s3/compressable_file_buffer.py \ + modules/python-modules/syslogng/modules/s3/s3_destination.py \ + modules/python-modules/syslogng/modules/s3/s3_object.py \ + modules/python-modules/syslogng/modules/s3/scl/s3.conf \ + modules/python-modules/syslogng/modules/webhook/__init__.py \ + modules/python-modules/syslogng/modules/webhook/scl/webhook.conf \ + modules/python-modules/syslogng/modules/webhook/source.py \ + modules/python-modules/syslogng/parser.py \ + modules/python-modules/syslogng/persist.py \ + modules/python-modules/syslogng/reloc.py \ + modules/python-modules/syslogng/source.py \ + modules/python-modules/syslogng/template.py \ + modules/python-modules/test_pymodules.sh + if ENABLE_PYTHON diff --git a/modules/python-modules/setup.py b/modules/python-modules/setup.py index 89b509db23..c52369218f 100755 --- a/modules/python-modules/setup.py +++ b/modules/python-modules/setup.py @@ -38,6 +38,7 @@ "syslogng.modules.kubernetes", "syslogng.modules.hypr", "syslogng.modules.s3", + "syslogng.modules.webhook", ] requires_addons=[ @@ -47,6 +48,8 @@ "requests", # s3 "boto3", + # webhook + "tornado", ] packages = packages_builtin diff --git a/modules/python-modules/syslogng/modules/webhook/__init__.py b/modules/python-modules/syslogng/modules/webhook/__init__.py new file mode 100644 index 0000000000..39f40cab43 --- /dev/null +++ b/modules/python-modules/syslogng/modules/webhook/__init__.py @@ -0,0 +1,27 @@ +############################################################################# +# Copyright (c) 2024 László Várady +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 as published +# by the Free Software Foundation, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# As an additional exemption you are allowed to compile & link against the +# OpenSSL libraries as published by the OpenSSL project. See the file +# COPYING for details. +# +############################################################################# + +from .source import HTTPSource + +__all__ = [ + "HTTPSource" +] diff --git a/docker/python-modules/webhook/scl/webhook.conf b/modules/python-modules/syslogng/modules/webhook/scl/webhook.conf similarity index 100% rename from docker/python-modules/webhook/scl/webhook.conf rename to modules/python-modules/syslogng/modules/webhook/scl/webhook.conf diff --git a/docker/python-modules/webhook/source.py b/modules/python-modules/syslogng/modules/webhook/source.py similarity index 98% rename from docker/python-modules/webhook/source.py rename to modules/python-modules/syslogng/modules/webhook/source.py index 22bcadda48..4f5418eaa6 100644 --- a/docker/python-modules/webhook/source.py +++ b/modules/python-modules/syslogng/modules/webhook/source.py @@ -34,6 +34,7 @@ signal.signal(signal.SIGTERM, signal.SIG_IGN) WEBHOOK_QUERY_NV_PREFIX = "webhook.query." + class Handler(tornado.web.RequestHandler): def initialize(self, source) -> None: self.source = source @@ -66,7 +67,7 @@ def _construct_msg(self, request, path_arguments) -> LogMessage: return msg def set_default_headers(self) -> None: - self.set_header("Server", "axosyslog"); + self.set_header("Server", "axosyslog") def get_current_user(self): if not self.source.auth_token: @@ -83,6 +84,7 @@ def get_current_user(self): def write_error(self, status_code: int, **kwargs: Any) -> None: self.set_status(status_code) + class HTTPSource(LogSource): def init(self, options: dict[str, Any]) -> bool: self.logger = logging.getLogger("http") @@ -139,7 +141,7 @@ def suspend(self) -> None: def wakeup(self) -> None: self.suspended.clear() - def request_exit(self) -> None : + def request_exit(self) -> None: asyncio.run_coroutine_threadsafe(self.stopServer(), self.event_loop) pass diff --git a/packaging/debian/axosyslog-mod-python.install b/packaging/debian/axosyslog-mod-python.install index f2527f3007..3f9c00857a 100644 --- a/packaging/debian/axosyslog-mod-python.install +++ b/packaging/debian/axosyslog-mod-python.install @@ -5,5 +5,6 @@ usr/lib/syslog-ng/*/python/syslogng/debuggercli/* usr/lib/syslog-ng/*/python/syslogng/modules/kubernetes/* usr/lib/syslog-ng/*/python/syslogng/modules/hypr/* usr/lib/syslog-ng/*/python/syslogng/modules/s3/* +usr/lib/syslog-ng/*/python/syslogng/modules/webhook/* usr/lib/syslog-ng/*/python/syslogng*.egg-info/* usr/bin/syslog-ng-update-virtualenv diff --git a/requirements.txt b/requirements.txt index 69cb8dbb5f..b44918196f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ six==1.16.0 urllib3==1.26.18 websocket-client==1.3.1 ply==3.11 +tornado==6.4 diff --git a/tests/copyright/policy b/tests/copyright/policy index 218a1e95b5..7322fef8df 100644 --- a/tests/copyright/policy +++ b/tests/copyright/policy @@ -239,6 +239,7 @@ modules/ebpf/.* modules/python-modules/syslogng/modules/hypr/.* modules/python-modules/syslogng/modules/example/.* modules/python-modules/syslogng/modules/s3/.* +modules/python-modules/syslogng/modules/webhook/.* modules/python-modules/setup\.py scripts/build-python-venv\.sh modules/syslogformat/sdata-parser\.[ch]