diff --git a/package-lock.json b/package-lock.json index bc853e7751..f6a969acc8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10592,6 +10592,10 @@ "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, + "node_modules/@opentelemetry/esbuild-plugin": { + "resolved": "packages/esbuild-plugin", + "link": true + }, "node_modules/@opentelemetry/exporter-jaeger": { "version": "1.25.0", "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-jaeger/-/exporter-jaeger-1.25.0.tgz", @@ -36850,6 +36854,31 @@ "integrity": "sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw==", "dev": true }, + "packages/esbuild-plugin": { + "version": "0.1.0", + "license": "Apache-2.0", + "devDependencies": { + "@types/mocha": "7.0.2", + "@types/node": "18.6.5", + "@types/sinon": "10.0.18", + "@types/triple-beam": "1.3.2", + "mocha": "7.2.0", + "nyc": "15.1.0", + "rimraf": "5.0.5", + "sinon": "15.2.0", + "ts-mocha": "10.0.0", + "typescript": "4.4.4" + }, + "engines": { + "node": ">=14" + } + }, + "packages/esbuild-plugin/node_modules/@types/triple-beam": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", + "integrity": "sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g==", + "dev": true + }, "packages/opentelemetry-host-metrics": { "name": "@opentelemetry/host-metrics", "version": "0.35.2", @@ -50934,6 +50963,29 @@ "@opentelemetry/semantic-conventions": "1.25.0" } }, + "@opentelemetry/esbuild-plugin": { + "version": "file:packages/esbuild-plugin", + "requires": { + "@types/mocha": "7.0.2", + "@types/node": "18.6.5", + "@types/sinon": "10.0.18", + "@types/triple-beam": "1.3.2", + "mocha": "7.2.0", + "nyc": "15.1.0", + "rimraf": "5.0.5", + "sinon": "15.2.0", + "ts-mocha": "10.0.0", + "typescript": "4.4.4" + }, + "dependencies": { + "@types/triple-beam": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", + "integrity": "sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g==", + "dev": true + } + } + }, "@opentelemetry/exporter-jaeger": { "version": "1.25.0", "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-jaeger/-/exporter-jaeger-1.25.0.tgz", diff --git a/packages/esbuild-plugin/README.md b/packages/esbuild-plugin/README.md new file mode 100644 index 0000000000..90bf6fcd98 --- /dev/null +++ b/packages/esbuild-plugin/README.md @@ -0,0 +1,10 @@ +# OTel esbuild-plugin + +This is a proposal for a `diagnostics_channel`-based mechanism for bundlers +to hand off a loaded module, at runtime, to possibly active OTel +instrumentations. This is an alternative proposal to +https://github.com/open-telemetry/opentelemetry-js-contrib/pull/1856 + +More details in the PR. + +XXX obviously I need to fill this all in diff --git a/packages/esbuild-plugin/example/.npmrc b/packages/esbuild-plugin/example/.npmrc new file mode 100644 index 0000000000..43c97e719a --- /dev/null +++ b/packages/esbuild-plugin/example/.npmrc @@ -0,0 +1 @@ +package-lock=false diff --git a/packages/esbuild-plugin/example/app.js b/packages/esbuild-plugin/example/app.js new file mode 100644 index 0000000000..f155fc690a --- /dev/null +++ b/packages/esbuild-plugin/example/app.js @@ -0,0 +1,42 @@ +const http = require('http'); +const fastify = require('fastify'); +const {createClient} = require('redis'); +// const clientS3 = require('@aws-sdk/client-s3'); +// console.log('XXX client-s3: ', !!clientS3); + +const redis = createClient(); + +const server = fastify(); +server.get('/ping', async (req, reply) => { + const bar = await redis.get('bar'); + reply.send(`pong (redis key "bar" is: ${bar})`); +}); + +async function main() { + await redis.connect(); + await redis.set('bar', 'baz'); + + await server.listen({port: 3000}); + const port = server.server.address().port; + await new Promise((resolve) => { + http.get(`http://localhost:${port}/ping`, (res) => { + const chunks = []; + res.on('data', (chunk) => { chunks.push(chunk); }); + res.on('end', () => { + console.log('client res: status=%s headers=%s body=%s', + res.statusCode, res.headers, Buffer.concat(chunks).toString()); + resolve(); + }); + }); + }); + server.close(); + + await redis.quit(); + + setTimeout(function () { + console.log('Done lame wait for batch span send.') + // console.log('XXX ', process._getActiveHandles()); + }, 10000); +} + +main(); diff --git a/packages/esbuild-plugin/example/esbuild.mjs b/packages/esbuild-plugin/example/esbuild.mjs new file mode 100644 index 0000000000..6c033811c2 --- /dev/null +++ b/packages/esbuild-plugin/example/esbuild.mjs @@ -0,0 +1,235 @@ + +import * as assert from 'assert'; +import * as fs from 'fs'; +import * as module from 'module'; +import * as path from 'path'; +import * as esbuild from 'esbuild'; +import { FastifyInstrumentation } from '@opentelemetry/instrumentation-fastify'; +import { RedisInstrumentation } from '@opentelemetry/instrumentation-redis-4'; +import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; + +// XXX esbuild plugin for OTel, heavily influenced by https://github.com/DataDog/dd-trace-js/tree/master/packages/datadog-esbuild/ +// TODO: add DD copyright to top of file? e.g. similar to https://github.com/open-telemetry/opentelemetry-js/blob/main/experimental/packages/opentelemetry-instrumentation/hook.mjs + +// XXX does this plugin need to be CommonJS so a CJS-using esbuild.js file can used it? Probably, yes. + +const NAME = '@opentelemetry/esbuild-plugin' +const DEBUG = ['all', 'verbose', 'debug'].includes(process.env.OTEL_LOG_LEVEL.toLowerCase()) +const debug = DEBUG + ? (msg, ...args) => { console.debug(`${NAME} debug: ${msg}`, ...args); } + : () => {}; + +// XXX doc this +function pkgInfoFromPath(abspath) { + const normpath = path.sep !== '/' + ? abspath.replaceAll(path.sep, '/') + : abspath; + const NM = 'node_modules/'; + let idx = normpath.lastIndexOf(NM); + if (idx < 0) { + return; + } + idx += NM.length; + let endIdx = normpath.indexOf('/', idx); + if (endIdx < 0) { + return; + } + if (normpath[idx] === '@') { + endIdx = normpath.indexOf('/', endIdx + 1); + if (endIdx < 0) { + return; + } + } + + assert.equal(path.sep.length, 1); + return { + name: normpath.slice(idx, endIdx), + // XXX doc normalization + fullModulePath: normpath.slice(idx), + pjPath: path.join(abspath.slice(0, endIdx), 'package.json'), + }; +} + +/** + * How this works. Take `require('fastify')`, for example. + * + * - esbuild calls: + * onResolve({path: 'fastify', namespace: 'file' })` + * which the plugin resolves to: + * {path: 'fastify', namespace: 'otel', pluginData}` + * where `pluginData` includes the absolute path to load and the package + * version. Importantly the namespace is changed to 'otel'. + * + * - esbuild calls: + * onLoad({path: 'fastify', namespace: 'otel', pluginData}) + * which the plugin resolves to a stub module that does: + * - `require('${absolute path to module}')`, + * - sends a diag chan message to a possibly waiting OTel SDK to optionally + * patch the loaded module exports, + * - re-exports the, possibly now patched, module + * + * - esbuild calls: + * onResolve({path: '/.../node_modules/fastify/fastify.js', namespace: 'otel' })` + * which the plugin resolves back to the 'file' namespace + * + * - esbuild's default file loading loads the "fastify.js" as usual + * + * Which module paths to stub depends on the patching data for each given OTel + * Instrumentation. Node.js builtin modules, like `net`, need not be stubbed + * because they will be marked external (i.e. not inlined) by esbuild with the + * `platform: 'node'` config. + */ +const CHANNEL_NAME = 'otel:bundle:load'; +function otelPlugin(instrs) { + // XXX move 'intsr' to keyed option + // XXX add debug bool option so can choose in esbuild.mjs file + + return { + name: 'opentelemetry', + setup(build) { + // Skip out gracefully if Node.js is too old for this plugin. + // - Want `module.isBuiltin`, added in node v18.6.0, v16.17.0. + // - Want `diagch.subscribe` added in node v18.7.0, v16.17.0 + // (to avoid https://github.com/nodejs/node/issues/42170). + // Note: these constraints *could* be avoided with added code and deps if + // really necessary. + const [major, minor] = process.versions.node.split('.').map(Number); + if (major < 16 || major === 16 && minor < 17 || major === 18 && minor < 7) { + console.warn(`@opentelemetry/esbuild-plugin warn: this plugin requires at least Node.js v16.17.0, v18.7.0 to work; current version is ${process.version}`) + return; + } + + const externals = new Set(build.initialOptions.external || []); + + // From the given OTel Instrumentation instances, determine which + // load paths (e.g. 'fastify', 'mongodb/lib/sessions.js') will possibly + // need to be patched at runtime. + const pathsToStub = new Set(); + for (let instr of instrs) { + const defns = instr.getModuleDefinitions(); + for (let defn of defns) { + if (typeof defn.patch === 'function') { + pathsToStub.add(defn.name); + } + for (let fileDefn of defn.files) { + pathsToStub.add(fileDefn.name); + } + } + } + debug('module paths to stub:', pathsToStub); + + build.onResolve({ filter: /.*/ }, async (args) => { + if (externals.has(args.path)) { + // If this esbuild is configured to leave a package external, then + // no need to stub for it in the bundle. + return; + } + if (module.isBuiltin(args.path)) { + // Node.js builtin modules are left in the bundle as `require(...)`, + // so no need for stubbing. + return + } + + if (args.namespace === 'file') { + // console.log('XXX onResolve file:', args); + + // This resolves the absolute path of the module, which is used in the stub. + // XXX Not sure if should prefer: + // require.resolve(args.path, {paths: [args.resolveDir]}) + // Dev Note: Most of the bundle-time perf hit from this plugin is + // from this `build.resolve()`. + const resolved = await build.resolve(args.path, { + kind: args.kind, + resolveDir: args.resolveDir + // Implicit `namespace: ''` here avoids recursion. + }); + if (resolved.errors.length > 0) { + return { errors: resolved.errors }; + } + + // Get the package name and version. + const pkgInfo = pkgInfoFromPath(resolved.path) + if (!pkgInfo) { + debug(`skip resolved path, could not determine pkgInfo: "${resolved.path}"`); + return; + } + + let matchPath; + if (pathsToStub.has(args.path)) { + // E.g. `require('fastify')` matches + // `InstrumentationNodeModuleDefinition { name: 'fastify' }` + // from `@opentelemetry/instrumentation-fastify`. + matchPath = args.path; + } else if (pkgInfo.fullModulePath !== args.path && pathsToStub.has(pkgInfo.fullModulePath)) { + // E.g. `require('./multi-commander')` from `@redis/client/...` matches + // `InstrumentationNodeModuleFile { name: '@redis/client/dist/lib/client/multi-command.js' } + // from `@opentelemetry/instrumentation-fastify`. + matchPath = pkgInfo.fullModulePath; + } else { + // This module is not one that given instrumentations care about. + return; + } + + // Get the package version from its package.json. + let pkgVersion; + try { + const pjContent = await fs.promises.readFile(pkgInfo.pjPath); + pkgVersion = JSON.parse(pjContent).version; + } catch (err) { + debug(`skip "${matchPath}": could not determine package version: ${err.message}`); + return; + } + + return { + path: matchPath, + namespace: 'otel', + pluginData: { + fullPath: resolved.path, + pkgName: pkgInfo.name, + pkgVersion, + } + }; + + } else if (args.namespace === 'otel') { + return { + path: args.path, + namespace: 'file', + // We expect `args.path` to always be an absolute path (from + // resolved.path above), so `resolveDir` isn't necessary. + }; + } + }) + + build.onLoad({ filter: /.*/, namespace: 'otel' }, async (args) => { + debug(`stub module "${args.path}"`); + return { + contents: ` + const diagch = require('diagnostics_channel'); + const ch = diagch.channel('${CHANNEL_NAME}'); + const mod = require('${args.pluginData.fullPath}'); + const message = { + name: '${args.path}', + version: '${args.pluginData.pkgVersion}', + exports: mod, + }; + ch.publish(message); + module.exports = message.exports; + `, + loader: 'js', + } + }) + }, + } +} + +await esbuild.build({ + entryPoints: ['app.js'], + bundle: true, + platform: 'node', + target: ['node14'], + outdir: 'build', + plugins: [otelPlugin( + // [ new FastifyInstrumentation(), new RedisInstrumentation(), ] + getNodeAutoInstrumentations(), + )], +}); diff --git a/packages/esbuild-plugin/example/minisdk.js b/packages/esbuild-plugin/example/minisdk.js new file mode 100644 index 0000000000..f9443db183 --- /dev/null +++ b/packages/esbuild-plugin/example/minisdk.js @@ -0,0 +1,5 @@ +var diagch = require("diagnostics_channel"); +diagch.subscribe("otel:bundle:load", (message, name) => { + console.log('minisdk received message:', name, message); +}); + diff --git a/packages/esbuild-plugin/example/package.json b/packages/esbuild-plugin/example/package.json new file mode 100644 index 0000000000..829f47c5c7 --- /dev/null +++ b/packages/esbuild-plugin/example/package.json @@ -0,0 +1,20 @@ +{ + "name": "opentelemetry-esbuild-plugin-example", + "version": "0.1.0", + "private": true, + "scripts": { + "build": "OTEL_LOG_LEVEL=debug node esbuild.mjs" + }, + "XXX dependencies": { + "@opentelemetry/auto-instrumentations-node": "^0.47.1", + "@opentelemetry/instrumentation": "file:../../../../opentelemetry-js10/experimental/packages/opentelemetry-instrumentation", + "@opentelemetry/instrumentation-fastify": "^0.37.0", + "tabula": "^1.10.0" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.600.0", + "@opentelemetry/auto-instrumentations-node": "^0.47.1", + "fastify": "^4.28.0", + "redis": "^4.6.14" + } +} diff --git a/packages/esbuild-plugin/lib/index.js b/packages/esbuild-plugin/lib/index.js new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/esbuild-plugin/package.json b/packages/esbuild-plugin/package.json new file mode 100644 index 0000000000..13724083e8 --- /dev/null +++ b/packages/esbuild-plugin/package.json @@ -0,0 +1,58 @@ +{ + "name": "@opentelemetry/esbuild-plugin", + "version": "0.1.0", + "description": "XXX", + "main": "lib/index.js", + "XXXmain": "build/src/index.js", + "types": "build/src/index.d.ts", + "repository": { + "type": "git", + "url": "git+https://github.com/open-telemetry/opentelemetry-js-contrib.git" + }, + "scripts": { + "test": "nyc ts-mocha -p tsconfig.json 'test/**/*.test.ts'", + "tdd": "npm run test -- --watch-extensions ts --watch", + "clean": "rimraf build/*", + "lint": "eslint . --ext .ts", + "lint:fix": "eslint . --ext .ts --fix", + "precompile": "tsc --version && lerna run version:update --scope @opentelemetry/esbuild-plugin --include-dependencies", + "prewatch": "npm run precompile", + "prepublishOnly": "npm run compile", + "version:update": "node ../../scripts/version-update.js", + "compile": "tsc -p ." + }, + "keywords": [ + "nodejs", + "opentelemetry", + "esbuild", + "bundling" + ], + "author": "OpenTelemetry Authors", + "license": "Apache-2.0", + "engines": { + "node": ">=16.17.0" + }, + "XXXfiles": [ + "build/src/**/*.js", + "build/src/**/*.js.map", + "build/src/**/*.d.ts" + ], + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "@types/mocha": "7.0.2", + "@types/node": "18.6.5", + "@types/sinon": "10.0.18", + "mocha": "7.2.0", + "nyc": "15.1.0", + "rimraf": "5.0.5", + "sinon": "15.2.0", + "ts-mocha": "10.0.0", + "typescript": "4.4.4" + }, + "homepage": "https://github.com/open-telemetry/opentelemetry-js-contrib/tree/main/packages/esbuild-plugin", + "bugs": { + "url": "https://github.com/open-telemetry/opentelemetry-js-contrib/issues" + } +}