fix: Expose nodejs templates to django (#28078)

frontend/src/scenes/pipeline/hogfunctions/HogFunctionConfiguration.tsx

@@ -100,7 +100,7 @@ export function HogFunctionConfiguration({
         return <NotFound object="Hog function" />
     }
 
-    const isLegacyPlugin = hogFunction?.template?.id?.startsWith('plugin-')
+    const isLegacyPlugin = (template?.id || hogFunction?.template?.id)?.startsWith('plugin-')
 
     const headerButtons = (
         <>
@@ -167,7 +167,7 @@ export function HogFunctionConfiguration({
 
     const showFilters =
         displayOptions.showFilters ??
-        ['destination', 'internal_destination', 'site_destination', 'broadcast', 'transformation'].includes(type)
+        ['destination', 'internal_destination', 'site_destination', 'broadcast'].includes(type)
     const showExpectedVolume = displayOptions.showExpectedVolume ?? ['destination', 'site_destination'].includes(type)
     const showStatus =
         displayOptions.showStatus ?? ['destination', 'internal_destination', 'email', 'transformation'].includes(type)
@@ -266,8 +266,7 @@ export function HogFunctionConfiguration({
 
                                 {isLegacyPlugin ? (
                                     <LemonBanner type="warning">
-                                        This destination is one of our legacy plugins. It will be deprecated and you
-                                        should instead upgrade
+                                        This is part of our legacy plugins and will eventually be deprecated.
                                     </LemonBanner>
                                 ) : hogFunction?.template && !hogFunction.template.id.startsWith('template-blank-') ? (
                                     <LemonDropdown

mypy-baseline.txt

@@ -149,7 +149,6 @@ posthog/hogql_queries/legacy_compatibility/filter_to_query.py:0: error: Dict ent
 posthog/hogql_queries/legacy_compatibility/filter_to_query.py:0: error: Dict entry 0 has incompatible type "str": "StickinessFilter"; expected "str": "TrendsFilter"  [dict-item]
 posthog/session_recordings/models/session_recording.py:0: error: Argument "distinct_id" to "MissingPerson" has incompatible type "str | None"; expected "str"  [arg-type]
 posthog/session_recordings/models/session_recording.py:0: error: Incompatible type for lookup 'persondistinctid__team_id': (got "Team", expected "str | int")  [misc]
-posthog/models/hog_functions/hog_function.py:0: error: Argument 1 to "get" of "dict" has incompatible type "str | None"; expected "str"  [arg-type]
 ee/tasks/subscriptions/slack_subscriptions.py:0: error: Item "None" of "datetime | None" has no attribute "strftime"  [union-attr]
 posthog/warehouse/models/table.py:0: error: Item "None" of "DataWarehouseCredential | None" has no attribute "access_key"  [union-attr]
 posthog/warehouse/models/table.py:0: error: Item "None" of "DataWarehouseCredential | None" has no attribute "access_secret"  [union-attr]
@@ -743,11 +742,6 @@ posthog/warehouse/api/external_data_schema.py:0: note: def [_T] get(self, Type,
 posthog/warehouse/api/table.py:0: error: Unused "type: ignore" comment  [unused-ignore]
 posthog/warehouse/api/table.py:0: error: Unused "type: ignore" comment  [unused-ignore]
 posthog/warehouse/api/table.py:0: error: Unused "type: ignore" comment  [unused-ignore]
-posthog/temporal/tests/batch_exports/test_run_updates.py:0: error: Unused "type: ignore" comment  [unused-ignore]
-posthog/temporal/tests/batch_exports/test_run_updates.py:0: error: Unused "type: ignore" comment  [unused-ignore]
-posthog/temporal/tests/batch_exports/test_run_updates.py:0: error: Unused "type: ignore" comment  [unused-ignore]
-posthog/temporal/tests/batch_exports/test_run_updates.py:0: error: Unused "type: ignore" comment  [unused-ignore]
-posthog/temporal/tests/batch_exports/test_batch_exports.py:0: error: TypedDict key must be a string literal; expected one of ("_timestamp", "created_at", "distinct_id", "elements", "elements_chain", ...)  [literal-required]
 posthog/temporal/data_modeling/run_workflow.py:0: error: Dict entry 20 has incompatible type "str": "Literal['complex']"; expected "str": "Literal['text', 'double', 'bool', 'timestamp', 'bigint', 'binary', 'json', 'decimal', 'wei', 'date', 'time']"  [dict-item]
 posthog/temporal/data_modeling/run_workflow.py:0: error: Dict entry 21 has incompatible type "str": "Literal['complex']"; expected "str": "Literal['text', 'double', 'bool', 'timestamp', 'bigint', 'binary', 'json', 'decimal', 'wei', 'date', 'time']"  [dict-item]
 posthog/temporal/data_modeling/run_workflow.py:0: error: Dict entry 22 has incompatible type "str": "Literal['complex']"; expected "str": "Literal['text', 'double', 'bool', 'timestamp', 'bigint', 'binary', 'json', 'decimal', 'wei', 'date', 'time']"  [dict-item]
@@ -787,10 +781,11 @@ posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid
 posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid index type "str" for "dict[Type, Sequence[str]]"; expected type "Type"  [index]
 posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid index type "str" for "dict[Type, Sequence[str]]"; expected type "Type"  [index]
 posthog/temporal/tests/data_imports/test_end_to_end.py:0: error: Unused "type: ignore" comment  [unused-ignore]
-posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: Need type annotation for "_execute_calls" (hint: "_execute_calls: list[<type>] = ...")  [var-annotated]
-posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: Need type annotation for "_execute_async_calls" (hint: "_execute_async_calls: list[<type>] = ...")  [var-annotated]
-posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: Need type annotation for "_cursors" (hint: "_cursors: list[<type>] = ...")  [var-annotated]
-posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: List item 0 has incompatible type "tuple[str, str, int, int, int, int, str, int]"; expected "tuple[str, str, int, int, str, str, str, str]"  [list-item]
+posthog/temporal/tests/batch_exports/test_run_updates.py:0: error: Unused "type: ignore" comment  [unused-ignore]
+posthog/temporal/tests/batch_exports/test_run_updates.py:0: error: Unused "type: ignore" comment  [unused-ignore]
+posthog/temporal/tests/batch_exports/test_run_updates.py:0: error: Unused "type: ignore" comment  [unused-ignore]
+posthog/temporal/tests/batch_exports/test_run_updates.py:0: error: Unused "type: ignore" comment  [unused-ignore]
+posthog/temporal/tests/batch_exports/test_batch_exports.py:0: error: TypedDict key must be a string literal; expected one of ("_timestamp", "created_at", "distinct_id", "elements", "elements_chain", ...)  [literal-required]
 posthog/api/test/test_capture.py:0: error: Statement is unreachable  [unreachable]
 posthog/api/test/test_capture.py:0: error: Incompatible return value type (got "_MonkeyPatchedWSGIResponse", expected "HttpResponse")  [return-value]
 posthog/api/test/test_capture.py:0: error: Module has no attribute "utc"  [attr-defined]
@@ -803,6 +798,10 @@ posthog/api/test/test_capture.py:0: error: Dict entry 0 has incompatible type "s
 posthog/api/test/test_capture.py:0: error: Dict entry 0 has incompatible type "str": "float"; expected "str": "int"  [dict-item]
 posthog/api/test/test_capture.py:0: error: Dict entry 0 has incompatible type "str": "float"; expected "str": "int"  [dict-item]
 posthog/api/test/test_capture.py:0: error: Dict entry 0 has incompatible type "str": "float"; expected "str": "int"  [dict-item]
+posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: Need type annotation for "_execute_calls" (hint: "_execute_calls: list[<type>] = ...")  [var-annotated]
+posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: Need type annotation for "_execute_async_calls" (hint: "_execute_async_calls: list[<type>] = ...")  [var-annotated]
+posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: Need type annotation for "_cursors" (hint: "_cursors: list[<type>] = ...")  [var-annotated]
+posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: List item 0 has incompatible type "tuple[str, str, int, int, int, int, str, int]"; expected "tuple[str, str, int, int, str, str, str, str]"  [list-item]
 posthog/temporal/tests/batch_exports/test_redshift_batch_export_workflow.py:0: error: Incompatible types in assignment (expression has type "str | int", variable has type "int")  [assignment]
 posthog/api/test/batch_exports/conftest.py:0: error: Signature of "run" incompatible with supertype "Worker"  [override]
 posthog/api/test/batch_exports/conftest.py:0: note: Superclass:

plugin-server/src/cdp/cdp-api.ts

@@ -8,6 +8,7 @@ import { FetchExecutorService } from './services/fetch-executor.service'
 import { HogExecutorService, MAX_ASYNC_STEPS } from './services/hog-executor.service'
 import { HogFunctionManagerService } from './services/hog-function-manager.service'
 import { HogWatcherService, HogWatcherState } from './services/hog-watcher.service'
+import { HOG_FUNCTION_TEMPLATES } from './templates'
 import { HogFunctionInvocationResult, HogFunctionQueueParametersFetchRequest, HogFunctionType, LogEntry } from './types'
 
 export class CdpApi {
@@ -42,10 +43,15 @@ export class CdpApi {
         router.post('/api/projects/:team_id/hog_functions/:id/invocations', asyncHandler(this.postFunctionInvocation))
         router.get('/api/projects/:team_id/hog_functions/:id/status', asyncHandler(this.getFunctionStatus()))
         router.patch('/api/projects/:team_id/hog_functions/:id/status', asyncHandler(this.patchFunctionStatus()))
+        router.get('/api/hog_function_templates', this.getHogFunctionTemplates)
 
         return router
     }
 
+    private getHogFunctionTemplates = (req: express.Request, res: express.Response): void => {
+        res.json(HOG_FUNCTION_TEMPLATES)
+    }
+
     private getFunctionStatus =
         () =>
         async (req: express.Request, res: express.Response): Promise<void> => {

plugin-server/src/cdp/hog-transformations/hog-transformer.service.test.ts

@@ -57,10 +57,12 @@ describe('HogTransformer', () => {
 
         hub.mmdb = Reader.openBuffer(brotliDecompressSync(mmdbBrotliContents))
         hogTransformer = new HogTransformerService(hub)
+        await hogTransformer.start()
     })
 
     afterEach(async () => {
         await closeHub(hub)
+        await hogTransformer.stop()
 
         jest.spyOn(hogTransformer['pluginExecutor'], 'execute')
     })
@@ -81,7 +83,7 @@ describe('HogTransformer', () => {
 
             // Start the transformer after inserting functions because it is
             // starting the hogfunction manager which updates the cache
-            await hogTransformer.start()
+            await hogTransformer['hogFunctionManager'].reloadAllHogFunctions()
 
             const event: PluginEvent = createPluginEvent({}, teamId)
             const result = await hogTransformer.transformEvent(event)
@@ -188,7 +190,7 @@ describe('HogTransformer', () => {
             await insertHogFunction(hub.db.postgres, teamId, defaultTransformationFunction)
             await insertHogFunction(hub.db.postgres, teamId, geoIpTransformationFunction)
 
-            await hogTransformer.start()
+            await hogTransformer['hogFunctionManager'].reloadAllHogFunctions()
 
             const createHogFunctionInvocationSpy = jest.spyOn(hogTransformer as any, 'createHogFunctionInvocation')
 
@@ -267,7 +269,7 @@ describe('HogTransformer', () => {
             await insertHogFunction(hub.db.postgres, teamId, deletingTransformationFunction)
             await insertHogFunction(hub.db.postgres, teamId, addingTransformationFunction)
 
-            await hogTransformer.start()
+            await hogTransformer['hogFunctionManager'].reloadAllHogFunctions()
 
             const createHogFunctionInvocationSpy = jest.spyOn(hogTransformer as any, 'createHogFunctionInvocation')
 
@@ -366,7 +368,7 @@ describe('HogTransformer', () => {
             await insertHogFunction(hub.db.postgres, teamId, thirdTransformationFunction)
             await insertHogFunction(hub.db.postgres, teamId, secondTransformationFunction)
             await insertHogFunction(hub.db.postgres, teamId, firstTransformationFunction)
-            await hogTransformer.start()
+            await hogTransformer['hogFunctionManager'].reloadAllHogFunctions()
 
             const createHogFunctionInvocationSpy = jest.spyOn(hogTransformer as any, 'createHogFunctionInvocation')
 
@@ -410,7 +412,7 @@ describe('HogTransformer', () => {
             })
 
             await insertHogFunction(hub.db.postgres, teamId, filterOutPlugin)
-            await hogTransformer.start()
+            await hogTransformer['hogFunctionManager'].reloadAllHogFunctions()
 
             // Set up the spy after hogTransformer is initialized
             executeSpy = jest.spyOn(hogTransformer['pluginExecutor'], 'execute')

plugin-server/src/cdp/hog-transformations/hog-transformer.service.ts

@@ -94,6 +94,10 @@ export class HogTransformerService {
         await this.hogFunctionManager.start(hogTypes)
     }
 
+    public async stop(): Promise<void> {
+        await this.hogFunctionManager.stop()
+    }
+
     private produceAppMetric(metric: HogFunctionAppMetric): Promise<void> {
         const appMetric: AppMetric2Type = {
             app_source: 'hog_function',

plugin-server/src/cdp/legacy-plugins/_transformations/language-url-splitter-app/template.ts

@@ -6,7 +6,7 @@ export const template: HogFunctionTemplate = {
     id: 'plugin-language-url-splitter-app',
     name: 'Language URL splitter',
     description: 'Splits the language from the URL',
-    icon_url: 'https://raw.githubusercontent.com/posthog/language-url-splitter-app/main/logo.png',
+    icon_url: '/static/hedgehog/builder-hog-01.png',
     category: ['Transformation'],
     hog: `return event`,
     inputs_schema: [

plugin-server/src/cdp/legacy-plugins/_transformations/property-filter-plugin/template.ts

@@ -6,7 +6,7 @@ export const template: HogFunctionTemplate = {
     id: 'plugin-property-filter-plugin',
     name: 'Property Filter',
     description: 'This plugin will set all configured properties to null inside an ingested event.',
-    icon_url: 'https://raw.githubusercontent.com/posthog/posthog-property-filter-plugin/main/logo.png',
+    icon_url: 'https://raw.githubusercontent.com/posthog/property-filter-plugin/dev/logo.png',
     category: ['Transformation'],
     hog: `return event`,
     inputs_schema: [

plugin-server/src/cdp/legacy-plugins/_transformations/semver-flattener-plugin/template.ts

@@ -6,7 +6,7 @@ export const template: HogFunctionTemplate = {
     id: 'plugin-semver-flattener-plugin',
     name: 'SemVer Flattener',
     description: 'This plugin will flatten semver versions in the specified properties.',
-    icon_url: 'https://raw.githubusercontent.com/posthog/posthog-semver-flattener-plugin/main/logo.png',
+    icon_url: '/static/transformations/semver-flattener.png',
     category: ['Transformation'],
     hog: `return event`,
     inputs_schema: [

plugin-server/src/cdp/legacy-plugins/_transformations/user-agent-plugin/template.ts

@@ -7,7 +7,7 @@ export const template: HogFunctionTemplate = {
     name: 'User Agent Populator',
     description:
         "Enhances events with user agent details. User Agent plugin allows you to populate events with the $browser, $browser_version for PostHog Clients that don't  typically populate these properties",
-    icon_url: 'https://raw.githubusercontent.com/posthog/useragent-plugin/main/logo.png',
+    icon_url: '/static/transformations/user-agent.png',
     category: ['Transformation'],
     hog: `return event`,
     inputs_schema: [

plugin-server/src/cdp/templates/_transformations/geoip/geoip.template.ts

@@ -6,7 +6,7 @@ export const template: HogFunctionTemplate = {
     id: 'template-geoip',
     name: 'GeoIP',
     description: 'Adds geoip data to the event',
-    icon_url: '/static/hedgehog/builder-hog-01.png',
+    icon_url: '/static/transformations/geoip.png',
     category: ['Custom'],
     hog: `
 // Define the properties to be added to the event

plugin-server/src/cdp/templates/index.ts

@@ -0,0 +1,36 @@
+import { template as downsamplingPlugin } from '../legacy-plugins/_transformations/downsampling-plugin/template'
+import { template as languageUrlSplitterTemplate } from '../legacy-plugins/_transformations/language-url-splitter-app/template'
+import { template as posthogAppUrlParametersToEventPropertiesTemplate } from '../legacy-plugins/_transformations/posthog-app-url-parameters-to-event-properties/template'
+import { template as posthogFilterOutTemplate } from '../legacy-plugins/_transformations/posthog-filter-out-plugin/template'
+import { template as posthogUrlNormalizerTemplate } from '../legacy-plugins/_transformations/posthog-url-normalizer-plugin/template'
+import { template as propertyFilterTemplate } from '../legacy-plugins/_transformations/property-filter-plugin/template'
+import { template as semverFlattenerTemplate } from '../legacy-plugins/_transformations/semver-flattener-plugin/template'
+import { template as taxonomyTemplate } from '../legacy-plugins/_transformations/taxonomy-plugin/template'
+import { template as timestampParserTemplate } from '../legacy-plugins/_transformations/timestamp-parser-plugin/template'
+import { template as userAgentTemplate } from '../legacy-plugins/_transformations/user-agent-plugin/template'
+import { template as webhookTemplate } from './_destinations/webhook/webhook.template'
+import { template as defaultTransformationTemplate } from './_transformations/default/default.template'
+import { template as geoipTemplate } from './_transformations/geoip/geoip.template'
+import { HogFunctionTemplate } from './types'
+
+export const HOG_FUNCTION_TEMPLATES_DESTINATIONS: HogFunctionTemplate[] = [webhookTemplate]
+
+export const HOG_FUNCTION_TEMPLATES_TRANSFORMATIONS: HogFunctionTemplate[] = [
+    defaultTransformationTemplate,
+    geoipTemplate,
+    downsamplingPlugin,
+    languageUrlSplitterTemplate,
+    posthogAppUrlParametersToEventPropertiesTemplate,
+    posthogFilterOutTemplate,
+    posthogUrlNormalizerTemplate,
+    propertyFilterTemplate,
+    semverFlattenerTemplate,
+    taxonomyTemplate,
+    timestampParserTemplate,
+    userAgentTemplate,
+]
+
+export const HOG_FUNCTION_TEMPLATES: HogFunctionTemplate[] = [
+    ...HOG_FUNCTION_TEMPLATES_DESTINATIONS,
+    ...HOG_FUNCTION_TEMPLATES_TRANSFORMATIONS,
+]

plugin-server/src/ingestion/ingestion-consumer.test.ts

@@ -323,6 +323,11 @@ describe('IngestionConsumer', () => {
     })
 
     describe('event batching', () => {
+        beforeEach(async () => {
+            ingester = new IngestionConsumer(hub)
+            await ingester.start()
+        })
+
         it('should batch events based on the distinct_id', async () => {
             const messages = createKafkaMessages([
                 createEvent({ distinct_id: 'distinct-id-1' }),

plugin-server/src/ingestion/ingestion-consumer.ts

@@ -121,7 +121,8 @@ export class IngestionConsumer {
         await this.batchConsumer?.stop()
         status.info('🔁', `${this.name} - stopping kafka producer`)
         await this.kafkaProducer?.disconnect()
-
+        status.info('🔁', `${this.name} - stopping hog transformer`)
+        await this.hogTransformer.stop()
         status.info('👍', `${this.name} - stopped!`)
     }
 

posthog/api/hog_function.py

@@ -13,14 +13,13 @@
 
 from posthog.api.app_metrics2 import AppMetricsMixin
 from posthog.api.forbid_destroy_model import ForbidDestroyModel
-from posthog.api.hog_function_template import HogFunctionTemplateSerializer
+from posthog.api.hog_function_template import HogFunctionTemplateSerializer, HogFunctionTemplates
 from posthog.api.log_entries import LogEntryMixin
 from posthog.api.routing import TeamAndOrgViewSetMixin
 from posthog.api.shared import UserBasicSerializer
 
 from posthog.cdp.filters import compile_filters_bytecode, compile_filters_expr
 from posthog.cdp.services.icons import CDPIconsService
-from posthog.cdp.templates import HOG_FUNCTION_TEMPLATES_BY_ID
 from posthog.cdp.templates._internal.template_legacy_plugin import create_legacy_plugin_template
 from posthog.cdp.validation import compile_hog, generate_template_bytecode, validate_inputs, validate_inputs_schema
 from posthog.cdp.site_functions import get_transpiled_function
@@ -156,7 +155,7 @@ def validate(self, attrs):
         is_create = self.context.get("view") and self.context["view"].action == "create"
 
         template_id = attrs.get("template_id", instance.template_id if instance else None)
-        template = HOG_FUNCTION_TEMPLATES_BY_ID.get(template_id, None)
+        template = HogFunctionTemplates.template(template_id) if template_id else None
 
         if template_id and template_id.startswith("plugin-"):
             template = create_legacy_plugin_template(template_id)