Add initial version of C# support to dmSDK (defold#9109)

* Initial extension C api

* Fixed C api test for extension.h

* Added initial Zig support to extender config

* Moved dmExtensionDesc to public header so that Zig can allocate it.

* fixes

* Merge fixes

* Added Zig syntax highlighting support in the editor (defold#7888)

* Removed experimental enum macro

* Added first test for cpp header generation

* Updated generation script

* Fixed compile errors and made the C functions the base implementation

* Added support for enums

* Updated extension.h to generate .hpp header

* build fix for windows

* test fix

* win32 compile fix

* Fixes to generator scripts

* Updated resource system

* Updated libs to use latest resource headers

* Unified back into a single resource.h

* Update documentation. Renamed some struct arguments to make them more consistent

* review cleanup

* html build fix

* Added initial csharp version of dlib

* First version of building testing csharp in dlib

* Added initial  sdk generation for C#

* Added csharp project for the extension library

* Added unit test for C# config file api

* cleanup of unit test

* Add dmsdk.csproj + .cs files to dmSDK

* Added Lua C# wrapper

* Added first iteration of a dmSDK for C#

* Added iOS link support for C# tests

* Update csproj files for iOS support

* Added better fallback for NUGET_PACKAGES

* Use splitlines instead

* Updated projects to dotnet 9 preview

* Install dotnet on CI

* Added msvs for building C# tests

* Added msvs for building C# tests

* Added msvs for building C# tests

* Added msvs for building C# tests

* Fallback to python3 on CI

* Install correct msvc arch for each platform

* workflow fix

* Updated syntax in main-ci.yml

* Added dotnet to android builds

* Disable csharp for html5

* run python3 on ci

* Use python3 in ci.py

* Only build/run dotnet tests if supported

* fix for default platform

* debugging python paths

* x86_64-macos linker fix

* Set fallback PATH in ci.py to find python

Revert some changes,

* Test fix for windows build

* trick to download the dot net sdk

* another cs warmup test

* added debug printouts

* More prewarm debugging

* more debug

* Build sith dotnet daily build

* Only enable csharp for macOS

* Added install instructions for DotNet

* Review cleanup

---------

Co-authored-by: Mats Gisselson <[email protected]>

README_SETUP.md

@@ -95,6 +95,38 @@ You also need `easy_install` to install additional packages.
 </p></details>
 
 
+
+### Required Software - DotNet 9
+
+(optional)
+
+*NOTE* The DotNet 9 preview currently only supports macOS platform
+
+In order to build and test the csharp languange bindings locally, you need to install DotNet.
+
+<details><summary>Install...</summary><p>
+
+There are a few ways to install the DotNet sdk:
+
+* Install via https://dotnet.microsoft.com/en-us/download/dotnet/9.0
+* Install via your package manager
+  * macOS: `brew install dotnet-sdk@preview`
+  * Windows: `choco install dotnet --pre`
+
+* Install via [dotnet-install.sh](https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script):
+
+Bash:
+```sh
+> ./dotnet-install.sh --channel 9.0 --quality preview
+```
+
+PowerShell (Windows):
+```sh
+> ./dotnet-install.ps1 -Channel 9.0 -Quality preview
+```
+</p></details>
+
+
 ### Required Software
 
 <details><summary>macOS...</summary><p>

build_tools/waf_csharp.py

@@ -0,0 +1,209 @@
+# Copyright 2020-2024 The Defold Foundation
+# Copyright 2014-2020 King
+# Copyright 2009-2014 Ragnar Svensson, Christian Murray
+# Licensed under the Defold License version 1.0 (the "License"); you may not use
+# this file except in compliance with the License.
+#
+# You may obtain a copy of the License, together with FAQs at
+# https://www.defold.com/license
+#
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+
+import os, sys, subprocess, shutil, re, socket, stat, glob, zipfile, tempfile, configparser
+from waflib.Configure import conf
+from waflib import Utils, Build, Options, Task, Logs
+from waflib.TaskGen import extension, feature, after, before, task_gen
+from waflib.Logs import error
+from waflib.Task import RUN_ME
+from BuildUtility import BuildUtility, BuildUtilityException, create_build_utility
+import run
+import sdk
+
+
+# For properties, see https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-publish
+Task.task_factory('csproj_stlib', '${DOTNET} publish -c ${MODE} -o ${BUILD_DIR} -v q --artifacts-path ${ARTIFACTS_PATH} -r ${RUNTIME} -p:PublishAot=true -p:NativeLib=Static -p:PublishTrimmed=true -p:IlcDehydrate=false ${SRC[0].abspath()}',
+                      color='BROWN',
+                      after='cxxstlib')
+
+@feature('cs_stlib')
+@before('process_source')
+def compile_csharp_lib(self):
+    if not self.env['DOTNET']:
+        print("Skipping %s, as C# is not supported on this platform", self.name)
+        return
+
+    project = self.path.find_resource(self.project)
+    if not project:
+        self.bld.fatal("Couldn't find project '%s'" % self.project)
+
+    csnodes = project.parent.ant_glob('**/*.cs', quiet=False)
+    if not csnodes:
+        self.bld.fatal("No source files found in project '%s'" % self.project)
+
+
+    libname = os.path.basename(self.project)
+    if libname.startswith('lib'):
+        libname = libname[3:]
+
+    libname_no_suffix = os.path.splitext(libname)[0]
+
+    platform = self.env.PLATFORM
+    if platform in ['armv7-android']:
+        Logs.info("Platform '%s' does not yet support C# building" % platform)
+        return
+
+    build_util = create_build_utility(self.env)
+    if build_util.get_target_os() in ['win32']:
+        prefix = ''
+        suffix = '.lib'
+    else:
+        prefix = 'lib'
+        suffix = '.a'
+
+    def defold_platform_to_dotnet(target_os, target_arch):
+        # https://learn.microsoft.com/en-us/dotnet/core/rid-catalog
+        os_to_dotnet_os = {
+            'macos': 'osx',
+            'win32': 'win',
+        }
+        target_os = os_to_dotnet_os.get(target_os, target_os) # translate the name, or use the original one
+
+        arch_to_dotnet_arch = {
+            'x86_64': 'x64',
+            'arm64': 'arm64',
+            'armv7': 'arm',
+        }
+        target_arch = arch_to_dotnet_arch.get(target_arch, target_arch) # translate the name, or use the original one
+
+        if target_os and target_arch:
+            return f'{target_os}-{target_arch}'
+        return None
+
+    arch = defold_platform_to_dotnet(build_util.get_target_os(), build_util.get_target_architecture())
+
+    tsk = self.create_task('csproj_stlib')
+    tsk.inputs.append(project)
+    tsk.inputs.extend(csnodes)
+
+    # build/cs/osx-arm64/publish/libdlib_cs.a
+    build_dir = self.path.get_bld().make_node(f'cs')
+    target = build_dir.make_node(f'{prefix}{libname_no_suffix}{suffix}')
+    tsk.outputs.append(target)
+
+    tsk.env['RUNTIME'] = arch
+    tsk.env['MODE'] = 'Release' # Debug
+    tsk.env['BUILD_DIR'] = build_dir.abspath()
+    tsk.env['ARTIFACTS_PATH'] = build_dir.abspath()
+
+    lib_path = os.path.join(self.env["PREFIX"], 'lib', self.env['PLATFORM'])
+    inst_to=getattr(self,'install_path', lib_path)
+    if inst_to:
+        install_task = self.add_install_files(install_to=inst_to, install_from=tsk.outputs)
+
+
+def _get_dotnet_version():
+    result = run.shell_command('dotnet --info')
+    lines = result.splitlines()
+    lines = [x.strip() for x in lines]
+
+    i = lines.index('Host:')
+    version = lines[i+1].strip().split()[1]
+
+    for l in lines:
+        if l.startswith('Base Path:'):
+            sdk_dir = l.split('Base Path:')[1].strip()
+            while sdk_dir.endswith('/'):
+                sdk_dir = sdk_dir[:-1]
+            sdk_dir = os.path.dirname(sdk_dir)
+            break
+    return version, sdk_dir
+
+def _get_dotnet_aot_base(nuget_path, dotnet_platform, dotnet_version):
+    return f"{nuget_path}/microsoft.netcore.app.runtime.nativeaot.{dotnet_platform}/{dotnet_version}/runtimes/{dotnet_platform}/native"
+
+def _get_dotnet_nuget_path():
+    result = run.shell_command("dotnet nuget locals global-packages -l")
+    if result is not None and 'global-packages:' in result:
+        nuget_path = result.split('global-packages:')[1].strip()
+
+    if not nuget_path:
+        if build_util.get_target_os() in ('win32'):
+            nuget_path = os.path.expanduser('%%userprofile%%/.nuget/packages')
+        else:
+            nuget_path = os.path.expanduser('~/.nuget/packages')
+    return nuget_path
+
+
+def configure(conf):
+    platform = getattr(Options.options, 'platform', sdk.get_host_platform())
+    if platform == '':
+        platform = sdk.get_host_platform()
+
+    if platform not in ['x86_64-macos', 'arm64-macos']:
+        return # not currently supported
+
+    conf.find_program('dotnet', var='DOTNET', mandatory = True)
+    conf.env.DOTNET_VERSION, conf.env.DOTNET_SDK = _get_dotnet_version()
+
+    build_util = create_build_utility(conf.env)
+
+    dotnet_arch ='arm64'
+    if 'x86_64' in platform:
+        dotnet_arch = 'x64'
+
+    platform_to_cs = {
+        'macos': 'osx',
+        'win32': 'win'
+    }
+
+    dotnet_os = platform_to_cs.get(build_util.get_target_os(), build_util.get_target_os())
+    dotnet_platform = '%s-%s' % (dotnet_os, dotnet_arch)
+
+    nuget_path = _get_dotnet_nuget_path()
+    conf.env.NUGET_PACKAGES = nuget_path
+    if not os.path.exists(conf.env.NUGET_PACKAGES):
+        print(f"'NUGET_PACKAGES' not found. Using NUGET_PACKAGES={nuget_path}")
+        os.makedirs(conf.env.NUGET_PACKAGES)
+        if not os.path.exists(conf.env.NUGET_PACKAGES):
+            conf.fatal("Couldn't find C# nuget packages: '%s'" % conf.env.NUGET_PACKAGES)
+
+    aot_base = _get_dotnet_aot_base(nuget_path, dotnet_platform, conf.env.DOTNET_VERSION)
+
+    if build_util.get_target_os() in ('win32'):
+        pass
+    else:
+        # Since there are dynamic libraries with the same name, ld will choose them by default.
+        # only way to override that behavior is to explicitly specify the paths to the static libraries
+        if build_util.get_target_os() in ('macos'):
+            conf.env['LINKFLAGS_CSHARP'] = [
+                f'{aot_base}/libbootstrapperdll.o',
+                f'{aot_base}/libRuntime.WorkstationGC.a',
+                f'{aot_base}/libeventpipe-enabled.a',
+                f'{aot_base}/libstandalonegc-enabled.a',
+                f'{aot_base}/libstdc++compat.a',
+                f'{aot_base}/libSystem.Native.a',
+                f'{aot_base}/libSystem.IO.Compression.Native.a',
+                f'{aot_base}/libSystem.Globalization.Native.a']
+
+            if platform == 'x86_64-macos':
+                conf.env.append_unique('LINKFLAGS_CSHARP', f'{aot_base}/libRuntime.VxsortDisabled.a')
+
+        elif build_util.get_target_os() in ('ios'):
+            conf.env['LINKFLAGS_CSHARP'] = [
+                f'{aot_base}/libbootstrapperdll.o',
+                f'{aot_base}/libRuntime.WorkstationGC.a',
+                f'{aot_base}/libeventpipe-disabled.a',
+                f'{aot_base}/libstandalonegc-disabled.a',
+                f'{aot_base}/libstdc++compat.a',
+                f'{aot_base}/libSystem.Native.a',
+                f'{aot_base}/libSystem.Globalization.Native.a',
+                f'{aot_base}/libSystem.IO.Compression.Native.a',
+                f'{aot_base}/libSystem.Net.Security.Native.a',
+                f'{aot_base}/libSystem.Security.Cryptography.Native.Apple.a',
+                f'{aot_base}/licucore.a']
+
+        if build_util.get_target_os() in ('macos'):
+            conf.env['FRAMEWORK_CSHARP'] = ["AGL","OpenAL","OpenGL","QuartzCore"]

build_tools/waf_dynamo.py

@@ -1854,6 +1854,7 @@ def detect(conf):
         conf.env['STLIB_VULKAN'] = Options.options.with_vulkan_validation and 'vulkan' or 'MoltenVK'
         conf.env['FRAMEWORK_VULKAN'] = ['Metal', 'IOSurface', 'QuartzCore']
         conf.env['FRAMEWORK_DMGLFW'] = ['QuartzCore']
+
     elif platform in ('arm64-ios','x86_64-ios'):
         conf.env['STLIB_VULKAN'] = 'MoltenVK'
         conf.env['FRAMEWORK_VULKAN'] = ['Metal', 'IOSurface']
@@ -1903,6 +1904,8 @@ def detect(conf):
             conf.env['LIB_JNI'] = ['jni']
             conf.env['LIB_JNI_NOASAN'] = ['jni_noasan']
 
+    conf.load('waf_csharp')
+
     if Options.options.generate_compile_commands:
         conf.load('clang_compilation_database')
 

ci/ci.sh

@@ -34,6 +34,7 @@ source ci/env.sh
 # so we've stored the old paths, and append them.
 export PATH=$PATH:$OLDPATH
 echo "PATH=" $PATH
+echo "PYTHON=" $(which python)
 
 echo "Calling ci.py with args: $@"
 # # -u to run python unbuffered to guarantee that output ends up in the correct order in logs

engine/dlib/sdk_gen.json

@@ -24,7 +24,43 @@
 					"ConfigFileGetInt": "GetInt",
 					"ConfigFileGetFloat": "GetFloat"
 				}
+			},
+			"csharp": {
+				"namespace": "Dlib",
+				"class": "ConfigFile",
+				"dllimport": "dlib",
+				"ignore": [
+					"ConfigFileRegisterExtension",
+					"FConfigFileCreate",
+					"FConfigFileDestroy",
+					"FConfigFileGetString",
+					"FConfigFileGetInt",
+					"FConfigFileGetFloat"
+				],
+				"rename": {
+					"HConfigFile": "Config*",
+					"ConfigFileGetString": "GetString",
+					"ConfigFileGetInt": "GetInt",
+					"ConfigFileGetFloat": "GetFloat",
+					"ConfigFile": "Config"
+				}
+			}
+		}},
+
+		{"file": "src/dmsdk/dlib/hash.h", "languages": {
+			"csharp": {
+				"namespace": "Dlib",
+				"class": "Hash",
+				"dllimport": "dlib",
+				"ignore": [
+					"dmHashReverse32",
+					"dmHashReverse64"
+				],
+				"rename": {
+					"dmHash": "Hash",
+					"":""
+				}
 			}
 		}}
 	]
-}
+}

engine/dlib/src/cs/dlib_cs.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net9.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <PublishAot>true</PublishAot>
+    <!-- Workaround for restoring iOS runtime packs for NativeAOT -->
+    <PublishAotUsingRuntimePack>true</PublishAotUsingRuntimePack>
+    <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
+    <HybridGlobalization>true</HybridGlobalization>
+    <TargetName>$(MSBuildProjectName)</TargetName>
+    <TargetName Condition="'$(OS)' != 'Windows_NT'">lib$(TargetName)</TargetName>
+    <StackTraceSupport>true</StackTraceSupport>
+    <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
+    <AppendRuntimeIdentifierToOutputPath>false</AppendRuntimeIdentifierToOutputPath>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <!-- <PackageReference Include="Microsoft.DotNet.ILCompiler" Version="8.0.0" /> -->
+    <!-- <DirectPInvoke Include="dlib" /> -->
+  </ItemGroup>
+
+</Project>

engine/dlib/src/cs/dmsdk/dlib/configfile_gen.cs

@@ -0,0 +1,53 @@
+// Generated, do not edit!
+// Generated with cwd=/Users/mathiaswesterdahl/work/defold/engine/dlib and cmd=/Users/mathiaswesterdahl/work/defold/scripts/dmsdk/gen_sdk.py -i /Users/mathiaswesterdahl/work/defold/engine/dlib/sdk_gen.json
+
+// Copyright 2020-2024 The Defold Foundation
+// Copyright 2014-2020 King
+// Copyright 2009-2014 Ragnar Svensson, Christian Murray
+// Licensed under the Defold License version 1.0 (the "License"); you may not use
+// this file except in compliance with the License.
+// 
+// You may obtain a copy of the License, together with FAQs at
+// https://www.defold.com/license
+// 
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+using System.Runtime.InteropServices;
+using System.Reflection.Emit;
+
+
+namespace dmSDK.Dlib {
+    public unsafe partial class ConfigFile
+    {
+        /*#
+        * Generated from [ref:ConfigFile]
+        */
+        [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi, Pack=1)]
+        public struct Config
+        {
+        }
+
+        /*#
+        * Generated from [ref:ConfigFileGetString]
+        */
+        [DllImport("dlib", EntryPoint="ConfigFileGetString", CallingConvention = CallingConvention.Cdecl)]
+        public static extern String GetString(Config* config, String key, String default_value);
+
+        /*#
+        * Generated from [ref:ConfigFileGetInt]
+        */
+        [DllImport("dlib", EntryPoint="ConfigFileGetInt", CallingConvention = CallingConvention.Cdecl)]
+        public static extern int GetInt(Config* config, String key, int default_value);
+
+        /*#
+        * Generated from [ref:ConfigFileGetFloat]
+        */
+        [DllImport("dlib", EntryPoint="ConfigFileGetFloat", CallingConvention = CallingConvention.Cdecl)]
+        public static extern float GetFloat(Config* config, String key, float default_value);
+
+    } // ConfigFile
+} // dmSDK.Dlib
+