Rework SCons options and support crosscompiling (#9)

INSTALL.md

@@ -108,10 +108,11 @@ After building, tools and libraries are inside `bin/` directory.
 * `--with-targets=posix,stdio,gnu,uv,openfec,sox` - manually select source code directories to be included in build
 
 **Arguments**:
-* `variant=debug|release` - select build variant
-* `target=linux` - select target platform; currently only `linux` is supported
-* `compiler=<compiler>[-<version>]` - select compiler and version to use; currently, only `gcc` and `clang` are supported
-* `toolchain=<prefix>` - select toolchain prefix added to compiler, linker and other tools
+* `build={type}` - the type of system on which Roc is being compiled, e.g. `x86_64-pc-linux-gnu`, autodetected if empty
+* `host={type}` - the type of system on which Roc will run, e.g. `arm-linux-gnueabihf`, set equal to `build` if empty; used as prefix for copiler, linker, and similar tools
+* `platform={name}` - platform name on which Roc will run, e.g. `linux`, autodetected from `host` if empty
+* `compiler={name}` - compiler name, e.g. `gcc` or `clang`
+* `variant=${name}` - build variant, e.g. `release` (default) or `debug`
 
 **Build targets:**
 * *omitted* - build everything
@@ -120,8 +121,8 @@ After building, tools and libraries are inside `bin/` directory.
 * `clean` - remove build results
 * `fmt` - format source code (if [clang-format](http://clang.llvm.org/docs/ClangFormat.html) >= 3.6 is found in PATH, it's used with [.clang-format](.clang-format) config)
 * `tidy` - run clang static analyzer (requires clang-tidy to be installed)
-* `<module>` - build only specific module
-* `test/<module>` - build and run tests only for specific module
+* `{module}` - build only specific module
+* `test/{module}` - build and run tests only for specific module
 
 **Environment variables:**
 * `CC`, `CXX`, `LD`, `AR`, `RANLIB`, `GENGETOPT`, `DOXYGEN`, `PKG_CONFIG` - overwrite tools to use

SConstruct

@@ -160,30 +160,30 @@ supported_variants = [
     'release',
 ]
 
-host = '%s_%s' % (
-    platform.system().lower(), platform.machine().lower())
+# type of system on which Roc is being compiled, e.g. 'x86_64-pc-linux-gnu'
+build = ARGUMENTS.get('build', '')
 
-target = ARGUMENTS.get('target', host)
-variant = ARGUMENTS.get('variant', 'release')
-toolchain = ARGUMENTS.get('toolchain', '')
+# type of system on which Roc will run, e.g. 'arm-linux-gnueabihf'
+host = ARGUMENTS.get('host', '')
+
+# platform name on which Roc will run, e.g. 'linux'
+platform = ARGUMENTS.get('platform', '')
+
+# compiler name, e.g. 'gcc'
 compiler = ARGUMENTS.get('compiler', '')
 
-try:
-    target_platform, target_arch = target.split('_', 1)
-except:
-    env.Die("bad target `%s', expected {platform}_{arch}", target)
+# build variant, e.g. 'debug'
+variant = ARGUMENTS.get('variant', 'release')
 
-if not target_platform in supported_platforms and not GetOption('with_targets'):
-    env.Die(("unknown target platform `%s' in `%s', expected one of: %s\n"+
-             "you should either provide known target platform or --with-targets option"),
-                target_platform, target, ', '.join(supported_platforms))
+# toolchain prefix for compiler, linker, etc.
+toolchain = host
 
 if not variant in supported_variants:
     env.Die("unknown variant `%s', expected one of: %s",
             variant, ', '.join(supported_variants))
 
 if not compiler:
-    if host == target and env.Which('clang'):
+    if not toolchain and env.Which('clang'):
         compiler = 'clang'
     else:
         compiler = 'gcc'
@@ -192,18 +192,38 @@ if '-' in compiler:
     compiler, compiler_ver = compiler.split('-')
     compiler_ver = tuple(map(int, compiler_ver.split('.')))
 else:
-    compiler_ver = env.CompilerVersion(compiler)
+    if toolchain:
+        compiler_ver = env.CompilerVersion('%s-%s' % (toolchain, compiler))
+    else:
+        compiler_ver = env.CompilerVersion(compiler)
 
 if not compiler in supported_compilers:
     env.Die("unknown compiler `%s', expected one of: %s",
             compiler, ', '.join(supported_compilers))
 
 if not compiler_ver:
-    env.Die("can't detect compiler version")
+    env.Die("can't detect compiler version for compiler `%s'",
+            '-'.join([s for s in [toolchain, compiler] if s]))
+
+if not build:
+    build = env.CompilerTarget(compiler)
+    if not build:
+        env.Die(("can't detect system type, please specify `build={type}' manually, "+
+                 "e.g. `build=x86_64-pc-linux-gnu'"))
+
+if not host:
+    host = build
+
+if not platform:
+    if 'linux' in host:
+        platform = 'linux'
+
+if not platform in supported_platforms and not GetOption('with_targets'):
+    env.Die(("unknown platform `%s', expected one of: %s\n"+
+             "you should either use known platform or provide --with-targets option"),
+                platform, ', '.join(supported_platforms))
 
-if not toolchain and host != target:
-    env.Die(("toolchain option is required when cross-compiling: "+
-            "host is `%s', tagret is `%s'"), host, target)
+crosscompile = (host != build)
 
 conf = Configure(env, custom_tests=env.CustomTests)
 
@@ -290,9 +310,8 @@ env = conf.Finish()
 compiler_ver = env.CompilerVersion(env['CXX'])
 
 build_dir = 'build/%s/%s' % (
-    '-'.join([s for s in [
-        target, toolchain, compiler, '.'.join(map(str, compiler_ver))] if s]),
-    variant)
+    host,
+    '-'.join([s for s in [compiler, '.'.join(map(str, compiler_ver)), variant] if s]))
 
 if compiler == 'clang':
     for var in ['CC', 'CXX']:
@@ -305,7 +324,7 @@ if GetOption('with_targets'):
     for t in GetOption('with_targets').split(','):
         env['ROC_TARGETS'] += ['target_%s' % t]
 else:
-    if target_platform in ['linux']:
+    if platform in ['linux']:
         env.Append(ROC_TARGETS=[
             'target_posix',
             'target_stdio',
@@ -354,7 +373,7 @@ conf = Configure(env, custom_tests=env.CustomTests)
 if 'target_uv' in extdeps:
     env.TryParseConfig('--cflags --libs libuv')
 
-    if host == target:
+    if not crosscompile:
         if not conf.CheckLibWithHeaderExpr(
             'uv', 'uv.h', 'c', expr='UV_VERSION_MAJOR >= 1 && UV_VERSION_MINOR >= 4'):
             env.Die("libuv >= 1.4 not found (see `config.log' for details)")
@@ -364,7 +383,7 @@ if 'target_uv' in extdeps:
 
 if 'target_openfec' in extdeps:
     if not env.TryParseConfig('--silence-errors --cflags --libs openfec') \
-      and host == target:
+      and not crosscompile:
         for prefix in ['/usr/local', '/usr']:
             if os.path.exists('%s/include/openfec' % prefix):
                 env.Append(CPPPATH=[
@@ -394,7 +413,7 @@ if 'target_openfec' in extdeps:
 if 'target_sox' in extdeps:
     env.TryParseConfig('--cflags --libs sox')
 
-    if host == target:
+    if not crosscompile:
         if not conf.CheckLibWithHeaderExpr(
                 'sox', 'sox.h', 'c',
                 expr='SOX_LIB_VERSION_CODE >= SOX_LIB_VERSION(14, 4, 0)'):
@@ -428,36 +447,36 @@ test_env = test_conf.Finish()
 conf = Configure(env, custom_tests=env.CustomTests)
 
 if 'target_uv' in getdeps:
-    env.ThridParty(toolchain, 'uv-1.4.2')
+    env.ThirdParty(host, toolchain, 'uv-1.4.2')
 
 if 'target_openfec' in getdeps:
-    env.ThridParty(toolchain, 'openfec-1.4.2', includes=[
+    env.ThirdParty(host, toolchain, 'openfec-1.4.2', includes=[
         'lib_common',
         'lib_stable',
     ])
 
 if 'target_sox' in getdeps:
-    env.ThridParty(toolchain, 'sox-14.4.2')
+    env.ThirdParty(host, toolchain, 'alsa-1.0.29')
+    env.ThirdParty(host, toolchain, 'sox-14.4.2', deps=['alsa-1.0.29'])
 
     for lib in [
             'z', 'ltdl', 'magic',
             'sndfile', 'gsm', 'FLAC',
             'vorbis', 'vorbisenc', 'vorbisfile', 'ogg',
             'mad', 'mp3lame',
-            'asound',
             'pulse', 'pulse-simple']:
         conf.CheckLib(lib)
 
 if 'target_gengetopt' in getdeps:
-    env.ThridParty(toolchain, 'gengetopt-2.22.6')
+    env.ThirdParty(build, "", 'gengetopt-2.22.6')
 
     env['GENGETOPT'] = env.File(
-        '#3rdparty/gengetopt-2.22.6/bin/gengetopt' + env['PROGSUFFIX'])
+        '#3rdparty/%s/gengetopt-2.22.6/bin/gengetopt' % build + env['PROGSUFFIX'])
 
 env = conf.Finish()
 
 if 'target_cpputest' in getdeps:
-    test_env.ThridParty(toolchain, 'cpputest-3.6')
+    test_env.ThirdParty(host, toolchain, 'cpputest-3.6')
 
 if 'target_posix' in env['ROC_TARGETS']:
     env.Append(CPPDEFINES=[('_POSIX_C_SOURCE', '200809')])
@@ -467,8 +486,8 @@ for t in env['ROC_TARGETS']:
 
 env.Append(LIBPATH=['#bin'])
 
-if target_platform in ['linux']:
-    env.AppendUnique(LIBS=['rt'])
+if platform in ['linux']:
+    env.AppendUnique(LIBS=['rt', 'dl'])
 
 if compiler in ['gcc', 'clang']:
     env.Append(CXXFLAGS=[
@@ -558,7 +577,7 @@ if compiler == 'clang':
 
 if compiler in ['gcc', 'clang']:
     if not GetOption('disable_sanitizers') \
-      and host == target \
+      and not crosscompile \
       and variant == 'debug' and (
         (compiler == 'gcc' and compiler_ver[:2] >= (4, 9)) or
         (compiler == 'clang' and compiler_ver[:2] >= (3, 6))):

scripts/3rdparty.py

@@ -8,6 +8,7 @@
 import ssl
 import tarfile
 import fileinput
+import subprocess
 
 printdir = os.path.abspath('.')
 
@@ -72,13 +73,35 @@ def freplace(path, pat, to):
 def touch(path):
     open(path, 'w').close()
 
-if len(sys.argv) != 4:
-    print("usage: 3rdparty.py workdir toolchain name")
+def getsysroot(toolchain):
+    if not toolchain:
+        return ""
+    try:
+        cmd = ['%s-gcc' % toolchain, '-print-sysroot']
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+        return proc.stdout.read().strip()
+    except:
+        print("can't execute '%s'" % ' '.join(cmd), file=sys.stderr)
+        exit(1)
+
+def getexports(workdir, deplist):
+    inc=[]
+    lib=[]
+    for dep in deplist:
+        inc += [os.path.join(workdir, dep, 'include')]
+        lib += [os.path.join(workdir, dep, 'lib')]
+    return 'CFLAGS="%s" LDFLAGS="%s"' % (
+        ' '.join(['-I%s' % path for path in inc]),
+        ' '.join(['-L%s' % path for path in lib]))
+
+if len(sys.argv) != 5:
+    print("usage: 3rdparty.py workdir toolchain package deplist", file=sys.stderr)
     exit(1)
 
 workdir = os.path.abspath(sys.argv[1])
 toolchain = sys.argv[2]
 fullname = sys.argv[3]
+deplist = sys.argv[4].split(':')
 
 logfile = os.path.join(workdir, fullname, 'build.log')
 
@@ -89,13 +112,13 @@ def touch(path):
 os.chdir(os.path.join(workdir, fullname, 'src'))
 
 if name == 'uv':
-    download('https://github.com/libuv/libuv/archive/v%s.tar.gz' % ver,
-             'libuv-%s.tar.gz' % ver)
-    extract('libuv-%s.tar.gz' % ver,
-            'libuv-%s' % ver)
-    os.chdir('libuv-%s' % ver)
+    download('http://dist.libuv.org/dist/v%s/libuv-v%s.tar.gz' % (ver, ver),
+             'libuv-v%s.tar.gz' % ver)
+    extract('libuv-v%s.tar.gz' % ver,
+            'libuv-v%s' % ver)
+    os.chdir('libuv-v%s' % ver)
     execute('./autogen.sh', logfile)
-    execute('./configure --enable-static --target=%s' % toolchain, logfile)
+    execute('./configure --host=%s --enable-static' % toolchain, logfile)
     execute('make -j', logfile)
     install('include', os.path.join(workdir, fullname, 'include'))
     install('.libs/libuv.a', os.path.join(workdir, fullname, 'lib'))
@@ -110,7 +133,8 @@ def touch(path):
     os.mkdir('build')
     os.chdir('build')
     execute('cmake .. ' + ' '.join([
-        '-DCMAKE_SYSTEM_NAME=%s' % toolchain,
+        '-DCMAKE_C_COMPILER=%s' % '-'.join([s for s in [toolchain, 'gcc'] if s]),
+        '-DCMAKE_FIND_ROOT_PATH=%s' % getsysroot(toolchain),
         '-DCMAKE_BUILD_TYPE=Release',
         '-DDEBUG:STRING=OFF', # disable debug logs
         ]), logfile)
@@ -119,15 +143,34 @@ def touch(path):
     install('src', os.path.join(workdir, fullname, 'include'),
             ignore=['*.c', '*.txt'])
     install('bin/Release/libopenfec.a', os.path.join(workdir, fullname, 'lib'))
+elif name == 'alsa':
+    download(
+      'ftp://ftp.alsa-project.org/pub/lib/alsa-lib-%s.tar.bz2' % ver,
+        'alsa-lib-%s.tar.bz2' % ver)
+    extract('alsa-lib-%s.tar.bz2' % ver,
+            'alsa-lib-%s' % ver)
+    os.chdir('alsa-lib-%s' % ver)
+    execute('./configure --host=%s %s' % (
+        toolchain, ' '.join([
+            '--enable-static',
+            '--disable-shared',
+            '--disable-python',
+        ])), logfile)
+    execute('make -j', logfile)
+    install('include/alsa',
+            os.path.join(workdir, fullname, 'include', 'alsa'), ignore=['alsa'])
+    install('src/.libs/libasound.a', os.path.join(workdir, fullname, 'lib'))
 elif name == 'sox':
     download(
       'http://vorboss.dl.sourceforge.net/project/sox/sox/%s/sox-%s.tar.gz' % (ver, ver),
       'sox-%s.tar.gz' % ver)
     extract('sox-%s.tar.gz' % ver,
             'sox-%s' % ver)
     os.chdir('sox-%s' % ver)
-    execute(' ./configure --enable-static --disable-shared --target=%s %s' % (
-        toolchain, ' '.join([
+    execute('%s LIBS="-lpthread -ldl -lrt" ./configure --host=%s %s' % (
+        getexports(workdir, deplist), toolchain, ' '.join([
+            '--enable-static',
+            '--disable-shared',
             '--disable-openmp',
             '--without-id3tag',
             '--without-png',
@@ -155,13 +198,13 @@ def touch(path):
     # disable memory leak detection which is too hard to use properly
     # disable warnings, since CppUTest uses -Werror and may fail to build on old GCC
     execute('CXXFLAGS="-w" '
-        './configure --enable-static --disable-memory-leak-detection --target=%s' % (
+        './configure --host=%s --enable-static --disable-memory-leak-detection' % (
             toolchain), logfile)
     execute('make -j', logfile)
     install('include', os.path.join(workdir, fullname, 'include'))
     install('lib/libCppUTest.a', os.path.join(workdir, fullname, 'lib'))
 else:
-    print("unknown 3rdparty '%s'" % fullname)
+    print("unknown 3rdparty '%s'" % fullname, file=sys.stderr)
     exit(1)
 
 touch(os.path.join(workdir, fullname, 'commit'))