Merge pull request ydb-platform#4119 from ydb-platform/mergelibs-2404…

…25-1630

Library import 240425-1630

build/conf/go.conf

@@ -1004,18 +1004,20 @@ macro GO_MOCKGEN_REFLECT() {
     PEERDIR(vendor/go.uber.org/mock/mockgen/model)
     PEERDIR(${MOCKGEN_FROM})
 
-    RUN_PROGRAM(vendor/go.uber.org/mock/mockgen -prog_only $MOCKGEN_MODULE $MOCKGEN_TYPES STDOUT main.go)
+    RUN_PROGRAM(vendor/go.uber.org/mock/mockgen -prog_only $MOCKGEN_MODULE $MOCKGEN_TYPES STDOUT main.go CWD $ARCADIA_BUILD_ROOT)
 }
 
-# tag:internal
-macro _TOOL(TOOL) {
-    .CMD=${tool:TOOL}
+# tag:go-specific
+_GO_EXE_SUFFIX=
+when ($OS_WINDOWS == "yes") {
+    _GO_EXE_SUFFIX=.exe
 }
 
 # tag:go-specific
 macro GO_MOCKGEN_MOCKS() {
     PEERDIR(${GOSTD}/reflect)
     PEERDIR(vendor/go.uber.org/mock/gomock)
 
-    .CMD=${tool:"vendor/go.uber.org/mock/mockgen"} -package mocks -exec_only $_TOOL($MODDIR/gen) ${GO_ARCADIA_PROJECT_PREFIX}${MOCKGEN_FROM} $MOCKGEN_TYPES ${output;stdout:"main.go"}
+    # Unfortunately ${rootrel;tool:TOOL} doesn't work currently, so we use this ugly workaround $MODDIR/gen/gen$_GO_EXE_SUFFIX
+    RUN_PROGRAM(vendor/go.uber.org/mock/mockgen -package mocks -exec_only $MODDIR/gen/gen$_GO_EXE_SUFFIX ${GO_ARCADIA_PROJECT_PREFIX}${MOCKGEN_FROM} $MOCKGEN_TYPES STDOUT main.go TOOL $MODDIR/gen CWD $ARCADIA_BUILD_ROOT)
 }

build/conf/license.conf

@@ -434,7 +434,13 @@ macro LICENSE(Flags...) {
     # NOTICE: final value of MODVER might not be set yet if VERSION macro is called after LICENSE. Var expansion is
     #         escaped here to prevent eager value substitution by SET but allow delayed value substitution when
     #         adding linking SBOM data command to graph.
-    SET_APPEND(_SBOM_INFO_GLOBAL "path=${MODDIR};ver=\${join=.:MODVER};lang=${MODULE_LANG}")
+    when ($_NEED_SBOM_INFO == "yes") {
+        _SBOM_INFO_LOCAL="path=${MODDIR};ver=${join=.:MODVER};lang=${MODULE_LANG}"
+    } 
+    otherwise {
+        _SBOM_INFO_LOCAL=
+    }
+    SET_APPEND(_SBOM_INFO_GLOBAL $_SBOM_INFO_LOCAL)
 }
 
 ### @usage LICENSE_RESTRICTION(ALLOW_ONLY|DENY LicenseProperty...)

build/conf/ts/ts_package.conf

@@ -5,6 +5,10 @@ TS_PACK=$TOUCH_UNIT \
     && $_TS_FILES_COPY_CMD \
     ${kv;hide:"p TS_PKG"} ${kv;hide:"pc magenta"}
 
+macro _TS_PACKAGE_EPILOGUE() {
+  _TS_PACKAGE_CHECK_FILES()
+}
+
 ### @usage: TS_PACKAGE()
 ###
 ### The TypeScript/JavaScript library module, that does not need any compilation,
@@ -23,6 +27,7 @@ TS_PACK=$TOUCH_UNIT \
 multimodule TS_PACKAGE {
     module BUILD: _TS_BASE_UNIT {
         .CMD=TS_PACK
+        .EPILOGUE=_TS_PACKAGE_EPILOGUE
         .ALLOWED=TS_FILES
         .ALIASES=FILES=TS_FILES SRCS=TS_FILES
         .PEERDIRSELF=TS_PREPARE_DEPS

build/export_generators/hardcoded-cmake/common_cmake_lists.jinja

@@ -2,9 +2,9 @@
 {% include '[generator]/disclaimer.jinja' -%}
 {% set platforms_count = platform_names|length -%}
 {% if platforms_count > 0 -%}
-{% for i in range(platforms_count) %}
-{% if i != 0 %}else{% endif %}if ({{ platforms[platform_names[i]] }})
-  include({{ platform_cmakelists[i] }})
-{%- endfor %}
+{%     for i in range(platforms_count) %}
+{%         if i != 0 %}else{% endif %}if ({{ platforms[platform_names[i]] }})
+  include(CMakeLists.{{ platform_names[i] }}.txt)
+{%-    endfor %}
 endif()
 {% endif -%}

build/export_generators/hardcoded-cmake/dir_cmake_lists.jinja

@@ -1,4 +1,4 @@
-{% include prologue ignore missing %}
+{% include 'prologue.cmake' ignore missing %}
 {% include '[generator]/disclaimer.jinja' -%}
 
 {% for package in packages -%}
@@ -8,9 +8,11 @@ find_package({{ package[0] }} REQUIRED
         {%- endfor %}
     {%- endif %})
 {% endfor -%}
-{% for incl in includes -%}
+{% if includes is defined -%}
+{%     for incl in includes -%}
 include({{ incl }})
-{% endfor -%}
+{%     endfor -%}
+{% endif -%}
 {% for subdir in subdirs -%}
 add_subdirectory({{ subdir }})
 {% endfor -%}
@@ -21,15 +23,15 @@ add_subdirectory({{ subdir }})
 
 {% endfor -%}
 {% for target in targets %}
-{%- set macro = target.macro -%}
-    {%- set name = target.name -%}
-    {%- set is_interface = target.is_interface -%}
-    {%- set macro_args = target.macro_args -%}
-    {%- set properties = target.properties -%}
-    {%- set attributes = target.attributes -%}
-    {%- set target_dir_macros = target.target_dir_macros -%}
-    {%- set target_macros = target.target_macros -%}
-    {%- set dependencies = target.dependencies -%}
-    {%- include '[generator]/target_cmake_lists.jinja' %}{{"\n"}}
+{%-     set macro = target.macro -%}
+{%-     set name = target.name -%}
+{%-     set is_interface = target.is_interface -%}
+{%-     set macro_args = target.macro_args -%}
+{%-     set properties = target.properties -%}
+{%-     set attributes = target.attributes -%}
+{%-     set target_dir_macros = target.target_dir_macros -%}
+{%-     set target_macros = target.target_macros -%}
+{%-     set dependencies = target.dependencies -%}
+{%-     include '[generator]/target_cmake_lists.jinja' %}{{"\n"}}
 {%- endfor %}
-{% include epilogue ignore missing %}
+{% include 'epilogue.cmake' ignore missing %}

build/export_generators/hardcoded-cmake/generator.toml

@@ -49,15 +49,12 @@ template={ path="dir_cmake_lists.jinja", dest="CMakeLists{PLATFORM}.txt" }
 [attrs.root]
 project_name="str"
 project_language_list="list"
-platform_cmakelists="list"
 platform_vars="list"
 includes="sorted_set"
 use_conan="bool"
 vanilla_protobuf="flag"
 
 [attrs.dir]
-prologue="str"
-epilogue="str"
 
 [attrs.target]
 is_interface="bool"

build/export_generators/hardcoded-cmake/root_cmake_lists.jinja

@@ -23,8 +23,12 @@ cmake_policy(SET CMP0104 OLD)
 
 include(cmake/global_flags.cmake)
 include(cmake/global_vars.cmake)
-{% for path in includes %}include({{path}})
-{%endfor%}{% if use_conan %}
+{% if includes is defined -%}
+{%     for incl in includes -%}
+include({{ incl }})
+{%     endfor -%}
+{% endif -%}
+{% if use_conan %}
 if (CMAKE_CROSSCOMPILING)
   include(${PROJECT_BINARY_DIR}/conan_paths.cmake)
 else()
@@ -49,7 +53,7 @@ set(vanilla_protobuf "yes")
 {%  if platforms_count > 0 -%}
 {%      for i in range(platforms_count) -%}
 {%          if i != 0 %}else{% endif %}if ({{ platforms[platform_names[i]] }})
-  include({{ platform_cmakelists[i] }})
+  include(CMakeLists.{{ platform_names[i] }}.txt)
 {%      endfor -%}
 endif()
 {%  endif %}

build/export_generators/hardcoded-cmake/target_cmake_lists.jinja

@@ -1,9 +1,9 @@
 {{ macro }}({{ name }}
 {%- if is_interface %} INTERFACE{% endif %}
 {%- if macro_args|length > 0 %}
-{%- for arg in macro_args %}
+{%-     for arg in macro_args %}
   {{ arg }}
-{%- endfor -%}
+{%-     endfor -%}
 {% endif %})
 
 {% for property in properties -%}
@@ -13,39 +13,39 @@ set_property(TARGET {{ name }} PROPERTY
 )
 {% endfor -%}
 {% for attribute in attributes %}
-{%- if attribute[1].iface is defined %}
+{%-    if attribute[1].iface is defined %}
 {{ attribute[0] }}({{ name }} INTERFACE
-{% for val in attribute[1].iface %}  {{ val }}
-{% endfor %})
-{%- endif %}
-{%- if attribute[1].pub is defined %}
+{%         for val in attribute[1].iface %}  {{ val }}
+{%         endfor %})
+{%-    endif %}
+{%-    if attribute[1].pub is defined %}
 {{ attribute[0] }}({{ name }} PUBLIC
-{% for val in attribute[1].pub %}  {{ val }}
-{% endfor %})
-{%- endif %}
-{%- if attribute[1].priv is defined %}
+{%         for val in attribute[1].pub %}  {{ val }}
+{%         endfor %})
+{%-    endif %}
+{%-    if attribute[1].priv is defined %}
 {{ attribute[0] }}({{ name }} PRIVATE
-{% for val in attribute[1].priv %}  {{ val }}
-{% endfor %})
-{%- endif %}
+{%         for val in attribute[1].priv %}  {{ val }}
+{%         endfor %})
+{%-    endif %}
 {% endfor %}
 {% for dir_macro in target_dir_macros %}
 {{ dir_macro[0] }}(
-{% for value in dir_macro[1] %}  {{ value }}
-{% endfor %})
+{%     for value in dir_macro[1] %}  {{ value }}
+{%     endfor %})
 {% endfor %}
 {% for target_macro in target_macros %}
 {{ target_macro[0] }}({{ name }}
-{%- if target_macro[1]|length > 0 %}
-{% for value in target_macro[1] -%}
+{%-     if target_macro[1]|length > 0 %}
+{%          for value in target_macro[1] -%}
   {{ value }}
-{% endfor %}
-{%- endif %})
+{%          endfor %}
+{%-     endif %})
 {% endfor %}
 {% if dependencies|length > 0 -%}
 if(NOT CMAKE_CROSSCOMPILING)
   add_dependencies({{ name }}
-{% for dep in dependencies %}    {{ dep }}
-{% endfor %})
+{%     for dep in dependencies %}    {{ dep }}
+{%     endfor %})
 endif()
 {%- endif -%}

build/external_resources/yexport/resources.json

@@ -1,13 +1,13 @@
 {
     "by_platform": {
         "darwin": {
-            "uri": "sbr:6090203401"
+            "uri": "sbr:6229343240"
         },
         "darwin-arm64": {
-            "uri": "sbr:6090203088"
+            "uri": "sbr:6229342354"
         },
         "linux": {
-            "uri": "sbr:6090202665"
+            "uri": "sbr:6229341588"
         }
     }
 }

build/mapping.conf.json

@@ -252,20 +252,26 @@
         "6028550009": "https://devtools-registry.s3.yandex.net/6028550009",
         "6071059003": "https://devtools-registry.s3.yandex.net/6071059003",
         "6090207712": "https://devtools-registry.s3.yandex.net/6090207712",
+        "6222517102": "https://devtools-registry.s3.yandex.net/6222517102",
+        "6229387247": "https://devtools-registry.s3.yandex.net/6229387247",
         "5811823398": "https://devtools-registry.s3.yandex.net/5811823398",
         "5840611310": "https://devtools-registry.s3.yandex.net/5840611310",
         "5860185593": "https://devtools-registry.s3.yandex.net/5860185593",
         "6024274139": "https://devtools-registry.s3.yandex.net/6024274139",
         "6028549510": "https://devtools-registry.s3.yandex.net/6028549510",
         "6071058230": "https://devtools-registry.s3.yandex.net/6071058230",
         "6090207496": "https://devtools-registry.s3.yandex.net/6090207496",
+        "6222515843": "https://devtools-registry.s3.yandex.net/6222515843",
+        "6229385673": "https://devtools-registry.s3.yandex.net/6229385673",
         "5811822876": "https://devtools-registry.s3.yandex.net/5811822876",
         "5840610640": "https://devtools-registry.s3.yandex.net/5840610640",
         "5860184285": "https://devtools-registry.s3.yandex.net/5860184285",
         "6024272217": "https://devtools-registry.s3.yandex.net/6024272217",
         "6028548957": "https://devtools-registry.s3.yandex.net/6028548957",
         "6071057694": "https://devtools-registry.s3.yandex.net/6071057694",
         "6090206998": "https://devtools-registry.s3.yandex.net/6090206998",
+        "6222514531": "https://devtools-registry.s3.yandex.net/6222514531",
+        "6229383678": "https://devtools-registry.s3.yandex.net/6229383678",
         "5766172292": "https://devtools-registry.s3.yandex.net/5766172292",
         "5805431504": "https://devtools-registry.s3.yandex.net/5805431504",
         "5829027626": "https://devtools-registry.s3.yandex.net/5829027626",
@@ -712,20 +718,26 @@
         "6028550009": "devtools/yexport/bin/yexport for darwin",
         "6071059003": "devtools/yexport/bin/yexport for darwin",
         "6090207712": "devtools/yexport/bin/yexport for darwin",
+        "6222517102": "devtools/yexport/bin/yexport for darwin",
+        "6229387247": "devtools/yexport/bin/yexport for darwin",
         "5811823398": "devtools/yexport/bin/yexport for darwin-arm64",
         "5840611310": "devtools/yexport/bin/yexport for darwin-arm64",
         "5860185593": "devtools/yexport/bin/yexport for darwin-arm64",
         "6024274139": "devtools/yexport/bin/yexport for darwin-arm64",
         "6028549510": "devtools/yexport/bin/yexport for darwin-arm64",
         "6071058230": "devtools/yexport/bin/yexport for darwin-arm64",
         "6090207496": "devtools/yexport/bin/yexport for darwin-arm64",
+        "6222515843": "devtools/yexport/bin/yexport for darwin-arm64",
+        "6229385673": "devtools/yexport/bin/yexport for darwin-arm64",
         "5811822876": "devtools/yexport/bin/yexport for linux",
         "5840610640": "devtools/yexport/bin/yexport for linux",
         "5860184285": "devtools/yexport/bin/yexport for linux",
         "6024272217": "devtools/yexport/bin/yexport for linux",
         "6028548957": "devtools/yexport/bin/yexport for linux",
         "6071057694": "devtools/yexport/bin/yexport for linux",
         "6090206998": "devtools/yexport/bin/yexport for linux",
+        "6222514531": "devtools/yexport/bin/yexport for linux",
+        "6229383678": "devtools/yexport/bin/yexport for linux",
         "5766172292": "devtools/ymake/bin/ymake for darwin",
         "5805431504": "devtools/ymake/bin/ymake for darwin",
         "5829027626": "devtools/ymake/bin/ymake for darwin",

build/plugins/lib/nots/package_manager/base/node_modules_bundler.py

@@ -1,9 +1,5 @@
 import os
-import sys
-import subprocess
-import tarfile
-
-from io import BytesIO
+import tempfile
 
 from .utils import build_nm_path
 
@@ -25,24 +21,30 @@ def bundle_node_modules(build_root, peers, node_modules_path, bundle_path):
     :param bundle_path: tarball path
     :type bundle_path: str
     """
-    with tarfile.open(bundle_path, "w") as tf:
-        tf.add(node_modules_path, arcname=".")
+    import library.python.archive as archive
 
-        # Peers' node_modules.
-        added_peers = []
-        for p in peers:
-            peer_nm_path = build_nm_path(os.path.join(build_root, p))
-            peer_bundled_nm_path = build_nm_path(os.path.join(PEERS_DIR, p))
-            if not os.path.isdir(peer_nm_path):
-                continue
-            tf.add(peer_nm_path, arcname=peer_bundled_nm_path)
-            added_peers.append(p)
+    paths_to_pack = []
+    paths_to_pack.append((node_modules_path, "."))
+
+    # Peers' node_modules.
+    added_peers = []
+    for p in peers:
+        peer_nm_path = build_nm_path(os.path.join(build_root, p))
+        peer_bundled_nm_path = build_nm_path(os.path.join(PEERS_DIR, p))
+        if not os.path.isdir(peer_nm_path):
+            continue
+        paths_to_pack.append((peer_nm_path, peer_bundled_nm_path))
+        added_peers.append(p)
 
-        # Peers index.
-        peers_index = "\n".join(added_peers)
-        ti = tarfile.TarInfo(name=os.path.join(PEERS_DIR, PEERS_INDEX))
-        ti.size = len(peers_index)
-        tf.addfile(ti, BytesIO(peers_index.encode()))
+    # Peers index.
+    with tempfile.TemporaryDirectory() as temp_dir:
+        peers_index_tmppath = os.path.join(temp_dir, PEERS_INDEX)
+        peers_index_relpath = os.path.join(PEERS_DIR, PEERS_INDEX)
+        with open(peers_index_tmppath, "w") as peers_index:
+            peers_index.write("\n".join(added_peers))
+        paths_to_pack.append((peers_index_tmppath, peers_index_relpath))
+
+        archive.tar(paths_to_pack, bundle_path, compression_filter=None, compression_level=None, fixed_mtime=0)
 
 
 def extract_node_modules(build_root, node_modules_path, bundle_path):
@@ -55,16 +57,10 @@ def extract_node_modules(build_root, node_modules_path, bundle_path):
     :param bundle_path: tarball path
     :type bundle_path: str
     """
-    os.makedirs(node_modules_path, exist_ok=True)
-    tar_unpack_cmd = ["tar", "xf", bundle_path, "-C", node_modules_path]
-    p = subprocess.run(tar_unpack_cmd, capture_output=True, text=True)
-    if p.returncode != 0:
-        if p.stdout:
-            sys.stderr.write(f"stdout:\n{p.stdout}\n")
-        if p.stderr:
-            sys.stderr.write(f"stderr:\n{p.stderr}\n")
+    import library.python.archive as archive
 
-        return False
+    os.makedirs(node_modules_path, exist_ok=True)
+    archive.extract_tar(bundle_path, node_modules_path, fail_on_duplicates=False)
 
     with open(os.path.join(node_modules_path, PEERS_DIR, PEERS_INDEX)) as peers_file:
         peers = peers_file.read().split("\n")