fix network gen + ansible

.env.sample

@@ -0,0 +1,3 @@
+SSH_PORT=
+SSH_USER=
+SSH_PATH=

.gitignore

@@ -78,4 +78,7 @@ playground.py
 .env
 xrpld/
 xahau/
-rippled
+rippled
+rippled.batch.v0
+
+feature-batch-cluster

tests/integration/test_network.py

@@ -77,7 +77,7 @@ def _test_enable_amendment(cls):
 
 
 class TestINetGenRippled(BaseTestConfig):
-    def test_create_rippled_network(cls):
+    def _test_create_rippled_network(cls):
         create_network(
             "trace",
             None,
@@ -86,7 +86,33 @@ def test_create_rippled_network(cls):
             2,  # num peers
             21336,  # network id
             "https://github.com/Transia-RnD/rippled/tree/feature-batch",  # build server
-            "b3136a243443214a46ad0a24a352db9dea27281b",  # build version
+            "70e993185c7bf5e77dc03729d6dc4e22d8607571",  # build version
             True,
             3,
         )
+
+    def test_create_ansible(cls):
+        create_ansible(
+            "trace",
+            None,
+            "xrpl",  # protocol
+            6,  # num validators
+            2,  # num peers
+            21336,  # network id
+            "https://github.com/Transia-RnD/rippled/tree/feature-batch",  # build server
+            "70e993185c7bf5e77dc03729d6dc4e22d8607571",  # build version
+            True,
+            3,
+            [
+                "79.110.60.99",
+                "79.110.60.100",
+                "79.110.60.101",
+                "79.110.60.102",
+                "79.110.60.103",
+                "79.110.60.104",
+            ],
+            [
+                "79.110.60.105",
+                "79.110.60.106",
+            ],
+        )

xrpld_netgen/deploykit/ansible/main.yml

@@ -3,6 +3,10 @@
   remote_user: root
 
   tasks:
+  - name: Copy directory to the remote server
+    copy:
+      src: "{{ config_path }}"
+      dest: /opt/ripple/config/
   - name: set docker to use systemd cgroups driver
     copy:
       dest: "/etc/docker/daemon.json"
@@ -33,5 +37,6 @@
       image: "{{ docker_image_name }}"
       ports: "{{ docker_container_ports }}"
       volumes: "{{ docker_volumes }}"
+      env: "{{ docker_env_variables }}"
       state: started
       restart_policy: always

xrpld_netgen/deploykit/network.entrypoint

@@ -0,0 +1,27 @@
+#!/bin/bash
+# ./entrypoint.sh ledgerfile quorum
+# ./entrypoint.sh /genesis.json 5
+
+# Check if $1 is passed
+if [[ -n "$1" ]]; then
+    ledgerfile="--ledgerfile $1"
+else
+    ledgerfile=""
+fi
+
+# Check if $2 is passed
+if [[ -n "$2" ]]; then
+    quorum="--quorum=$2"
+else
+    quorum=""
+fi
+
+# Check if $3 is passed
+if [[ -n "$3" ]]; then
+    standalone="$3"
+else
+    standalone=""
+fi
+
+# Start rippled with or without ledgerfile and quorum
+exec /app/rippled $ledgerfile $quorum $standalone --conf /opt/ripple/config/rippled.cfg

xrpld_netgen/genesis.xrpl.json

@@ -18,6 +18,15 @@
           "Flags": 0,
           "LedgerEntryType": "Amendments",
           "index": "7DB0788C020F02780A673DC74757F23823FA3014C1866E72CC4CD8B226CD6EF4"
+        },
+        {
+          "BaseFee": "A",
+          "Flags": 0,
+          "LedgerEntryType": "FeeSettings",
+          "ReferenceFeeUnits": 10,
+          "ReserveBase": 1000000,
+          "ReserveIncrement": 200000,
+          "index": "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A651"
         }
       ],
       "account_hash": "5DF3A98772FB73E782B8740E87885C6BAD9BA486422E3626DEF968AD2CB2C514",

xrpld_netgen/main.py

@@ -145,6 +145,7 @@ def create_standalone_folder(
     print(f"✅ {bcolors.CYAN}Updating features")
 
     dockerfile: str = create_dockerfile(
+        False,
         binary,
         name,
         image,
@@ -334,6 +335,7 @@ def create_binary_folder(
     #     print(f"{bcolors.GREEN}feature: {bcolors.BLUE}{k}")
 
     dockerfile: str = create_dockerfile(
+        False,
         binary,
         name,
         image,

xrpld_netgen/network.py

@@ -128,9 +128,9 @@ def create_node_folders(
             peer,
             "huge",
             10000,
-            "/var/lib/rippled/db/nudb",
-            "/var/lib/rippled/db",
-            "/var/log/rippled/debug.log",
+            "/opt/ripple/lib/db/nudb",
+            "/opt/ripple/lib/db",
+            "/opt/ripple/log/debug.log",
             log_level,
             tokens[i - 1],
             [v for v in validators if v != validators[i - 1]],
@@ -175,6 +175,7 @@ def create_node_folders(
         os.chmod(f"{basedir}/{name}-cluster/{node_dir}/rippled.{name}", 0o755)
 
         dockerfile: str = create_dockerfile(
+            True,
             binary,
             name,
             image,
@@ -191,7 +192,7 @@ def create_node_folders(
             file.write(dockerfile)
 
         shutil.copyfile(
-            f"{basedir}/deploykit/{protocol}.entrypoint",
+            f"{basedir}/deploykit/network.entrypoint",
             f"{basedir}/{name}-cluster/{node_dir}/entrypoint",
         )
 
@@ -213,8 +214,9 @@ def create_node_folders(
                 f"{peer}:{peer}",
             ],
             "volumes": [
-                f"{pwd_str}/vnode{i}/log:/var/log/rippled",
-                f"{pwd_str}/vnode{i}/lib:/var/lib/rippled",
+                f"{pwd_str}/{name}-cluster/vnode{i}/config:/opt/ripple/config",
+                f"{pwd_str}/{name}-cluster/vnode{i}/log:/opt/ripple/log",
+                f"{pwd_str}/{name}-cluster/vnode{i}/lib:/opt/ripple/lib",
             ],
             "networks": [f"{name}-network"],
         }
@@ -236,9 +238,9 @@ def create_node_folders(
             peer,
             "huge",
             None,
-            "/var/lib/rippled/db/nudb",
-            "/var/lib/rippled/db",
-            "/var/log/rippled/debug.log",
+            "/opt/ripple/lib/db/nudb",
+            "/opt/ripple/lib/db",
+            "/opt/ripple/log/debug.log",
             log_level,
             None,
             validators,
@@ -282,6 +284,7 @@ def create_node_folders(
         os.chmod(f"{basedir}/{name}-cluster/{node_dir}/rippled.{name}", 0o755)
 
         dockerfile: str = create_dockerfile(
+            True,
             binary,
             name,
             image,
@@ -298,7 +301,7 @@ def create_node_folders(
             file.write(dockerfile)
 
         shutil.copyfile(
-            f"{basedir}/deploykit/{protocol}.entrypoint",
+            f"{basedir}/deploykit/network.entrypoint",
             f"{basedir}/{name}-cluster/{node_dir}/entrypoint",
         )
 
@@ -320,8 +323,9 @@ def create_node_folders(
                 f"{peer}:{peer}",
             ],
             "volumes": [
-                f"{pwd_str}/pnode{i}/log:/var/log/rippled",
-                f"{pwd_str}/pnode{i}/lib:/var/lib/rippled",
+                f"{pwd_str}/{name}-cluster/pnode{i}/config:/opt/ripple/config",
+                f"{pwd_str}/{name}-cluster/pnode{i}/log:/opt/ripple/log",
+                f"{pwd_str}/{name}-cluster/pnode{i}/lib:/opt/ripple/lib",
             ],
             "networks": [f"{name}-network"],
         }
@@ -576,6 +580,29 @@ def create_ansible(
         download_binary(url, f"{basedir}/{name}-cluster/rippled.{build_version}")
         image: str = "ubuntu:jammy"
 
+    if protocol == "xrpl":
+        if build_server.startswith("https://github.com/"):
+            name: str = build_server.split(
+                "https://github.com/Transia-RnD/rippled/tree/"
+            )[-1]
+            os.makedirs(f"{basedir}/{name}-cluster", exist_ok=True)
+            owner = "Transia-RnD"
+            repo = "rippled"
+            copy_file(f"./rippled", f"{basedir}/{name}-cluster/rippled.{name}")
+            content: str = download_file_at_commit_or_tag(
+                owner, repo, build_version, "src/libxrpl/protocol/Feature.cpp"
+            )
+            image: str = "ubuntu:jammy"
+        else:
+            name: str = build_version
+            os.makedirs(f"{basedir}/{name}-cluster", exist_ok=True)
+            owner = "XRPLF"
+            repo = "rippled"
+            content: str = download_file_at_commit_or_tag(
+                owner, repo, build_version, "src/libxrpl/protocol/Feature.cpp"
+            )
+            image: str = f"{build_server}/{build_version}"
+
     client = PublisherClient()
     client.create_keys()
     keys = client.get_keys()
@@ -649,6 +676,7 @@ def create_ansible(
 
     os.makedirs(f"{basedir}/{name}-cluster/ansible", exist_ok=True)
     os.makedirs(f"{basedir}/{name}-cluster/ansible/host_vars", exist_ok=True)
+
     shutil.copytree(
         f"{basedir}/deploykit/ansible",
         f"{basedir}/{name}-cluster/ansible",
@@ -663,57 +691,68 @@ def create_ansible(
             c_name: str = v["container_name"]
             ports: List[str] = v["ports"]
             vars = DockerVars(
+                f"{basedir}/{name}-cluster/{c_name}/config/",
                 ssh_port,
+                [
+                    f'RPC_PUBLIC: {ports[0].split(':')[0]}',
+                    f'RPC_ADMIN: {ports[1].split(':')[0]}',
+                    f'WS_PUBLIC: {ports[2].split(':')[0]}',
+                    f'WS_ADMIN: {ports[3].split(':')[0]}',
+                    f'PEER: {ports[4].split(':')[0]}',
+                ],
                 int(ports[2].split(":")[-1]),
                 int(ports[4].split(":")[-1]),
-                f"transia/cluster-{c_name}:{image_name}",
+                f"transia/cluster:{image_name}",
                 c_name,
                 ports,
                 [
-                    "/var/log/rippled:/var/log/rippled",
-                    "/var/lib/rippled:/var/lib/rippled",
+                    "/opt/ripple/config:/opt/ripple/config",
+                    "/opt/ripple/log:/opt/ripple/log",
+                    "/opt/ripple/lib:/opt/ripple/lib",
                 ],
-                ["/var/log/rippled", "/var/lib/rippled"],
+                ["/opt/ripple/log", "/opt/ripple/lib"],
             )
             create_ansible_vars_file(
                 f"{basedir}/{name}-cluster/ansible/host_vars", vips[index - 1], vars
             )
-            run_command(
-                f"{basedir}/{name}-cluster/{c_name}",
-                f"docker build -f Dockerfile --platform linux/x86_64 --tag transia/cluster-{c_name}:{image_name} .",  # noqa: E501
-            )
-            run_command(
-                f"{basedir}/{name}-cluster/{c_name}",
-                f"docker push transia/cluster-{c_name}:{image_name}",
-            )
+
+        run_command(
+            f"{basedir}/{name}-cluster/vnode1",
+            f"docker build -f Dockerfile --platform linux/x86_64 --tag transia/cluster:{image_name} .",  # noqa: E501
+        )
+        run_command(
+            f"{basedir}/{name}-cluster/vnode1",
+            f"docker push transia/cluster:{image_name}",
+        )
         if k[:5] == "pnode":
             index: int = int(k[5:])
             c_name: str = v["container_name"]
             ports: List[str] = v["ports"]
             vars = DockerVars(
+                f"{basedir}/{name}-cluster/{c_name}/config/",
                 ssh_port,
+                [
+                    f'RPC_PUBLIC: {ports[0].split(':')[0]}',
+                    f'RPC_ADMIN: {ports[1].split(':')[0]}',
+                    f'WS_PUBLIC: {ports[2].split(':')[0]}',
+                    f'WS_ADMIN: {ports[3].split(':')[0]}',
+                    f'PEER: {ports[4].split(':')[0]}',
+                ],
                 int(ports[2].split(":")[-1]),
                 int(ports[4].split(":")[-1]),
-                f"transia/cluster-{c_name}:{image_name}",
+                f"transia/cluster:{image_name}",
                 c_name,
                 ports,
                 [
-                    "/var/log/rippled:/var/log/rippled",
-                    "/var/lib/rippled:/var/lib/rippled",
+                    "/opt/ripple/config:/opt/ripple/config",
+                    "/opt/ripple/log:/opt/ripple/log",
+                    "/opt/ripple/lib:/opt/ripple/lib",
                 ],
-                ["/var/log/rippled", "/var/lib/rippled"],
+                [ "/opt/ripple/log", "/opt/ripple/lib"],
             )
             create_ansible_vars_file(
                 f"{basedir}/{name}-cluster/ansible/host_vars", pips[index - 1], vars
             )
-            run_command(
-                f"{basedir}/{name}-cluster/{c_name}",
-                f"docker build -f Dockerfile --platform linux/x86_64 --tag transia/cluster-{c_name}:{image_name} .",  # noqa: E501
-            )
-            run_command(
-                f"{basedir}/{name}-cluster/{c_name}",
-                f"docker push transia/cluster-{c_name}:{image_name}",
-            )
     hosts_content: str = """
 # this is a basic file putting different hosts into categories
 # used by ansible to determine which actions to run on which hosts