Merge pull request #10 from xsuite/release/v0.1.1

Release 0.1.1

.gitignore

@@ -1,3 +1,5 @@
+xboinc/user_data.json
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

pyproject.toml

@@ -5,7 +5,7 @@
 
 [tool.poetry]
 name = "xboinc"
-version = "0.1.0"
+version = "0.1.1"
 description = "Xsuite BOINC interface"
 homepage = "https://github.com/xsuite/xboinc"
 repository = "https://github.com/xsuite/xboinc"

tests/test_version.py

@@ -1,5 +1,5 @@
 from xboinc import __version__
 
 def test_version():
-    assert __version__ == '0.1.0'
+    assert __version__ == '0.1.1'
 

version.sh

@@ -19,6 +19,22 @@ fi
 bump=$1
 force=$2
 
+# Check our working directory is clean
+git diff --quiet
+if [ $? -ne 0 ]
+then
+    echo "Some changes are not staged."
+    echo "Stage and commit before bumping the version."
+    exit 1
+fi
+git diff --staged --quiet
+if [ $? -ne 0 ]
+then
+    echo "Some changes are staged."
+    echo "Commit before bumping the version."
+    exit 1
+fi
+
 # Check we are not on main
 branch=$(git status | head -1 | awk '{print $NF}')
 if [[ "$branch" == "main" ]]
@@ -35,7 +51,7 @@ fi
 expected_ver=$(poetry version $bump --dry-run | awk '{print $6;}')
 if [[ "$force" != "--force" ]]
 then
-    if [[ "$branch" != "release$expected_ver" ]]
+    if [[ "$branch" != "release/v$expected_ver" ]]
     then
         echo "Error: you are bumping to $expected_ver but this branch is $branch."
         echo "If this is intentional, use version.sh 'value' --force."
@@ -73,6 +89,7 @@ esac
 current_ver=(${current_ver//./ })
 current_ver=${current_ver[0]}.${current_ver[1]}
 minor_ver=(${expected_ver//./ })
+# TODO: exec_ver should come from python app_version_int
 exec_ver=$(( 1000*${minor_ver[0]} + ${minor_ver[1]} ))
 minor_ver=${minor_ver[0]}.${minor_ver[1]}
 if [[ "$minor_ver" != "$current_ver" ]]

xboinc/__init__.py

@@ -12,7 +12,7 @@
 from .register import register
 from .submit import SubmitJobs
 
-from .simulation_io import SimState, SimConfig
+from .simulation_io import SimState, SimConfig, app_version, app_version_int
 from .executable import generate_executable_source, generate_executable
 
 

xboinc/executable/generate_executable_source.py

@@ -4,7 +4,7 @@
 # ######################################### #
 
 import xobjects as xo
-from ..simulation_io import SimState, SimConfig, SimVersion, get_default_tracker
+from ..simulation_io import SimState, SimConfig, SimVersion, app_version, get_default_tracker
 from ..general import _pkg_root, __version__
 
 from pathlib import Path
@@ -78,8 +78,7 @@ def generate_executable(keep_source=False):
     else:
         raise RuntimeError("Neither clang or gcc are found. Install a C compiler.")
 
-    tag = '_'
-    tag += '.'.join(__version__.split('.')[:2])
+    tag = f'_{app_version}'
     cmd = subprocess.run(['uname', '-ms'], stdout=subprocess.PIPE)
     if cmd.returncode == 0:
         tag += '-' + cmd.stdout.decode('UTF-8').strip().lower().replace(' ','-')

xboinc/executable/main.c

@@ -64,8 +64,6 @@ int main(){
     // Get sim config
     SimConfig sim_config = (SimConfig) sim_buffer;
     int64_t input_version = SimConfig_get_sim_state_version_xboinc_version(sim_config);
-    // Compatible if major and minor versions match (no new executables are made at patches)
-    input_version = input_version/1000;
     if (input_version != xboinc_exec_version){
         printf("Error: Xboinc version of executable and input file do not match!\n");
         return -1;  // error

xboinc/general.py

@@ -12,7 +12,7 @@
 # Do not change
 # ==========================================================================================
 
-__version__ = '0.1.0'
+__version__ = '0.1.1'
 
 # These are the xsuite modules that are used by boinc and the versions they are tied to.
 # This will be automatically updated from the active environment when making a minor release.

xboinc/register.py

@@ -7,9 +7,10 @@
 from pathlib import Path
 
 from .general import _pkg_root
+from .user import update_user_data
 from .server import server_account
 from .server.paths import dropdir
-from .server.eos import cp_to_eos
+from .server.eos import eos_exists, eos_rm, cp_to_eos
 from .server.afs import afs_add_acl
 
 
@@ -28,7 +29,7 @@ def _create_json(user, folder):
     data = {'user': user, 'folder': folder.as_posix(), 'domain': domain}
     with user_file.open('w') as f:
         json.dump(data, f)
-    return user_file, domain
+    return user_file, data
 
 
 def _set_rights(folder, domain):
@@ -42,15 +43,19 @@ def register(user, folder):
     folder = Path(folder).expanduser().resolve()
     if not folder.is_dir():
         raise ValueError(f"Folder {folder} not found or not a folder (or no permissions)!")
-    user_file, domain = _create_json(user, folder)
+    user_file, data = _create_json(user, folder)
     try:
-        _set_rights(folder, domain)
+        _set_rights(folder, data['domain'])
     except Exception as e:
         user_file.unlink()
         raise Exception(e)
+    if eos_exists(dropdir / user_file.name):
+        eos_rm(dropdir / user_file.name)
+        print("Replaced existing registration file on server dropdir.")
     if cp_to_eos(user_file, dropdir, maximum_trials=1):  # returncode 1 means error
         user_file.unlink()
         raise Exception(f"Failed to copy register file to server dropdir.\n"
                        + "Do you have permissions?\n"
                        + "If not, add yourself to the CERN xboinc-submitters egroup.")
-    user_file.rename(user_data_file)
+    user_file.unlink()
+    update_user_data(user, data)

xboinc/server/afs.py

@@ -72,3 +72,75 @@ def afs_print_acl(folder, is_server=False):
         log_error(f"Failed to remove ACL on {folder} for user {user}.\n{stderr}", e, is_server=is_server)
         return 1
 
+
+def afs_rm(path, is_server=False):
+    try:
+        path = Path(path).expanduser().resolve()
+        cmd = subprocess.run(['rm', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if cmd.returncode == 0:
+            return 0
+        else:
+            stderr = cmd.stderr.decode('UTF-8').strip().split('\n')
+            log_error(f"Failed afs_rm for {path}!\n{stderr}", is_server=is_server)
+            return 1
+    except Exception as e:
+        log_error(f"Failed afs_rm for {path}!\n", e, is_server=is_server)
+        return 1
+
+
+# Always use this file by file, not for a list of files
+def cp_from_afs(source, target, maximum_trials=10, wait=2.7, is_server=False):
+    try:
+        source = Path(source).expanduser().resolve()
+        target = Path(target).expanduser().resolve()
+        for i in range(maximum_trials):
+            cmd = subprocess.run(['cp', source.as_posix(), target.as_posix()])
+            if cmd.returncode == 0 and target.exists() and target.stat().st_size!=0:
+                return 0
+            if i != maximum_trials-1:
+                log_debug(f"Failed to copy {str(source)} to {str(target)}!\nRetrying ({i})..", is_server=is_server)
+                sleep(abs(wait+random.normalvariate(0, 0.1*wait)))
+        log_error(f"Failed to copy {str(source)} to {str(target)}!", is_server=is_server)
+        if not target.exists() or target.stat().st_size==0:
+            log_error(f"Command succeeds but destination file is not created!", is_server=is_server)
+        else:
+            stderr = cmd.stderr.decode('UTF-8').strip().split('\n')
+            log_error(f"Command failed: {stderr}", is_server=is_server)
+        log_error("\nGiving up.", is_server=is_server)
+        return 1
+    except Exception as e:
+        log_error(f"Failed to copy {str(source)} to {str(target)}!\n", e, is_server=is_server)
+        return 1
+
+def mv_from_afs(source, target, maximum_trials=10, wait=2.7, is_server=False):
+    if not cp_from_afs(source, target, maximum_trials, wait, is_server=is_server):   # returncode 0 means success
+        afs_rm(source, is_server=is_server)
+
+
+# Always use this file by file, not for a list of files
+def cp_to_afs(source, target, maximum_trials=10, wait=2.7, is_server=False):
+    try:
+        source = Path(source).expanduser().resolve()
+        target = Path(target).expanduser().resolve()
+        for i in range(maximum_trials):
+            cmd = subprocess.run(['cp', source.as_posix(), target.as_posix()])
+            if cmd.returncode == 0 and target.exists() and target.stat().st_size!=0:
+                return 0
+            if i != maximum_trials-1:
+                log_debug(f"Failed to copy {str(source)} to {str(target)}!\nRetrying ({i})..", is_server=is_server)
+                sleep(abs(wait+random.normalvariate(0, 0.1*wait)))
+        log_error(f"Failed to copy {str(source)} to {str(target)}!", is_server=is_server)
+        if not target.exists() or target.stat().st_size==0:
+            log_error(f"Command succeeds but destination file is not created!", is_server=is_server)
+        else:
+            stderr = cmd.stderr.decode('UTF-8').strip().split('\n')
+            log_error(f"Command failed: {stderr}", is_server=is_server)
+        log_error("\nGiving up.", is_server=is_server)
+        return 1
+    except Exception as e:
+        log_error(f"Failed to copy {str(source)} to {str(target)}!\n", e, is_server=is_server)
+        return 1
+
+def mv_to_afs(source, target, maximum_trials=10, wait=2.7, is_server=False):
+    if not cp_to_afs(source, target, maximum_trials, wait, is_server=is_server):   # returncode 0 means success
+        source.unlink()

xboinc/server/eos.py

@@ -66,7 +66,7 @@ def eos_exists(path, is_server=False):
 def eos_rm(path, is_server=False):
     try:
         path = Path(path).expanduser().resolve()
-        cmd = subprocess.run(['eos', 'rm', _eos_path(path)], stdout=subprocess.PIPE, env=eos_env)
+        cmd = subprocess.run(['eos', 'rm', _eos_path(path)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=eos_env)
         if cmd.returncode == 0:
             return 0
         else:

xboinc/simulation_io/__init__.py

@@ -4,5 +4,6 @@
 # ######################################### #
 
 from .default_tracker import get_default_tracker
-from .output import SimState, SimVersion
+from .output import SimState
 from .input import SimConfig
+from .version import SimVersion, app_version, app_version_int

xboinc/simulation_io/output.py

@@ -3,48 +3,13 @@
 # Copyright (c) CERN, 2023.                 #
 # ######################################### #
 
-import sys
 from pathlib import Path
 import numpy as np
 
 import xobjects as xo
 import xpart as xp
 
-from ..general import __version__
-
-
-# XXX.YYY.ZZZ to XXXYYYZZZ
-def _version_to_int(version):
-    vers = version.split('.')
-    return int(vers[0])*1000000 + int(vers[1])*1000 + int(vers[2])
-
-# XXXYYYZZZ to XXX.YYY.ZZZ
-def _int_to_version(verint):
-    XXX = int(verint / 1000000)
-    YYY = int((verint - XXX*1000000)/1000)
-    ZZZ = int(verint - XXX*1000000 - YYY*1000)
-    return f"{XXX}.{YYY}.{ZZZ}"
-
-
-# This class overloads the first field from the SimState,
-# in order to read the correct version from the binary
-class SimVersion(xo.HybridClass):
-    _xofields = {
-        'xboinc_version':   xo.Int64,    # version XXX.YYY.ZZZ as int
-    }
-
-    def __init__(self, **kwargs):
-        if '_xobject' not in kwargs:
-            kwargs['xboinc_version'] = _version_to_int(__version__)
-        super().__init__(**kwargs)
-
-    def assert_version(self):
-        if int(_version_to_int(__version__)/1000) != int(self.xboinc_version/1000):
-            error += f"Incompatible xboinc version! Output file needs "
-            error +  f"{_int_to_version(self.xboinc_version)}, "
-            error +  f"but current version is {__version__}.\n"
-            raise ValueError(error)
-
+from .version import SimVersion
 
 class SimState(xo.HybridClass):
     _xofields = {

xboinc/simulation_io/version.py

@@ -0,0 +1,45 @@
+# copyright ############################### #
+# This file is part of the Xboinc Package.  #
+# Copyright (c) CERN, 2023.                 #
+# ######################################### #
+
+import xobjects as xo
+import xpart as xp
+
+from ..general import __version__
+
+
+# XXX.YYY.ZZZ to XXXYYY  (ignore patch)
+def _version_to_int(version):
+    vers = version.split('.')
+    return int(vers[0])*1000 + int(vers[1])
+
+# XXXYYY to XXX.YYY
+def _int_to_version(verint):
+    XXX = int(verint / 1000)
+    YYY = int(verint - XXX*1000)
+    return f"{XXX}.{YYY}"
+
+
+app_version = '.'.join(__version__.split('.')[:2])
+app_version_int = _version_to_int(__version__)
+
+
+# This class overloads the first field from the SimState,
+# in order to read the correct version from the binary
+class SimVersion(xo.HybridClass):
+    _xofields = {
+        'xboinc_version': xo.Int64,    # version XXX.YYY as int
+    }
+
+    def __init__(self, **kwargs):
+        if '_xobject' not in kwargs:
+            kwargs['xboinc_version'] = _version_to_int(__version__)
+        super().__init__(**kwargs)
+
+    def assert_version(self):
+        if app_version_int != self.xboinc_version:
+            error += f"Incompatible xboinc version! Output file needs "
+            error +  f"{_int_to_version(self.xboinc_version)}, "
+            error +  f"but current version is {__version__}.\n"
+            raise ValueError(error)