Skip to content

Commit

Permalink
More doc
Browse files Browse the repository at this point in the history
  • Loading branch information
@chaen
chaen committed Feb 5, 2025
1 parent 19913ec commit e10ba32
Show file tree
Hide file tree
Showing 5 changed files with 160 additions and 39 deletions.
145 changes: 112 additions & 33 deletions diracx-cli/src/diracx/cli/internal/legacy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import hashlib
import json
import os
import re
from pathlib import Path
from typing import TYPE_CHECKING, cast
from urllib.parse import urljoin, urlparse
Expand All @@ -26,6 +27,12 @@
app = AsyncTyper()


BASE_64_URL_SAFE_PATTERN = (
r"(?:[A-Za-z0-9\-_]{4})*(?:[A-Za-z0-9\-_]{2}==|[A-Za-z0-9\-_]{3}=)?"
)
LEGACY_EXCHANGE_PATTERN = rf"diracx:legacy:({BASE_64_URL_SAFE_PATTERN})"


class IdPConfig(BaseModel):
URL: str
ClientID: str
Expand Down Expand Up @@ -182,12 +189,13 @@ def generate_helm_values(
"developer": {"enabled": False},
"initCs": {"enabled": True},
"initSecrets": {"enabled": True},
"initSql": {"enabled": False, "env": {}},
"initSql": {"enabled": False},
"cert-manager": {"enabled": False},
"cert-manager-issuer": {"enabled": False},
"minio": {"enabled": False},
"dex": {"enabled": False},
"opensearch": {"enabled": False},
# This is Openshift specific, change it maybe
"ingress": {
"enabled": True,
"className": None,
Expand All @@ -199,12 +207,7 @@ def generate_helm_values(
},
"rabbitmq": {"enabled": False},
"mysql": {"enabled": False},
"diracx": {
"manageOSIndices": False,
"mysqlDatabases": [],
"osDatabases": [],
"settings": {},
},
"global": {"images": {"services": "FILL ME"}, "storageClassName": "FILL ME"},
}

cfg = diraccfg.CFG().loadFromBuffer(public_cfg.read_text())
Expand All @@ -218,13 +221,11 @@ def generate_helm_values(
diracx_hostname = urlparse(diracx_url).netloc.split(":", 1)[0]
# Remove the port
diracx_config = {
"manageOSIndices": False,
"mysqlDatabases": [],
"osDatabases": [],
"settings": {},
"sqlDbs": {},
"osDbs": {},
}

diracx_settings: dict[str, str] = {}
diracx_settings: dict[str, str] = {"DIRACX_CONFIG_BACKEND_URL": "FILL ME"}
diracx_config["settings"] = diracx_settings
helm_values["diracx"] = diracx_config
diracx_config["hostname"] = diracx_hostname
Expand All @@ -236,47 +237,122 @@ def generate_helm_values(
]
)

### SQL DBs

default_db_user = cfg["Systems"].get("Databases", {}).get("User")
default_db_password = cfg["Systems"].get("Databases", {}).get("Password")

default_setup = cfg["DIRAC"]["Setup"]
default_db_host = cfg["Systems"].get("Databases", {}).get("Host", "FILL ME")
default_db_port = cfg["Systems"].get("Databases", {}).get("Port", "FILL ME")

all_db_configs = {}
for system, system_config in cfg["Systems"].items():
system_setup = cfg["DIRAC"]["Setups"][default_setup].get(system, None)
if system_setup:
all_db_configs.update(system_config[system_setup].get("Databases", {}))
sql_dbs = {
"dbs": {},
"default": {
"host": f"{default_db_host}:{default_db_port}",
"password": default_db_password,
"rootPassword": "FILL ME",
"rootUser": "FILL ME",
"user": default_db_user,
},
}
for _system, system_config in cfg["Systems"].items():
all_db_configs.update(system_config.get("Databases", {}))

from diracx.core.extensions import select_from_extension

for entry_point in select_from_extension(group="diracx.db.sql"):

db_name = entry_point.name
db_config = all_db_configs.get(db_name, {})

sql_dbs["dbs"][db_name] = {}
# There is a DIRAC AuthDB, but it is not the same
# as the DiracX one
if db_name == "AuthDB":
url_name = "DIRACX_DB_URL_AUTHDB"
connection_string = "FILL ME: I am a new DB, create me"
else:
db_config = all_db_configs[db_name]
url_name = f"DIRACX_DB_URL_{entry_point.name.upper()}"
db_user = db_config.get("User", default_db_user)
db_password = db_config.get("Password", default_db_password)
db_host = db_config["Host"]
db_port = db_config["Port"]
sql_dbs["dbs"]["AuthDB"] = {"internalName": "DiracXAuthDB"}

if "DBName" in db_config:
indb_name = db_config["DBName"]
if indb_name != db_name:
sql_dbs["dbs"]["internalName"] = indb_name
if "User" in db_config:
sql_dbs["dbs"][db_name]["user"] = db_config.get("User")
if "Password" in db_config:
sql_dbs["dbs"][db_name]["password"] = db_config.get("Password")
if "Host" in db_config or "Port" in db_config:
sql_dbs["dbs"][db_name][
"host"
] = f"{db_config.get('Host', default_db_host)}:{db_config.get('Port', default_db_port)}"
if not sql_dbs["dbs"][db_name]:
sql_dbs["dbs"][db_name] = None

diracx_config["sqlDbs"] = sql_dbs

#### END SQL DB

# #### OS DBs

default_os_db_user = cfg["Systems"].get("NoSQLDatabases", {}).get("User")
default_os_db_password = cfg["Systems"].get("NoSQLDatabases", {}).get("Password")
default_os_db_host = cfg["Systems"].get("NoSQLDatabases", {}).get("Host", "FILL ME")

os_dbs = {
"dbs": {},
"default": {
"host": f"{default_os_db_host}",
"password": default_os_db_password,
"rootPassword": "FILL ME",
"rootUser": "FILL ME",
"user": default_os_db_user,
},
}

for entry_point in select_from_extension(group="diracx.db.os"):
db_name = entry_point.name
db_config = all_db_configs.get(db_name, {})

os_dbs["dbs"][db_name] = {}
# There is a DIRAC AuthDB, but it is not the same
# as the DiracX one

if "DBName" in db_config:
indb_name = db_config["DBName"]
if indb_name != db_name:
os_dbs["dbs"]["internalName"] = indb_name
if "User" in db_config:
os_dbs["dbs"][db_name]["user"] = db_config["User"]
if "Password" in db_config:
os_dbs["dbs"][db_name]["password"] = db_config["Password"]
if "Host" in db_config:
os_dbs["dbs"][db_name]["host"] = db_config["Host"]

connection_string = f"mysql+aiomysql://{db_user}:{db_password}@{db_host}:{db_port}/{indb_name}"
diracx_settings[url_name] = connection_string
if not os_dbs["dbs"][db_name]:
os_dbs["dbs"][db_name] = None

diracx_config["osDbs"] = os_dbs

#### End OS DBs

# Settings for the legacy
try:
if match := re.fullmatch(
LEGACY_EXCHANGE_PATTERN, cfg["DiracX"]["LegacyExchangeApiKey"]
):
raw_token = base64.urlsafe_b64decode(match.group(1))
else:
raise ValueError(
"Invalid authorization header",
)

diracx_settings["DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY"] = hashlib.sha256(
base64.urlsafe_b64decode(cfg["DiracX"]["LegacyExchangeApiKey"])
raw_token
).hexdigest()
except KeyError:
typer.echo(
"ERROR: you must have '/DiracX/LegacyExchangeApiKey' already set", err=True
)
error_msg = """
ERROR: you must have '/DiracX/LegacyExchangeApiKey' already set.
See the `legacy_exchange` function definition for how to generate it in python
"""
typer.echo(error_msg, err=True)
raise typer.Exit(1) from None
# Sandboxstore settings
# TODO: Integrate minio for production use (ingress, etc)
Expand All @@ -295,3 +371,6 @@ def generate_helm_values(
diracx_settings["DIRACX_SERVICE_JOBS_ENABLED"] = "true"
diracx_settings["DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET"] = "true"
output_file.write_text(yaml.safe_dump(helm_values))
typer.echo(
"The file is incomplete and needs manual editing (grep for 'FILL ME')", err=True
)
4 changes: 2 additions & 2 deletions diracx-cli/tests/legacy/cs_sync/test_cssync.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
def test_cs_sync(tmp_path, monkeypatch):
monkeypatch.setenv("DIRAC_COMPAT_ENABLE_CS_CONVERSION", "Yes")

output_file = tmp_path / "default.yaml"
output_file = tmp_path / "default.yml"

result = runner.invoke(
app,
Expand All @@ -40,7 +40,7 @@ def test_disabled_vos_empty(tmp_path, monkeypatch):
# # DisabledVOs cannot be set if any Legacy clients are enabled
monkeypatch.setenv("DIRAC_COMPAT_ENABLE_CS_CONVERSION", "Yes")

output_file = tmp_path / "default.yaml"
output_file = tmp_path / "default.yml"

result = runner.invoke(
app,
Expand Down
34 changes: 31 additions & 3 deletions docs/CONFIGURATION.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ We recommend that this is stored within a Git repository, and DiracX provides tw
## Structure of the CS

The canonical way of accessing the DiracX configuration from within code is via the corresponding pydantic model.
This provides strong typing of all values in the CS and enables the schema of the `default.yaml` file to be validated.
This provides strong typing of all values in the CS and enables the schema of the `default.yml` file to be validated.
The pydantic model is defined in `diracx.core.config`.

## Client access
Expand All @@ -28,11 +28,39 @@ These headers can be used to efficiently check for updates without needing to do
Currently the canonical source of configuration is from the legacy DIRAC Configuration Service.
We forsee this will continue to be the case until the migration from DIRAC -> DiracX is complete.
During this time, the DiracX configuration is not intended to be edited directly.
The DiracX `default.yaml` file differs in structure and contents from the legacy DIRAC Configuration Service.
The DiracX `default.yml` file differs in structure and contents from the legacy DIRAC Configuration Service.
The legacy DIRAC CFG file can be converted into the new YAML format with:

```bash
DIRAC_COMPAT_ENABLE_CS_CONVERSION=true dirac internal legacy cs-sync dirac-cs.cfg diracx-config/default.yaml
DIRAC_COMPAT_ENABLE_CS_CONVERSION=true dirac internal legacy cs-sync dirac-cs.cfg diracx-config/default.yml
```

The following can be run on any client with a proxy

```python
#!/usr/bin/env python
import subprocess
import os
import tempfile
import zlib
from pathlib import Path

import DIRAC
DIRAC.initialize()
from DIRAC import gConfig
from DIRAC.Core.Utilities.ReturnValues import returnValueOrRaise
from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient

client = ConfigurationClient(url=gConfig.getValue("/DIRAC/Configuration/MasterServer", ""))
data = returnValueOrRaise(client.getCompressedData())
data = zlib.decompress(data)
with tempfile.NamedTemporaryFile() as tmp:
tmp.write(data)
tmp.flush()
cmd = ["dirac", "internal", "legacy", "cs-sync", tmp.name, "default.yml"]
subprocess.run(cmd, env=os.environ | {"DIRAC_COMPAT_ENABLE_CS_CONVERSION": "yes"}, check=True)

print("Synced CS to default.yml, now you can review the changes and commit/push them")
```

TODO: Document how we will actually do the sync for production deployments...
12 changes: 11 additions & 1 deletion extensions/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,17 @@ The requirements are the following:

To create a client extension:
* mirror the structure of the `diracx-client`
* Generate a client in `generated` using `Autorest`
* Generate a client in `generated` using `Autorest` For this the best is to have a temporary router test writing the `openapi.json` somewhere
```python
r = normal_user_client.get("/api/openapi.json")
with open('/tmp/openapi.json', 'wt') as f:
json.dump(r.json(), f, indent=2)
```
* The autorest command then looks something like
```bash
autorest --python --input-file=/tmp/openapi.json --models-mode=msrest --namespace=generated --output-folder=gubbins-client/src/gubbins/
```

* Create the `patches` directory, simply exporting the generated `clients`(both [sync](gubbins/gubbins-client/src/gubbins/client/patches/__init__.py) and [async](gubbins/gubbins-client/src/gubbins/client/patches/aio/__init__.py))
* Define the base modules to export what is needed
* The [top init](gubbins/gubbins-client/src/gubbins/client/__init__.py) MUST have
Expand Down
4 changes: 4 additions & 0 deletions extensions/gubbins_values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ global:
imagePullPolicy: IfNotPresent
images:
services: gubbins/services

developer:
enabled: true

diracx:
sqlDbs:
dbs:
Expand Down

0 comments on commit e10ba32

Please sign in to comment.