Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
nguu0123 committed Aug 26, 2024
1 parent c40b4eb commit 74ac4b3
Show file tree
Hide file tree
Showing 17 changed files with 127 additions and 1,468 deletions.
38 changes: 19 additions & 19 deletions example/docker_report/config/client1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,23 +15,23 @@ connector:
# exchange_name: test_qoa4ml
# exchange_type: topic
# out_routing_key: test.client1
probes:
- probe_type: "docker"
frequency: 9999999
require_register: false
log_latency_flag: false
environment: Edge
container_list: ["test"]
# probes:
# - probe_type: "docker"
# frequency: 9999999
# require_register: false
# log_latency_flag: false
# environment: Edge
# container_list: ["test"]

# - probe_type: "system"
# frequency: 1
# require_register: false
# log_latency_flag: false
# environment: Edge
# node_name: "Edge1"
#
# - probe_type: "process"
# frequency: 1
# require_register: false
# log_latency_flag: false
# environment: Edge
# - probe_type: "system"
# frequency: 1
# require_register: false
# log_latency_flag: false
# environment: Edge
# node_name: "Edge1"
#
# - probe_type: "process"
# frequency: 1
# require_register: false
# log_latency_flag: false
# environment: Edge
6 changes: 5 additions & 1 deletion example/docker_report/docker_probe.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

import random
import signal
import sys
import time
Expand All @@ -17,7 +18,10 @@ def signal_handler(sig, frame):

signal.signal(signal.SIGINT, signal_handler)
if __name__ == "__main__":
client1.start_all_probes()
# client1.start_all_probes()
# NOTE: here the probe will run in the background so if the script ends fast, the probe will result in error
for _ in range(5):
client1.observe_metric("metric1", random.randint(1, 100), 0)
client1.observe_metric("metric2", random.randint(1, 100), 0)
client1.report(submit=True)
time.sleep(1)
27 changes: 14 additions & 13 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,19 +31,20 @@ classifiers = [
]

dependencies = [
"pika>=1.3.2",
"psutil>=5.9.2",
"PyYAML>=6.0.1",
"requests>=2.31.0",
"fastapi>=0.110.2",
"flatten_dict>=0.4.2",
"pydantic>=2.7.4,<2.8",
"tinyflux>=1.0.0",
"uvicorn>=0.29.0",
"lazy-import>=0.2.2",
"confluent-kafka>=2.4.0",
"docker==7.1.0",
"numpy>=1.23.5",
"pika>=1.3.2",
"psutil>=5.9.2",
"PyYAML>=6.0.1",
"requests>=2.31.0",
"fastapi>=0.110.2",
"flatten_dict>=0.4.2",
"pydantic>=2.7.4,<2.8",
"tinyflux>=1.0.0",
"uvicorn>=0.29.0",
"lazy-import>=0.2.2",
"confluent-kafka>=2.4.0",
"docker==7.1.0",
"numpy>=1.23.5",
"jetson-stats>=4.2.9",
]

optional-dependencies.ml = [
Expand Down
6 changes: 6 additions & 0 deletions requirements-dev.lock
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ deprecated==1.2.14
devtools==0.12.2
distlib==0.3.8
# via virtualenv
distro==1.9.0
# via jetson-stats
docker==7.1.0
# via qoa4ml
exceptiongroup==1.2.2
Expand Down Expand Up @@ -88,6 +90,8 @@ importlib-metadata==8.0.0
# via opentelemetry-api
iniconfig==2.0.0
# via pytest
jetson-stats==4.2.9
# via qoa4ml
jinja2==3.1.4
# via mkdocs
# via mkdocs-material
Expand Down Expand Up @@ -240,6 +244,8 @@ six==1.16.0
# via python-dateutil
# via tensorboard
# via tensorflow
smbus2==0.4.3
# via jetson-stats
sniffio==1.3.1
# via anyio
starlette==0.38.2
Expand Down
6 changes: 6 additions & 0 deletions requirements.lock
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ confluent-kafka==2.5.0
deprecated==1.2.14
# via opentelemetry-api
# via opentelemetry-semantic-conventions
distro==1.9.0
# via jetson-stats
docker==7.1.0
# via qoa4ml
exceptiongroup==1.2.2
Expand Down Expand Up @@ -58,6 +60,8 @@ idna==3.7
# via requests
importlib-metadata==8.0.0
# via opentelemetry-api
jetson-stats==4.2.9
# via qoa4ml
keras==3.5.0
# via tensorflow
lazy-import==0.2.2
Expand Down Expand Up @@ -147,6 +151,8 @@ six==1.16.0
# via lazy-import
# via tensorboard
# via tensorflow
smbus2==0.4.3
# via jetson-stats
sniffio==1.3.1
# via anyio
starlette==0.38.2
Expand Down
20 changes: 17 additions & 3 deletions src/qoa4ml/config/configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,21 @@
class ClientInfo(BaseModel):
id: str = Field(
default="",
description="Should be not be set manually, auto-generated by QoAClient",
description="The id of the QoAClient, should be not be set manually, auto-generated by QoAClient",
)
name: str = ""
name: str = Field(
default="",
description="The name of the QoAClient user, for example, AaltoSEA",
)

user_id: str = Field(
default="",
description="Can be set if you have external user_id, otherwise auto generated",
)
username: str = Field(default="", description="The name of the user")
instance_id: str = Field(
default="",
description="Should be not be set manually, auto-generated by QoAClient",
description="Can be set manually, if not, auto-generated by QoAClient",
)
instance_name: str = Field(default="", description="The name of the instance")
stage_id: str = ""
Expand Down Expand Up @@ -181,6 +185,16 @@ class DockerProbeConfig(ProbeConfig):
container_list: list[str] = []


class JetsonSystemProbeConfig(ProbeConfig):
probe_type: str = Field("jetson_sys")
node_name: str | None


class JetsonProcessesProbeConfig(ProbeConfig):
probe_type: str = Field("jetson_sys")
node_name: str | None


class NodeAggregatorConfig(BaseModel):
socket_collector_config: SocketCollectorConfig
environment: EnvironmentEnum
Expand Down
9 changes: 6 additions & 3 deletions src/qoa4ml/qoa_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ def __init__(
# Set default connector for sending monitoring data if not specify
self.default_connector = next(iter(self.connector_list.keys()))

self.probes_list = None
if self.configuration.probes:
self.probes_list = self.init_probes(self.configuration.probes)
# lock report to guarantee consistency
Expand Down Expand Up @@ -265,9 +266,11 @@ def asyn_report(self, body_mess: str, connectors: Optional[list] = None):
if connectors is None:
# if connectors are not specify, use default
if self.default_connector:
self.connector_list[self.default_connector].send_report(
body_mess, corr_id=str(uuid.uuid4())
)
chosen_connector = self.connector_list[self.default_connector]
if isinstance(chosen_connector, AmqpConnector):
chosen_connector.send_report(body_mess, corr_id=str(uuid.uuid4()))
else:
chosen_connector.send_report(body_mess)
else:
qoa_logger.error(
"No default connector, please specify the connector to use"
Expand Down
1 change: 1 addition & 0 deletions src/qoa4ml/reports/resources_report_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ class ProcessMetadata(BaseModel):

class SystemMetadata(BaseModel):
node_name: str
model: str | None = None


class ResourceReport(BaseModel):
Expand Down
113 changes: 0 additions & 113 deletions src/qoa4ml/utils/jetson/command.py

This file was deleted.

Loading

0 comments on commit 74ac4b3

Please sign in to comment.