Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ENH: ssh/scp data for tcbsd PLCs #25

Merged
merged 17 commits into from
Mar 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/standard.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,6 @@ jobs:
# Extras to be installed only for conda-based testing:
conda-testing-extras: ""
# Extras to be installed only for pip-based testing:
pip-testing-extras: "PyQt5"
pip-testing-extras: ""
# Set if using setuptools-scm for the conda-build workflow
use-setuptools-scm: true
43 changes: 43 additions & 0 deletions conda-recipe/meta.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
{% set package_name = "pmpsdb_client" %}
{% set import_name = "pmpsdb_client" %}
{% set version = load_file_regex(load_file=os.path.join(import_name, "_version.py"), regex_pattern=".*version = '(\S+)'").group(1) %}

package:
name: {{ package_name }}
version : {{ version }}

source:
path: ..

build:
number: 0
noarch: python
script: {{ PYTHON }} -m pip install . -vv

requirements:
build:
- python >=3.9
- pip
- setuptools_scm
run:
- python >=3.9
- fabric
- ophyd
- pcdscalc
- pcdsutils
- prettytable
- qtpy
run_constrained:
- pyqt =5

test:
requires:
- pytest
- pyqt=5.15
imports:
- {{ import_name }}

about:
home: https://github.com/pcdshub/pcdsdevices
license: SLAC Open License
summary: IOC definitions for LCLS Beamline Devices
2 changes: 2 additions & 0 deletions dev-requirements.txt
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
PyQt5
pytest
setuptools-scm
8 changes: 7 additions & 1 deletion pmpsdb_client/cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,18 @@ def _main(args: argparse.Namespace) -> int:
print(version)
return 0
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s %(levelname)s: %(name)s %(message)s",
)
else:
logging.basicConfig(
level=logging.INFO,
format='%(levelname)s: %(message)s',
)
# Noisy log messages from ssh transport layer
for module in ("fabric", "paramiko", "intake"):
logging.getLogger(module).setLevel(logging.WARNING)
if args.export_dir:
from ..export_data import set_export_dir
set_export_dir(args.export_dir)
Expand Down
4 changes: 2 additions & 2 deletions pmpsdb_client/cli/transfer_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from typing import Optional

from ..export_data import ExportFile, get_latest_exported_files
from ..ftp_data import (compare_file, download_file_text, list_file_info,
from ..plc_data import (compare_file, download_file_text, list_file_info,
upload_filename)

logger = logging.getLogger(__name__)
Expand All @@ -22,7 +22,7 @@ def _list_files(hostname: str) -> int:
infos = list_file_info(hostname=hostname)
for data in infos:
print(
f'{data.filename} uploaded at {data.create_time.ctime()} '
f'{data.filename} uploaded at {data.last_changed.ctime()} '
f'({data.size} bytes)'
)
if not infos:
Expand Down
28 changes: 28 additions & 0 deletions pmpsdb_client/data_types.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
"""
This module defines important data structures centrally.

This helps us compare the same kinds of data to each other,
even when this data comes from different sources.

Note that all the dataclasses are frozen: editing these data
structures is not in scope for this library, it is only intended
to move these files around and compare them to each other.
"""
import dataclasses
import datetime


@dataclasses.dataclass(frozen=True)
class FileInfo:
"""
Generalized file info.

Only contains fields available to both ftp and ssh.

This class has no constructor helpers here.
Each data source will need to implement a unique
constructor for this.
"""
filename: str
size: int
last_changed: datetime.datetime
157 changes: 27 additions & 130 deletions pmpsdb_client/ftp_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@

import datetime
import ftplib
import json
import logging
import os
import typing
from contextlib import contextmanager
from dataclasses import dataclass
from typing import BinaryIO, Iterator, TypeVar

from .data_types import FileInfo

DEFAULT_PW = (
('Administrator', '1'),
Expand All @@ -23,10 +24,11 @@
DIRECTORY = 'pmps'

logger = logging.getLogger(__name__)
T = TypeVar("T")


@contextmanager
def ftp(hostname: str, directory: typing.Optional[str] = None) -> ftplib.FTP:
def ftp(hostname: str, directory: str | None = None) -> Iterator[ftplib.FTP]:
"""
Context manager that manages an FTP connection.

Expand All @@ -50,7 +52,7 @@ def ftp(hostname: str, directory: typing.Optional[str] = None) -> ftplib.FTP:
# Default directory
directory = directory or DIRECTORY
# Create without connecting
ftp_obj = ftplib.FTP(hostname, timeout=2.0)
ftp_obj = ftplib.FTP(hostname, timeout=1.0)
# Beckhoff docs recommend active mode
ftp_obj.set_pasv(False)
# Best-effort login using default passwords
Expand Down Expand Up @@ -86,15 +88,15 @@ def ftp(hostname: str, directory: typing.Optional[str] = None) -> ftplib.FTP:

def list_filenames(
hostname: str,
directory: typing.Optional[str] = None,
directory: str | None = None,
) -> list[str]:
"""
List the filenames that are currently saved on the PLC.

Parameters
----------
hostname : str
The plc hostname to upload to.
The plc hostname to check.
directory : str, optional
The ftp subdirectory to read and write from
A default directory pmps is used if this argument is omitted.
Expand All @@ -109,20 +111,18 @@ def list_filenames(
return ftp_obj.nlst()


@dataclass
class PLCFile:
@dataclass(frozen=True)
class FTPFileInfo(FileInfo):
"""
Information about a file on the PLC as learned through ftp.

In the context of pmps, the create_time is the last time we
updated the database export file.
Contains very few fields: ftp doesn't give us a lot of info.
See data_types.FileInfo for the field information.
This protocol is what limits the amount of fields we can assume
are available when we don't know the PLC's type.
"""
filename: str
create_time: datetime.datetime
size: int

@classmethod
def from_list_line(cls, line: str) -> PLCFile:
def from_list_line(cls: type[T], line: str) -> T:
"""
Create a PLCFile from the output of the ftp LIST command.

Expand Down Expand Up @@ -150,15 +150,15 @@ def from_list_line(cls, line: str) -> PLCFile:
)
return cls(
filename=filename,
create_time=full_datetime,
size=int(size),
last_changed=full_datetime,
)


def list_file_info(
hostname: str,
directory: typing.Optional[str] = None,
) -> list[PLCFile]:
directory: str | None = None,
) -> list[FTPFileInfo]:
"""
Gather pertinent information about all the files.

Expand All @@ -180,14 +180,14 @@ def list_file_info(
lines = []
with ftp(hostname=hostname, directory=directory) as ftp_obj:
ftp_obj.retrlines('LIST', lines.append)
return [PLCFile.from_list_line(line) for line in lines]
return [FTPFileInfo.from_list_line(line) for line in lines]


def upload_file(
hostname: str,
target_filename: str,
fd: typing.BinaryIO,
directory: typing.Optional[str] = None,
fd: BinaryIO,
directory: str | None = None,
):
"""
Upload an open file to a PLC.
Expand Down Expand Up @@ -219,8 +219,8 @@ def upload_file(
def upload_filename(
hostname: str,
filename: str,
dest_filename: typing.Optional[str] = None,
directory: typing.Optional[str] = None,
dest_filename: str | None = None,
directory: str | None = None,
):
"""
Open and upload a file on your filesystem to a PLC.
Expand All @@ -230,7 +230,9 @@ def upload_filename(
hostname : str
The plc hostname to upload to.
filename : str
The name of the file on both your filesystem and on the PLC.
The name of the file on your filesystem.
dest_filename : str, optional
The name of the file on the PLC. If omitted, same as filename.
directory : str, optional
The ftp subdirectory to read and write from
A default directory pmps is used if this argument is omitted.
Expand All @@ -254,7 +256,7 @@ def upload_filename(
def download_file_text(
hostname: str,
filename: str,
directory: typing.Optional[str] = None,
directory: str | None = None,
) -> str:
"""
Download a file from the PLC to use in Python.
Expand Down Expand Up @@ -290,108 +292,3 @@ def download_file_text(
for chunk in byte_chunks:
contents += chunk.decode('ascii')
return contents


def download_file_json_dict(
hostname: str,
filename: str,
directory: typing.Optional[str] = None,
) -> dict[str, dict[str, typing.Any]]:
"""
Download a file from the PLC and interpret it as a json dictionary.

The result is suitable for comparing to json blobs exported from the
pmps database.

Parameters
----------
hostname : str
The plc hostname to download from.
filename : str
The name of the file on the PLC.
directory : str, optional
The ftp subdirectory to read and write from
A default directory pmps is used if this argument is omitted.

Returns
-------
data : dict
The dictionary data from the file stored on the plc.
"""
logger.debug(
'download_file_json_dict(%s, %s, %s)',
hostname,
filename,
directory,
)
return json.loads(
download_file_text(
hostname=hostname,
filename=filename,
directory=directory,
)
)


def local_file_json_dict(filename: str) -> dict[str, dict[str, typing.Any]]:
"""
Return the json dict from a local file.

Suitable for comparisons to files from the database or from the plc.

Parameters
----------
filename : str
The name of the file on the local filesystem.

Returns
-------
data : dict
The dictionary data from the file stored on the local drive.
"""
logger.debug('local_file_json_dict(%s)', filename)
with open(filename, 'r') as fd:
return json.load(fd)


def compare_file(
hostname: str,
local_filename: str,
plc_filename: typing.Optional[str] = None,
directory: typing.Optional[str] = None,
) -> bool:
"""
Compare a file saved locally to one on the PLC.

Parameters
----------
hostname : str
The plc hostname to download from.
local_filename: str
The full path the local file to compare with.
plc_filename: str, optional
The filename as saved on the PLC. If omitted, the local_filename's
basename will be used.
directory : str, optional
The ftp subdirectory to read and write from
A default directory pmps is used if this argument is omitted.

Returns
-------
same_file : bool
True if the contents of these two files are the same.
"""
logger.debug(
'compare_file(%s, %s, %s, %s)',
hostname,
local_filename,
plc_filename,
directory,
)
local_data = local_file_json_dict(filename=local_filename)
plc_data = download_file_json_dict(
hostname=hostname,
filename=plc_filename,
directory=directory,
)
return local_data == plc_data
Loading
Loading