Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for 3D boundary files #14

Open
krober10nd opened this issue Jan 29, 2022 · 4 comments
Open

Support for 3D boundary files #14

krober10nd opened this issue Jan 29, 2022 · 4 comments

Comments

@krober10nd
Copy link

krober10nd commented Jan 29, 2022

Hey Luc,

I'm working on some 3D TELEMAC modeling applications and I would like to produce the file used to force the boundary. In particular the file this script produces.

http://docs.opentelemac.org/doxypydocs/v8p2r0/html/namespacepretel_1_1convert__to__bnd.html

I've already successfully interpolated 3D CMEMs temperature, salinity, and the general circulation to an extruded 2D unstructured grid using PyTelTools so essentially I just can grab the values at the liquid boundary and write those to this file format.

I'm in the process of trying to write a script that uses PyTelTools instead of the official Python TELEMAC API to write the file, however I'm not confident it will work because it seems to be mixing 2D/3D data structures in the same file. For instance, it seems to have a header with both ikle2 (2D triangulation connectivity) and ikle3 (3D triangulation connectivity). The IKLE2 has been modified to only represent boundary segments through some very confusing numpy logic.

Any advice? Do you think this could be possible to create this file using the existing code in PyTelTools or would it require additional data structures that would need to be added to PyTelTools?

Thank you,

Keith

@krober10nd
Copy link
Author

krober10nd commented Jan 30, 2022

I think I got somewhere with this, although this Serafin file can't be visualized in BlueKenue (neither the original one produced by the pyTelemac API).

# from matplotlib import pyplot as plt
import numpy as np
from pyteltools.slf import Serafin
from tqdm import tqdm


def getFileContent(file):
    ilines = []
    SrcF = open(file, "r")
    for line in SrcF:
        ilines.append(line)
    SrcF.close()
    return ilines


class CONLIM:
    def __init__(self, file_name):
        self.file_name = file_name
        # ~~> Columns of integers and floats
        DTYPE = np.dtype(
            [
                ("lih", "<i4"),
                ("liu", "<i4"),
                ("liv", "<i4"),
                ("h", "<f4"),
                ("u", "<f4"),
                ("v", "<f4"),
                ("au", "<f4"),
                ("lit", "<i4"),
                ("t", "<f4"),
                ("at", "<f4"),
                ("bt", "<f4"),
                ("n", "<i4"),
                ("c", "<i4"),
            ],
        )

        if file_name != "":
            # ~~> Number of boundary points ( tuple() necessary for dtype parsing )
            core = [tuple(c.strip().split()[0:13]) for c in getFileContent(file_name)]
            self.NPTFR = len(core)
            self.BOR = np.array(core, DTYPE)
            # ~~> Dictionary of KFRGL
            self.KFRGL = dict(zip(self.BOR["n"] - 1, range(self.NPTFR)))
            # ~~> Filtering indices
            self.INDEX = np.array(range(self.NPTFR), dtype=int)


def make_3d_boundary(
    mesh_file, cmems_file, cli_file, output_name="BOUNDARY_TESTING.slf"
):
    # Read the mesh
    with Serafin.Read(mesh_file, "en") as rfile:
        rfile.read_header()
        nodes_x = rfile.header.x_stored
        nodes_y = rfile.header.y_stored
        nodes = np.column_stack((nodes_x, nodes_y))

    # Read the boundary info
    cli = CONLIM(cli_file)
    bor = np.extract(cli.BOR["lih"] != 2, cli.BOR["n"]) - 1
    bor_xy = nodes[bor, :]

    # verify the boundary nodes are correct
    # fig, ax = plt.subplots()
    # ax.plot(nodes[:,0],nodes[:,1],'r.')
    # ax.plot(nodes[bor-1,0],nodes[bor-1,1],'bs')
    # ax.set_aspect('equal')
    # plt.show()

    # Read the CMEMS file
    with Serafin.Read(cmems_file, "en") as resin:
        resin.read_header()
        print(resin.header.summary())
        # Define some variables
        nvar = resin.header.nb_var
        nplan = resin.header.nb_planes
        nelem2 = resin.header.nb_elements // (nplan - 1)
        npoin2 = len(bor_xy)
        npoin3 = npoin2 * nplan
        # Number of nodes per boundary element (ndp2 in 2D and ndp3 in 3D)
        ndp2 = 2
        ndp3 = 4
        # Create the boundary Serafin file
        with Serafin.Write(output_name, "en", overwrite=True) as resout:
            output_header = resin.header.copy()
            output_header.empty_variables()
            output_header.to_single_precision()
            # Add the variable heaer info back
            for j, var_ID in enumerate(resin.header.var_IDs):
                output_header.add_variable_str(
                    resin.header.var_IDs[j],
                    bytes.decode(resin.header.var_names[j]),
                    bytes.decode(resin.header.var_units[j]),
                )
            # Update the connectivity table for the boundary format
            # Forms the 2D IKLE table for the triangles
            _ikle2 = np.take(
                output_header.ikle,
                [2 * 3 * i + j for i in range(nelem2) for j in range(3)],
            )
            _ikle2 = _ikle2.reshape(nelem2, 3)
            _ikle2 -= 1  # Make it zero-based
            array_1d = np.in1d(_ikle2, np.sort(bor))
            mask = _ikle2[np.where(np.sum(array_1d.reshape(nelem2, 3), axis=1) == 2)]
            # The boundary edges
            # This ikle2 keeps the original numbering
            ikle2 = np.ravel(mask)[np.in1d(mask, np.sort(bor))].reshape(len(mask), 2)
            # Update length of elements
            nelem2 = len(ikle2)
            # Build the knolg table for renumbering to start from 0
            knolg, _ = np.unique(np.ravel(ikle2), return_index=True)
            knogl = dict(zip(knolg, range(len(knolg))))
            for k in range(len(ikle2)):
                # _ikle2 has a local numbering, fit to the boundary elements
                ikle2[k] = [knogl[ikle2[k][0]], knogl[ikle2[k][1]]]
            # 3D structures (IKLE table is updated to reflect boundary structure)
            _tmp1 = np.repeat(npoin2 * np.arange(nplan - 1), nelem2 * ndp3)
            _tmp1 = _tmp1.reshape((nelem2 * (nplan - 1), ndp3))
            _tmp2 = np.tile(
                np.add(np.tile(ikle2, 2), np.repeat(npoin2 * np.arange(2), ndp2)),
                (nplan - 1, 1),
            )
            _tmp1 += _tmp2
            # Update IKLE table
            output_header.ikle = np.ravel(_tmp1) + 1  # 1-indexed

            # Updated node coordinates
            output_header.x_stored = np.tile(bor_xy[:, 0], nplan)
            output_header.y_stored = np.tile(bor_xy[:, 1], nplan)

            # Number of boundary nodes in 3D
            output_header.nb_nodes = npoin3

            # Number of boundary elements in 3D
            output_header.nb_elements = nelem2 * (output_header.nb_planes - 1)
            output_header.nb_nodes_per_elem = ndp3  # 4

            # Update IPOBO (not used anyway)
            output_header.ipobo = np.zeros(output_header.nb_nodes, dtype=int)

            # Write the header
            print("\n")
            print("-----------------------OUTPUT-----------------------------")
            print("----------------------------------------------------------\n")
            print(output_header.summary())
            print("\n")
            resout.write_header(output_header)

            # Since the mesh is the same, by pass this bizzare interpolation thing.
            resin.get_time()
            times = np.array(resin.time)
            # For all timesteps in CMEMs file

            for time_index in tqdm(
                range(len(times)), ncols=75, desc="Writing time frames..."
            ):
                # time = times[time_index]

                data_at_boundary = []
                # For all variables extract at the boundary nodes
                for j, var_ID in enumerate(resin.header.var_IDs):
                    data = resin.read_var_in_frame_as_3d(time_index, var_ID)
                    # Layers x number of boundary points
                    _tmp = data[:, bor]
                    # Create synthetic data for testing
                    # each variable is a unique integer: nplan*j : nplan*j + nplan
                    # _tmp = np.tile(
                    #    np.arange(nplan * j, nplan * j + nplan), (1, len(bor))
                    # )
                    _tmp = np.ravel(_tmp)[
                        :, None
                    ]  # this extra dim is necessary for later tranpose
                    data_at_boundary.append(_tmp)
                data_at_boundary = np.asarray(data_at_boundary)
                # variables, then for each boundary node, the values at the 10 layers
                # [var0,..,varN][bouNode0@Layer0,..,bouNode0@LayerN,boudNode1@Layer0,..,bouNode1@LayerN,..,bouNodeN@LayerN]
                # Reorganize data so it appears as layers first then boundary nodes
                # number of variables x Layer0@BouNode0,..,Layer0@BouNodeN,Layer1@BouNode0,..Layer1@BouNodeN
                reogr_data_at_boundary = np.reshape(
                    np.ravel(data_at_boundary), (nvar, npoin2, nplan)
                )
                reogr_data_at_boundary = np.transpose(reogr_data_at_boundary, (0, 2, 1))
                reorg_data_at_boundary = np.reshape(
                    reogr_data_at_boundary, (nvar, npoin3)
                )
                # Reogranize the data to be number of layers by
                # data_at_boundary = np.reshape(
                #    np.transpose(
                #        np.reshape(np.ravel(data_at_boundary), (nvar, npoin2, nplan)),
                #        (0, 2, 1),
                #    ),
                #    (nvar, npoin3),
                # )
                resout.write_entire_frame(
                    output_header, time_index, reorg_data_at_boundary
                )


if __name__ == "__main__":
    mesh_file = "./PCCA_tmac_v4.1.slf"
    cli_file = "./PCCA_tmac_v4.1.UVH.cli"
    cmems_file = "./CMEM_Jul_6hrs_v4.1.slf"
    output_file = "BOUNDARY_TESTING.slf"

    make_3d_boundary(mesh_file, cmems_file, cli_file, output_name=output_file)

@lucduron
Copy link
Member

Dear Keith,

I see your message and the script you did.
Yes the data structure for 2D and 3D Serafin file are mixed in PyTelTools (as the file format are very close).
Unfortunately I never tried to generate the files your want : boundary condition file (cli) and tidal boundary Serafin file (specific SLF file).
I do not know how a the file with nb_nodes_per_elem (or npd) with a value different of 3 or 6, can be built.
The thing you will have probably noticed is that with your script is that you can not open such a file with PyTelTools :

def _check_dim(self):
# verify data consistence and determine 2D or 3D
if self.is_2d:
if self.nb_nodes_per_elem != 3:
raise SerafinValidationError('Unknown mesh type')
else:
if self.nb_nodes_per_elem != 6:
raise SerafinValidationError('The number of nodes per element is not equal to 6')

Sorry I am not sure I will be able to help and do not really know if your questions are still relevant with your last investigations.

Best Regards,
Luc

@krober10nd
Copy link
Author

Luc,

Yes, as you correctly point out, it's because this file type represents the boundary surface in 3D (four nodes per face) and boundary edges in 2D (2 nodes per segment).

I am modifying pyTelTools to support it on my end. I will report back if I can get something working.

@krober10nd
Copy link
Author

krober10nd commented Feb 1, 2022

I was able to produce reasonable simulations using the binary boundary condition produced via this script. Note that I've interpolated CMEMs onto a 3D Serafin mesh already here (kwarg cmems_file) so in this step, I just grab the border vertices that are on the open ocean boundary. There is a nice CLI reader which I stole from here http://docs.opentelemac.org/doxypydocs/v8p2r0/html/classdata__manip_1_1formats_1_1conlim_1_1_conlim.html

Perhaps there could be a Cli class part of PyTeltools?

# from matplotlib import pyplot as plt
import numpy as np
from pyteltools.slf import Serafin
from tqdm import tqdm


def getFileContent(file):
    ilines = []
    SrcF = open(file, "r")
    for line in SrcF:
        ilines.append(line)
    SrcF.close()
    return ilines


class CONLIM:
    def __init__(self, file_name):
        self.file_name = file_name
        # ~~> Columns of integers and floats
        DTYPE = np.dtype(
            [
                ("lih", "<i4"),
                ("liu", "<i4"),
                ("liv", "<i4"),
                ("h", "<f4"),
                ("u", "<f4"),
                ("v", "<f4"),
                ("au", "<f4"),
                ("lit", "<i4"),
                ("t", "<f4"),
                ("at", "<f4"),
                ("bt", "<f4"),
                ("n", "<i4"),
                ("c", "<i4"),
            ],
        )

        if file_name != "":
            # ~~> Number of boundary points ( tuple() necessary for dtype parsing )
            core = [tuple(c.strip().split()[0:13]) for c in getFileContent(file_name)]
            self.NPTFR = len(core)
            self.BOR = np.array(core, DTYPE)
            # ~~> Dictionary of KFRGL
            self.KFRGL = dict(zip(self.BOR["n"] - 1, range(self.NPTFR)))
            # ~~> Filtering indices
            self.INDEX = np.array(range(self.NPTFR), dtype=int)


def make_3d_boundary(
    mesh_file, cmems_file, cli_file, output_name="BOUNDARY_TESTING.slf"
):
    # Read the mesh
    with Serafin.Read(mesh_file, "en") as rfile:
        rfile.read_header()
        nodes_x = rfile.header.x_stored
        nodes_y = rfile.header.y_stored
        nodes = np.column_stack((nodes_x, nodes_y))

    # Read the boundary info
    cli = CONLIM(cli_file)
    bor = np.extract(cli.BOR["lih"] != 2, cli.BOR["n"]) - 1
    bor_xy = nodes[bor, :]

    # verify the boundary nodes are correct
    # fig, ax = plt.subplots()
    # ax.plot(nodes[:,0],nodes[:,1],'r.')
    # ax.plot(nodes[bor-1,0],nodes[bor-1,1],'bs')
    # ax.set_aspect('equal')
    # plt.show()

    # Read the CMEMS file
    with Serafin.Read(cmems_file, "en") as resin:
        resin.read_header()
        print(resin.header.summary())
        # Define some variables
        nplan = resin.header.nb_planes
        nelem2 = resin.header.nb_elements // (nplan - 1)
        npoin2 = len(bor_xy)
        npoin3 = npoin2 * nplan
        # Number of nodes per boundary element (ndp2 in 2D and ndp3 in 3D)
        ndp2 = 2
        ndp3 = 4
        # Create the boundary Serafin file
        with Serafin.Write(output_name, "en", overwrite=True) as resout:
            output_header = resin.header.copy()
            output_header.empty_variables()
            output_header.to_single_precision()
            # Add the variable heaer info back
            desired_order = ["Z", "SALINITY", "TEMPERATURE", "U", "V"]
            actual_order = resin.header.var_IDs
            reorder_index = [actual_order.index(name) for name in desired_order]
            for j, var_ID in enumerate(resin.header.var_IDs):
                if var_ID != "W":
                    ro = reorder_index[j]
                    output_header.add_variable_str(
                        resin.header.var_IDs[ro],
                        bytes.decode(resin.header.var_names[ro]),
                        bytes.decode(resin.header.var_units[ro]),
                    )
            nvar = output_header.nb_var

            # Update the connectivity table for the boundary format
            # Forms the 2D IKLE table for the triangles
            _ikle2 = np.take(
                output_header.ikle,
                [2 * 3 * i + j for i in range(nelem2) for j in range(3)],
            )
            _ikle2 = _ikle2.reshape(nelem2, 3)
            _ikle2 -= 1  # Make it zero-based
            array_1d = np.in1d(_ikle2, np.sort(bor))
            mask = _ikle2[np.where(np.sum(array_1d.reshape(nelem2, 3), axis=1) == 2)]
            # The boundary edges
            # This ikle2 keeps the original numbering
            ikle2 = np.ravel(mask)[np.in1d(mask, np.sort(bor))].reshape(len(mask), 2)
            # Update length of elements
            nelem2 = len(ikle2)
            # Build the knolg table for renumbering to start from 0
            knolg, _ = np.unique(np.ravel(ikle2), return_index=True)
            knogl = dict(zip(knolg, range(len(knolg))))
            for k in range(len(ikle2)):
                # _ikle2 has a local numbering, fit to the boundary elements
                ikle2[k] = [knogl[ikle2[k][0]], knogl[ikle2[k][1]]]
            # 3D structures (IKLE table is updated to reflect boundary structure)
            _tmp1 = np.repeat(npoin2 * np.arange(nplan - 1), nelem2 * ndp3)
            _tmp1 = _tmp1.reshape((nelem2 * (nplan - 1), ndp3))
            _tmp2 = np.tile(
                np.add(np.tile(ikle2, 2), np.repeat(npoin2 * np.arange(2), ndp2)),
                (nplan - 1, 1),
            )
            _tmp1 += _tmp2
            # Update IKLE table
            output_header.ikle = np.ravel(_tmp1) + 1  # 1-indexed

            # Updated node coordinates
            output_header.x_stored = np.tile(bor_xy[:, 0], nplan)
            output_header.y_stored = np.tile(bor_xy[:, 1], nplan)

            # Number of boundary nodes in 3D
            output_header.nb_nodes = npoin3

            # Number of boundary elements in 3D
            output_header.nb_elements = nelem2 * (output_header.nb_planes - 1)
            output_header.nb_nodes_per_elem = ndp3  # 4

            # Update IPOBO
            ipob3 = np.ravel(
                np.add(
                    np.repeat(bor, nplan).reshape((npoin2, nplan)),
                    npoin2 * np.arange(nplan),
                ).T
            )
            output_header.ipobo = ipob3 + 1  # 1-Indexed

            # Write the header
            print("\n")
            print("-----------------------OUTPUT-----------------------------")
            print("----------------------------------------------------------\n")
            print(output_header.summary())
            print(output_header.var_IDs)
            print("\n")
            resout.write_header(output_header)

            resin.get_time()
            times = np.array(resin.time)

            timestep = 3600 * 6  # CMEMS data is 6 hourly

            # For all timesteps in CMEMs file
            for time_index in tqdm(
                range(len(times)), ncols=75, desc="Writing time frames..."
            ):
                # time = times[time_index]

                data_at_boundary = []
                # For all variables extract at the boundary nodes
                for j, var_ID in enumerate(output_header.var_IDs):

                    data = resin.read_var_in_frame_as_3d(time_index, var_ID)
                    # Layers x number of boundary points
                    _tmp = data[:, bor]
                    _tmp = np.ravel(_tmp)[
                        :, None
                    ]  # this extra dim is necessary for later tranpose
                    data_at_boundary.append(_tmp)
                data_at_boundary = np.asarray(data_at_boundary)
                # variables, then for each boundary node, the values at the N layers
                # [var0,..,varN][bouNode0@Layer0,..,bouNode0@LayerN,boudNode1@Layer0,..,bouNode1@LayerN,..,bouNodeN@LayerN]
                reogr_data_at_boundary = np.zeros((nvar, len(bor) * nplan))
                for vid in range(nvar):
                    counter = 0
                    for ly in range(nplan):
                        for k in range(len(bor)):
                            reogr_data_at_boundary[vid, counter] = data_at_boundary[
                                vid
                            ][ly * len(bor) + k]
                            counter += 1
             
                resout.write_entire_frame(
                    output_header, time_index * timestep, reogr_data_at_boundary
                )


if __name__ == "__main__":
    mesh_file = "./PCCA_tmac_v4.1.slf"
    cli_file = "./PCCA_tmac_v4.1.UVH.cli"
    cmems_file = "./CMEM_Jul_Oct2020_PCCA_10lyrs_v4.1.slf"
    output_file = "./KJR_Offshore_BC_PCCA_v4.1.slf"

    make_3d_boundary(mesh_file, cmems_file, cli_file, output_name=output_file)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Projects
None yet
Development

No branches or pull requests

2 participants