From 69ceec60e9773d774b347a54a9f0f6a0d013d9d5 Mon Sep 17 00:00:00 2001 From: Krishna Iyer Easwaran Date: Wed, 24 Jan 2024 16:27:37 +0530 Subject: [PATCH 1/8] util: Add wanesy boilerplate --- cmd/root.go | 2 + cmd/wanesy/wanesy.go | 27 +++++ pkg/source/wanesy/config.go | 132 ++++++++++++++++++++++ pkg/source/wanesy/messages.go | 166 +++++++++++++++++++++++++++ pkg/source/wanesy/source.go | 204 ++++++++++++++++++++++++++++++++++ pkg/source/wanesy/wanesy.go | 43 +++++++ 6 files changed, 574 insertions(+) create mode 100644 cmd/wanesy/wanesy.go create mode 100644 pkg/source/wanesy/config.go create mode 100644 pkg/source/wanesy/messages.go create mode 100644 pkg/source/wanesy/source.go create mode 100644 pkg/source/wanesy/wanesy.go diff --git a/cmd/root.go b/cmd/root.go index 05750a4..5465583 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -23,6 +23,7 @@ import ( "go.thethings.network/lorawan-stack-migrate/cmd/firefly" "go.thethings.network/lorawan-stack-migrate/cmd/ttnv2" "go.thethings.network/lorawan-stack-migrate/cmd/tts" + "go.thethings.network/lorawan-stack-migrate/cmd/wanesy" "go.thethings.network/lorawan-stack-migrate/pkg/export" "go.thethings.network/lorawan-stack-migrate/pkg/source" ) @@ -81,4 +82,5 @@ func init() { rootCmd.AddCommand(tts.TTSCmd) rootCmd.AddCommand(chirpstack.ChirpStackCmd) rootCmd.AddCommand(firefly.FireflyCmd) + rootCmd.AddCommand(wanesy.WanesyCmd) } diff --git a/cmd/wanesy/wanesy.go b/cmd/wanesy/wanesy.go new file mode 100644 index 0000000..66e15f1 --- /dev/null +++ b/cmd/wanesy/wanesy.go @@ -0,0 +1,27 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wanesy + +import ( + "go.thethings.network/lorawan-stack-migrate/pkg/commands" + _ "go.thethings.network/lorawan-stack-migrate/pkg/source/wanesy" +) + +const sourceName = "wanesy" + +// WanesyCmd represents the Wanesy source. +var WanesyCmd = commands.Source(sourceName, + "Migrate from Wanesy Management Center", +) diff --git a/pkg/source/wanesy/config.go b/pkg/source/wanesy/config.go new file mode 100644 index 0000000..0f7cda2 --- /dev/null +++ b/pkg/source/wanesy/config.go @@ -0,0 +1,132 @@ +// Copyright © 2024 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wanesy + +import ( + "encoding/csv" + "encoding/json" + "net/http" + "os" + "strings" + + "github.com/spf13/pflag" + + "go.thethings.network/lorawan-stack-migrate/pkg/source" + "go.thethings.network/lorawan-stack/v3/pkg/fetch" + "go.thethings.network/lorawan-stack/v3/pkg/frequencyplans" + "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" +) + +type Config struct { + src source.Config + + appID string + frequencyPlanID string + csvPath string + + derivedMacVersion ttnpb.MACVersion + derivedPhyVersion ttnpb.PHYVersion + + flags *pflag.FlagSet + fpStore *frequencyplans.Store +} + +// NewConfig returns a new Wanesy configuration. +func NewConfig() *Config { + config := &Config{ + flags: &pflag.FlagSet{}, + } + config.flags.StringVar(&config.frequencyPlanID, + "frequency-plan-id", + "", + "Frequency Plan ID for the exported devices") + config.flags.StringVar(&config.appID, + "app-id", + "", + "Application ID for the exported devices") + config.flags.StringVar(&config.csvPath, + "csv-path", + "", + "Path to the CSV file exported from Wanesy Management Center") + return config +} + +// Initialize the configuration. +func (c *Config) Initialize(src source.Config) error { + c.src = src + + if c.appID = os.Getenv("APP_ID"); c.appID == "" { + return errNoAppID.New() + } + if c.frequencyPlanID = os.Getenv("FREQUENCY_PLAN_ID"); c.frequencyPlanID == "" { + return errNoFrequencyPlanID.New() + } + if c.csvPath = os.Getenv("CSV_PATH"); c.csvPath == "" { + return errNoCSVFileProvided.New() + } + + fpFetcher, err := fetch.FromHTTP(http.DefaultClient, src.FrequencyPlansURL) + if err != nil { + return err + } + c.fpStore = frequencyplans.NewStore(fpFetcher) + + return nil +} + +// Flags returns the flags for the configuration. +func (c *Config) Flags() *pflag.FlagSet { + return c.flags +} + +// ImportDevices imports the devices from the provided CSV file. +func (c *Config) ImportDevices() (Devices, error) { + raw, err := os.ReadFile(c.csvPath) + if err != nil { + return nil, err + } + reader := csv.NewReader(strings.NewReader(string(raw))) + readValues, err := reader.ReadAll() + if err != nil { + return nil, err + } + if len(readValues) < 2 { + return nil, errInvalidCSV.New() + } + values := make([]map[string]string, 0) + for i := 1; i < len(readValues); i++ { + keys := readValues[0] + value := make(map[string]string) + for j := 0; j < len(keys); j++ { + noOfcolumns := len(readValues[i]) + if j >= noOfcolumns { + value[keys[j]] = "" // Fill empty columns. + continue + } + value[keys[j]] = readValues[i][j] + } + values = append(values, value) + } + j, err := json.Marshal(values) + if err != nil { + return nil, err + } + devices := make(Devices) + err = devices.UnmarshalJSON(j) + if err != nil { + return nil, err + } + return devices, nil +} diff --git a/pkg/source/wanesy/messages.go b/pkg/source/wanesy/messages.go new file mode 100644 index 0000000..22c7ab7 --- /dev/null +++ b/pkg/source/wanesy/messages.go @@ -0,0 +1,166 @@ +// Copyright © 2024 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wanesy + +import ( + "encoding/json" + "strconv" + + "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" + "go.thethings.network/lorawan-stack/v3/pkg/types" +) + +// Devices is a list of devices. +type Devices map[types.EUI64]Device + +// UnmarshalJSON implements json.Unmarshaler. +func (d Devices) UnmarshalJSON(b []byte) error { + var devs []Device + if err := json.Unmarshal(b, &devs); err != nil { + return err + } + for _, dev := range devs { + var devEUI types.EUI64 + if err := devEUI.UnmarshalText([]byte(dev.DevEui)); err != nil { + return err + } + d[devEUI] = dev + } + return nil +} + +// Device is a LoRaWAN end device exported from Wanesy Management Center. +type Device struct { + Activation string `json:"activation"` + AdrEnabled string `json:"adrEnabled"` + Altitude string `json:"altitude"` + AppEui string `json:"appEui"` + AppKey string `json:"appKey"` + AppSKey string `json:"appSKey"` + CfList string `json:"cfList"` + ClassType string `json:"classType"` + ClusterID string `json:"clusterId"` + ClusterName string `json:"clusterName"` + Country string `json:"country"` + DevAddr string `json:"devAddr"` + DevEui string `json:"devEui"` + DevNonceCounter string `json:"devNonceCounter"` + DwellTime string `json:"dwellTime"` + FNwkSIntKey string `json:"fNwkSIntKey"` + FcntDown string `json:"fcntDown"` + FcntUp string `json:"fcntUp"` + Geolocation string `json:"geolocation"` + LastDataDownMessage string `json:"lastDataDownMessage"` + LastDataUpDr string `json:"lastDataUpDr"` + LastDataUpMessage string `json:"lastDataUpMessage"` + Latitude string `json:"latitude"` + Longitude string `json:"longitude"` + MacVersion string `json:"macVersion"` + Name string `json:"name"` + NwkSKey string `json:"nwkSKey"` + PingSlotDr string `json:"pingSlotDr"` + PingSlotFreq string `json:"pingSlotFreq"` + Profile string `json:"profile"` + RegParamsRevision string `json:"regParamsRevision"` + RfRegion string `json:"rfRegion"` + Rx1Delay string `json:"rx1Delay"` + Rx1DrOffset string `json:"rx1DrOffset"` + Rx2Dr string `json:"rx2Dr"` + Rx2Freq string `json:"rx2Freq"` + RxWindows string `json:"rxWindows"` + SNwkSIntKey string `json:"sNwkSIntKey"` + Status string `json:"status"` +} + +// EndDevice converts a Wanesy device to a TTS device. +func (d Device) EndDevice(applicationID, frequencyPlanID string) (*ttnpb.EndDevice, error) { + var devEUI, joinEUI types.EUI64 + if err := devEUI.UnmarshalText([]byte(d.DevEui)); err != nil { + return nil, err + } + if err := joinEUI.UnmarshalText([]byte(d.AppEui)); err != nil { + return nil, err + } + ret := &ttnpb.EndDevice{ + Name: d.Name, + FrequencyPlanId: frequencyPlanID, + Ids: &ttnpb.EndDeviceIdentifiers{ + ApplicationIds: &ttnpb.ApplicationIdentifiers{ApplicationId: applicationID}, + DevEui: devEUI.Bytes(), + JoinEui: joinEUI.Bytes(), + }, + MacSettings: &ttnpb.MACSettings{}, + SupportsClassC: d.ClassType == "C", + SupportsClassB: d.ClassType == "B", + SupportsJoin: d.Activation == "OTAA", + } + if d.Rx2Dr != "NULL" { + s, err := strconv.ParseUint(d.Rx2Dr, 16, 32) + if err != nil { + return nil, err + } + ret.MacSettings.DesiredRx2DataRateIndex = &ttnpb.DataRateIndexValue{ + Value: ttnpb.DataRateIndex(int32(s)), + } + } + switch d.MacVersion { + case "1.0.0": + ret.LorawanVersion = ttnpb.MACVersion_MAC_V1_0 + ret.LorawanPhyVersion = ttnpb.PHYVersion_TS001_V1_0 + case "1.0.1": + ret.LorawanVersion = ttnpb.MACVersion_MAC_V1_0_1 + ret.LorawanPhyVersion = ttnpb.PHYVersion_TS001_V1_0_1 + case "1.0.2": + ret.LorawanVersion = ttnpb.MACVersion_MAC_V1_0_2 + switch d.RegParamsRevision { + case "A": + ret.LorawanPhyVersion = ttnpb.PHYVersion_RP001_V1_0_2 + case "B": + ret.LorawanPhyVersion = ttnpb.PHYVersion_RP001_V1_0_2_REV_B + default: + return nil, errInvalidPHYForMACVersion.WithAttributes( + "phy_version", + d.RegParamsRevision, + "mac_version", + d.MacVersion, + ) + } + case "1.0.3": + ret.LorawanVersion = ttnpb.MACVersion_MAC_V1_0_3 + ret.LorawanPhyVersion = ttnpb.PHYVersion_RP001_V1_0_3_REV_A + case "1.0.4": + ret.LorawanVersion = ttnpb.MACVersion_MAC_V1_0_4 + ret.LorawanPhyVersion = ttnpb.PHYVersion_RP002_V1_0_4 + case "1.1.0": + ret.LorawanVersion = ttnpb.MACVersion_MAC_V1_1 + switch d.RegParamsRevision { + case "A": + ret.LorawanPhyVersion = ttnpb.PHYVersion_RP001_V1_1_REV_A + case "B": + ret.LorawanPhyVersion = ttnpb.PHYVersion_RP001_V1_1_REV_B + default: + return nil, errInvalidPHYForMACVersion.WithAttributes( + "phy_version", + d.RegParamsRevision, + "mac_version", + d.MacVersion, + ) + } + default: + return nil, errInvalidMACVersion.WithAttributes("mac_version", d.MacVersion) + } + + return ret, nil +} diff --git a/pkg/source/wanesy/source.go b/pkg/source/wanesy/source.go new file mode 100644 index 0000000..00b750b --- /dev/null +++ b/pkg/source/wanesy/source.go @@ -0,0 +1,204 @@ +// Copyright © 2024 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wanesy + +import ( + "context" + + "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" + "go.thethings.network/lorawan-stack/v3/pkg/types" + + "go.thethings.network/lorawan-stack-migrate/pkg/iterator" + "go.thethings.network/lorawan-stack-migrate/pkg/source" + "go.thethings.network/lorawan-stack-migrate/pkg/source/firefly/client" +) + +type Source struct { + *Config + *client.Client + + imported Devices +} + +func createNewSource(cfg *Config) source.CreateSource { + return func(ctx context.Context, src source.Config) (source.Source, error) { + if err := cfg.Initialize(src); err != nil { + return nil, err + } + devs, err := cfg.ImportDevices() + if err != nil { + return nil, err + } + return Source{ + Config: cfg, + imported: devs, + }, nil + } +} + +// Iterator implements source.Source. +func (s Source) Iterator(isApplication bool) iterator.Iterator { + // if !isApplication { + // return iterator.NewReaderIterator(os.Stdin, '\n') + // } + // if s.all { + // // The Firefly LNS does not group devices by an application. + // // When the "all" flag is set, we get all devices accessible by the API key. + // // We use a dummy "all" App ID to fallthrough to the RangeDevices method, + // // where the appID argument is unused. + // return iterator.NewListIterator( + // []string{"all"}, + // ) + // } + return iterator.NewNoopIterator() +} + +// ExportDevice implements the source.Source interface. +func (s Source) ExportDevice(devEUIString string) (*ttnpb.EndDevice, error) { + var devEUI, joinEUI types.EUI64 + if err := devEUI.UnmarshalText([]byte(devEUIString)); err != nil { + return nil, err + } + wmcdev, ok := s.imported[devEUI] + if !ok { + return nil, errNoDeviceFound.WithAttributes("eui", devEUIString) + } + + if err := joinEUI.UnmarshalText([]byte(wmcdev.AppEui)); err != nil { + return nil, err + } + v3dev, err := wmcdev.EndDevice(s.appID, s.frequencyPlanID) + if err != nil { + return nil, err + } + // v3dev := &ttnpb.EndDevice{ + // Name: wmcdev.Name, + // FrequencyPlanId: s.frequencyPlanID, + // Ids: &ttnpb.EndDeviceIdentifiers{ + // ApplicationIds: &ttnpb.ApplicationIdentifiers{ApplicationId: s.appID}, + // DevEui: devEUI.Bytes(), + // JoinEui: joinEUI.Bytes(), + // }, + // MacSettings: &ttnpb.MACSettings{ + // DesiredRx2DataRateIndex: &ttnpb.DataRateIndexValue{Value: ttnpb.DataRateIndex(wmcdev.Rx2Dr)}, + // }, + // SupportsClassC: wmcdev.ClassC, + // SupportsJoin: wmcdev.OTAA, + // LorawanVersion: s.derivedMacVersion, + // LorawanPhyVersion: s.derivedPhyVersion, + // } + // if wmcdev.Location != nil { + // v3dev.Locations = map[string]*ttnpb.Location{ + // "user": { + // Latitude: wmcdev.Location.Latitude, + // Longitude: wmcdev.Location.Longitude, + // Source: ttnpb.LocationSource_SOURCE_REGISTRY, + // }, + // } + // s.src.Logger.Debugw("Set location", "location", v3dev.Locations) + // } + // if v3dev.SupportsJoin { + // v3dev.RootKeys = &ttnpb.RootKeys{AppKey: &ttnpb.KeyEnvelope{}} + // v3dev.RootKeys.AppKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, wmcdev.ApplicationKey) + // if err != nil { + // return nil, err + // } + // } + // hasSession := wmcdev.Address != "" && wmcdev.NetworkSessionKey != "" && wmcdev.ApplicationSessionKey != "" + + // if hasSession || !v3dev.SupportsJoin { + // v3dev.Session = &ttnpb.Session{Keys: &ttnpb.SessionKeys{AppSKey: &ttnpb.KeyEnvelope{}, FNwkSIntKey: &ttnpb.KeyEnvelope{}}} + // v3dev.Session.DevAddr, err = util.UnmarshalTextToBytes(&types.DevAddr{}, wmcdev.Address) + // if err != nil { + // return nil, err + // } + // if v3dev.SupportsJoin { + // v3dev.Session.StartedAt = timestamppb.Now() + // v3dev.Session.Keys.SessionKeyId = random.Bytes(16) + // } + // v3dev.Session.Keys.AppSKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, wmcdev.ApplicationSessionKey) + // if err != nil { + // return nil, err + // } + // v3dev.Session.Keys.FNwkSIntKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, wmcdev.NetworkSessionKey) + // if err != nil { + // return nil, err + // } + // switch v3dev.LorawanVersion { + // case ttnpb.MACVersion_MAC_V1_1: + // v3dev.Session.Keys.NwkSEncKey = &ttnpb.KeyEnvelope{} + // v3dev.Session.Keys.NwkSEncKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, wmcdev.ApplicationSessionKey) + // if err != nil { + // return nil, err + // } + // v3dev.Session.Keys.SNwkSIntKey = &ttnpb.KeyEnvelope{} + // v3dev.Session.Keys.SNwkSIntKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, wmcdev.NetworkSessionKey) + // if err != nil { + // return nil, err + // } + // } + + // // Set FrameCounters + // packet, err := s.GetLastPacket(devEUIString) + // if err != nil { + // return nil, err + // } + // v3dev.Session.LastFCntUp = uint32(packet.FCnt) + // v3dev.Session.LastAFCntDown = uint32(wmcdev.FrameCounter) + // v3dev.Session.LastNFCntDown = uint32(wmcdev.FrameCounter) + + // // Create a MACState. + // if v3dev.MacState, err = mac.NewState(v3dev, s.fpStore, &ttnpb.MACSettings{}); err != nil { + // return nil, err + // } + // v3dev.MacState.CurrentParameters = v3dev.MacState.DesiredParameters + // v3dev.MacState.CurrentParameters.Rx1Delay = ttnpb.RxDelay_RX_DELAY_1 + // } + + // if s.invalidateKeys { + // s.src.Logger.Debugw("Increment the last byte of the device keys", "device_id", wmcdev.Name, "device_eui", wmcdev.EUI) + // // Increment the last byte of the device keys. + // // This makes it easier to rollback a migration if needed. + // updated := wmcdev.WithIncrementedKeys() + // err := s.UpdateDeviceByEUI(devEUIString, updated) + // if err != nil { + // return nil, err + // } + // } + + return v3dev, nil +} + +// RangeDevices implements the source.Source interface. +func (s Source) RangeDevices(_ string, f func(source.Source, string) error) error { + var ( + devs []client.Device + err error + ) + s.src.Logger.Debugw("Firefly LNS does not group devices by an application. Get all devices accessible by the API key") + devs, err = s.GetAllDevices() + if err != nil { + return err + } + for _, d := range devs { + if err := f(s, d.EUI); err != nil { + return err + } + } + return nil +} + +// Close implements the Source interface. +func (s Source) Close() error { return nil } diff --git a/pkg/source/wanesy/wanesy.go b/pkg/source/wanesy/wanesy.go new file mode 100644 index 0000000..e0b1c46 --- /dev/null +++ b/pkg/source/wanesy/wanesy.go @@ -0,0 +1,43 @@ +// Copyright © 2024 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package wanesy provides functions to parse the WMC csv file and create a TTS device JSON. +package wanesy + +import ( + "go.thethings.network/lorawan-stack-migrate/pkg/source" + "go.thethings.network/lorawan-stack/v3/pkg/errors" +) + +var ( + errInvalidCSV = errors.DefineInvalidArgument("invalid_csv", "invalid csv file") + errNoAppID = errors.DefineInvalidArgument("no_app_id", "no app id") + errNoCSVFileProvided = errors.DefineInvalidArgument("no_csv_file_provided", "no csv file provided") + errNoJoinEUI = errors.DefineInvalidArgument("no_join_eui", "no join eui") + errNoDeviceFound = errors.DefineInvalidArgument("no_device_found", "no device with eui `{eui}` found") + errNoFrequencyPlanID = errors.DefineInvalidArgument("no_frequency_plan_id", "no frequency plan ID") + errInvalidMACVersion = errors.DefineInvalidArgument("invalid_mac_version", "invalid MAC version `{mac_version}`") + errInvalidPHYForMACVersion = errors.DefineInvalidArgument("invalid_phy_for_mac_version", "invalid PHY version `{phy_version}` for MAC version `{mac_version}`") +) + +func init() { + cfg := NewConfig() + + source.RegisterSource(source.Registration{ + Name: "wanesy", + Description: "Migrate from Wanesy Management Center", + FlagSet: cfg.Flags(), + Create: createNewSource(cfg), + }) +} From ed4e5479c41c7d2f7d08f3dd1071fb40cb6ee21a Mon Sep 17 00:00:00 2001 From: Krishna Iyer Easwaran Date: Wed, 24 Jan 2024 16:51:49 +0530 Subject: [PATCH 2/8] util: Complete device --- pkg/source/wanesy/messages.go | 96 +++++++++++++++++++++++++++++++++- pkg/source/wanesy/source.go | 97 +---------------------------------- 2 files changed, 96 insertions(+), 97 deletions(-) diff --git a/pkg/source/wanesy/messages.go b/pkg/source/wanesy/messages.go index 22c7ab7..99c4316 100644 --- a/pkg/source/wanesy/messages.go +++ b/pkg/source/wanesy/messages.go @@ -18,8 +18,13 @@ import ( "encoding/json" "strconv" + "github.com/TheThingsNetwork/go-utils/random" + "go.thethings.network/lorawan-stack-migrate/pkg/util" + "go.thethings.network/lorawan-stack/v3/pkg/frequencyplans" + "go.thethings.network/lorawan-stack/v3/pkg/networkserver/mac" "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" "go.thethings.network/lorawan-stack/v3/pkg/types" + "google.golang.org/protobuf/types/known/timestamppb" ) // Devices is a list of devices. @@ -85,7 +90,7 @@ type Device struct { } // EndDevice converts a Wanesy device to a TTS device. -func (d Device) EndDevice(applicationID, frequencyPlanID string) (*ttnpb.EndDevice, error) { +func (d Device) EndDevice(fpStore *frequencyplans.Store, applicationID, frequencyPlanID string) (*ttnpb.EndDevice, error) { var devEUI, joinEUI types.EUI64 if err := devEUI.UnmarshalText([]byte(d.DevEui)); err != nil { return nil, err @@ -162,5 +167,94 @@ func (d Device) EndDevice(applicationID, frequencyPlanID string) (*ttnpb.EndDevi return nil, errInvalidMACVersion.WithAttributes("mac_version", d.MacVersion) } + if d.Longitude != "NULL" && d.Latitude != "NULL" && d.Altitude != "NULL" { + latitude, _ := strconv.ParseFloat(d.Latitude, 64) + longitude, _ := strconv.ParseFloat(d.Longitude, 64) + altitude, err := strconv.ParseUint(d.Rx2Dr, 16, 32) + if err != nil { + return nil, err + } + ret.Locations = map[string]*ttnpb.Location{ + "user": { + Latitude: latitude, + Longitude: longitude, + Altitude: int32(altitude), + Source: ttnpb.LocationSource_SOURCE_REGISTRY, + }, + } + } + if ret.SupportsJoin { + var err error + ret.RootKeys = &ttnpb.RootKeys{AppKey: &ttnpb.KeyEnvelope{}} + ret.RootKeys.AppKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, d.AppKey) + if err != nil { + return nil, err + } + } + + // Copy session information if available. + hasSession := d.DevAddr != "NULL" && d.NwkSKey != "NULL" && d.AppKey != "" + if hasSession || !ret.SupportsJoin { + var err error + ret.Session = &ttnpb.Session{Keys: &ttnpb.SessionKeys{AppSKey: &ttnpb.KeyEnvelope{}, FNwkSIntKey: &ttnpb.KeyEnvelope{}}} + ret.Session.DevAddr, err = util.UnmarshalTextToBytes(&types.DevAddr{}, d.DevAddr) + if err != nil { + return nil, err + } + if ret.SupportsJoin { + ret.Session.StartedAt = timestamppb.Now() + ret.Session.Keys.SessionKeyId = random.Bytes(16) + } + ret.Session.Keys.AppSKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, d.AppSKey) + if err != nil { + return nil, err + } + ret.Session.Keys.FNwkSIntKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, d.NwkSKey) + if err != nil { + return nil, err + } + switch ret.LorawanVersion { + case ttnpb.MACVersion_MAC_V1_1: + ret.Session.Keys.FNwkSIntKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, d.FNwkSIntKey) + if err != nil { + return nil, err + } + ret.Session.Keys.NwkSEncKey = &ttnpb.KeyEnvelope{} + ret.Session.Keys.NwkSEncKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, d.NwkSKey) + if err != nil { + return nil, err + } + ret.Session.Keys.SNwkSIntKey = &ttnpb.KeyEnvelope{} + ret.Session.Keys.SNwkSIntKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, d.SNwkSIntKey) + if err != nil { + return nil, err + } + } + + // Set FrameCounters + s, err := strconv.ParseUint(d.FcntUp, 16, 32) + if err != nil { + return nil, err + } + ret.Session.LastFCntUp = uint32(s) + s, err = strconv.ParseUint(d.FcntDown, 16, 32) + if err != nil { + return nil, err + } + ret.Session.LastAFCntDown = uint32(s) + ret.Session.LastNFCntDown = uint32(s) + + // Create a MACState. + if ret.MacState, err = mac.NewState(ret, fpStore, ret.MacSettings); err != nil { + return nil, err + } + ret.MacState.CurrentParameters = ret.MacState.DesiredParameters + s, err = strconv.ParseUint(d.Rx1Delay, 16, 32) + if err != nil { + return nil, err + } + ret.MacState.CurrentParameters.Rx1Delay = ttnpb.RxDelay(s) + } + return ret, nil } diff --git a/pkg/source/wanesy/source.go b/pkg/source/wanesy/source.go index 00b750b..26f250d 100644 --- a/pkg/source/wanesy/source.go +++ b/pkg/source/wanesy/source.go @@ -79,105 +79,10 @@ func (s Source) ExportDevice(devEUIString string) (*ttnpb.EndDevice, error) { if err := joinEUI.UnmarshalText([]byte(wmcdev.AppEui)); err != nil { return nil, err } - v3dev, err := wmcdev.EndDevice(s.appID, s.frequencyPlanID) + v3dev, err := wmcdev.EndDevice(s.fpStore, s.appID, s.frequencyPlanID) if err != nil { return nil, err } - // v3dev := &ttnpb.EndDevice{ - // Name: wmcdev.Name, - // FrequencyPlanId: s.frequencyPlanID, - // Ids: &ttnpb.EndDeviceIdentifiers{ - // ApplicationIds: &ttnpb.ApplicationIdentifiers{ApplicationId: s.appID}, - // DevEui: devEUI.Bytes(), - // JoinEui: joinEUI.Bytes(), - // }, - // MacSettings: &ttnpb.MACSettings{ - // DesiredRx2DataRateIndex: &ttnpb.DataRateIndexValue{Value: ttnpb.DataRateIndex(wmcdev.Rx2Dr)}, - // }, - // SupportsClassC: wmcdev.ClassC, - // SupportsJoin: wmcdev.OTAA, - // LorawanVersion: s.derivedMacVersion, - // LorawanPhyVersion: s.derivedPhyVersion, - // } - // if wmcdev.Location != nil { - // v3dev.Locations = map[string]*ttnpb.Location{ - // "user": { - // Latitude: wmcdev.Location.Latitude, - // Longitude: wmcdev.Location.Longitude, - // Source: ttnpb.LocationSource_SOURCE_REGISTRY, - // }, - // } - // s.src.Logger.Debugw("Set location", "location", v3dev.Locations) - // } - // if v3dev.SupportsJoin { - // v3dev.RootKeys = &ttnpb.RootKeys{AppKey: &ttnpb.KeyEnvelope{}} - // v3dev.RootKeys.AppKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, wmcdev.ApplicationKey) - // if err != nil { - // return nil, err - // } - // } - // hasSession := wmcdev.Address != "" && wmcdev.NetworkSessionKey != "" && wmcdev.ApplicationSessionKey != "" - - // if hasSession || !v3dev.SupportsJoin { - // v3dev.Session = &ttnpb.Session{Keys: &ttnpb.SessionKeys{AppSKey: &ttnpb.KeyEnvelope{}, FNwkSIntKey: &ttnpb.KeyEnvelope{}}} - // v3dev.Session.DevAddr, err = util.UnmarshalTextToBytes(&types.DevAddr{}, wmcdev.Address) - // if err != nil { - // return nil, err - // } - // if v3dev.SupportsJoin { - // v3dev.Session.StartedAt = timestamppb.Now() - // v3dev.Session.Keys.SessionKeyId = random.Bytes(16) - // } - // v3dev.Session.Keys.AppSKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, wmcdev.ApplicationSessionKey) - // if err != nil { - // return nil, err - // } - // v3dev.Session.Keys.FNwkSIntKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, wmcdev.NetworkSessionKey) - // if err != nil { - // return nil, err - // } - // switch v3dev.LorawanVersion { - // case ttnpb.MACVersion_MAC_V1_1: - // v3dev.Session.Keys.NwkSEncKey = &ttnpb.KeyEnvelope{} - // v3dev.Session.Keys.NwkSEncKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, wmcdev.ApplicationSessionKey) - // if err != nil { - // return nil, err - // } - // v3dev.Session.Keys.SNwkSIntKey = &ttnpb.KeyEnvelope{} - // v3dev.Session.Keys.SNwkSIntKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, wmcdev.NetworkSessionKey) - // if err != nil { - // return nil, err - // } - // } - - // // Set FrameCounters - // packet, err := s.GetLastPacket(devEUIString) - // if err != nil { - // return nil, err - // } - // v3dev.Session.LastFCntUp = uint32(packet.FCnt) - // v3dev.Session.LastAFCntDown = uint32(wmcdev.FrameCounter) - // v3dev.Session.LastNFCntDown = uint32(wmcdev.FrameCounter) - - // // Create a MACState. - // if v3dev.MacState, err = mac.NewState(v3dev, s.fpStore, &ttnpb.MACSettings{}); err != nil { - // return nil, err - // } - // v3dev.MacState.CurrentParameters = v3dev.MacState.DesiredParameters - // v3dev.MacState.CurrentParameters.Rx1Delay = ttnpb.RxDelay_RX_DELAY_1 - // } - - // if s.invalidateKeys { - // s.src.Logger.Debugw("Increment the last byte of the device keys", "device_id", wmcdev.Name, "device_eui", wmcdev.EUI) - // // Increment the last byte of the device keys. - // // This makes it easier to rollback a migration if needed. - // updated := wmcdev.WithIncrementedKeys() - // err := s.UpdateDeviceByEUI(devEUIString, updated) - // if err != nil { - // return nil, err - // } - // } - return v3dev, nil } From 777a52efe703e09c6514c04b73b8cc45712a73e2 Mon Sep 17 00:00:00 2001 From: Krishna Iyer Easwaran Date: Wed, 24 Jan 2024 16:58:02 +0530 Subject: [PATCH 3/8] util: Export all from CSV --- pkg/source/wanesy/config.go | 5 +++++ pkg/source/wanesy/source.go | 36 +++++++++++++----------------------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/pkg/source/wanesy/config.go b/pkg/source/wanesy/config.go index 0f7cda2..3b38820 100644 --- a/pkg/source/wanesy/config.go +++ b/pkg/source/wanesy/config.go @@ -35,6 +35,7 @@ type Config struct { appID string frequencyPlanID string csvPath string + all bool derivedMacVersion ttnpb.MACVersion derivedPhyVersion ttnpb.PHYVersion @@ -60,6 +61,10 @@ func NewConfig() *Config { "csv-path", "", "Path to the CSV file exported from Wanesy Management Center") + config.flags.BoolVar(&config.all, + "all", + false, + "Export all devices in the CSV. This is only used by the application command") return config } diff --git a/pkg/source/wanesy/source.go b/pkg/source/wanesy/source.go index 26f250d..0f43f03 100644 --- a/pkg/source/wanesy/source.go +++ b/pkg/source/wanesy/source.go @@ -16,6 +16,7 @@ package wanesy import ( "context" + "os" "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" "go.thethings.network/lorawan-stack/v3/pkg/types" @@ -50,18 +51,16 @@ func createNewSource(cfg *Config) source.CreateSource { // Iterator implements source.Source. func (s Source) Iterator(isApplication bool) iterator.Iterator { - // if !isApplication { - // return iterator.NewReaderIterator(os.Stdin, '\n') - // } - // if s.all { - // // The Firefly LNS does not group devices by an application. - // // When the "all" flag is set, we get all devices accessible by the API key. - // // We use a dummy "all" App ID to fallthrough to the RangeDevices method, - // // where the appID argument is unused. - // return iterator.NewListIterator( - // []string{"all"}, - // ) - // } + if !isApplication { + return iterator.NewReaderIterator(os.Stdin, '\n') + } + if s.all { + // WMC does not group devices by application. + // The `all` flag is used to export all the devices in the CSV. + return iterator.NewListIterator( + []string{"all"}, + ) + } return iterator.NewNoopIterator() } @@ -88,17 +87,8 @@ func (s Source) ExportDevice(devEUIString string) (*ttnpb.EndDevice, error) { // RangeDevices implements the source.Source interface. func (s Source) RangeDevices(_ string, f func(source.Source, string) error) error { - var ( - devs []client.Device - err error - ) - s.src.Logger.Debugw("Firefly LNS does not group devices by an application. Get all devices accessible by the API key") - devs, err = s.GetAllDevices() - if err != nil { - return err - } - for _, d := range devs { - if err := f(s, d.EUI); err != nil { + for eui := range s.imported { + if err := f(s, eui.String()); err != nil { return err } } From d682e70a54b6576c424f05cbc6e131a626df54b0 Mon Sep 17 00:00:00 2001 From: Krishna Iyer Easwaran Date: Fri, 26 Jan 2024 13:38:29 +0530 Subject: [PATCH 4/8] util: Require JoinEUI only for OTAA --- pkg/source/wanesy/messages.go | 40 ++++++++++++++++++++--------------- pkg/source/wanesy/source.go | 6 +----- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/pkg/source/wanesy/messages.go b/pkg/source/wanesy/messages.go index 99c4316..a528cf9 100644 --- a/pkg/source/wanesy/messages.go +++ b/pkg/source/wanesy/messages.go @@ -91,26 +91,37 @@ type Device struct { // EndDevice converts a Wanesy device to a TTS device. func (d Device) EndDevice(fpStore *frequencyplans.Store, applicationID, frequencyPlanID string) (*ttnpb.EndDevice, error) { - var devEUI, joinEUI types.EUI64 + var devEUI types.EUI64 if err := devEUI.UnmarshalText([]byte(d.DevEui)); err != nil { return nil, err } - if err := joinEUI.UnmarshalText([]byte(d.AppEui)); err != nil { - return nil, err - } ret := &ttnpb.EndDevice{ Name: d.Name, FrequencyPlanId: frequencyPlanID, Ids: &ttnpb.EndDeviceIdentifiers{ ApplicationIds: &ttnpb.ApplicationIdentifiers{ApplicationId: applicationID}, DevEui: devEUI.Bytes(), - JoinEui: joinEUI.Bytes(), }, MacSettings: &ttnpb.MACSettings{}, SupportsClassC: d.ClassType == "C", SupportsClassB: d.ClassType == "B", SupportsJoin: d.Activation == "OTAA", } + if ret.SupportsJoin { + var ( + joinEUI types.EUI64 + err error + ) + ret.RootKeys = &ttnpb.RootKeys{AppKey: &ttnpb.KeyEnvelope{}} + ret.RootKeys.AppKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, d.AppKey) + if err != nil { + return nil, err + } + if err := joinEUI.UnmarshalText([]byte(d.AppEui)); err != nil { + return nil, err + } + ret.Ids.JoinEui = joinEUI.Bytes() + } if d.Rx2Dr != "NULL" { s, err := strconv.ParseUint(d.Rx2Dr, 16, 32) if err != nil { @@ -183,14 +194,6 @@ func (d Device) EndDevice(fpStore *frequencyplans.Store, applicationID, frequenc }, } } - if ret.SupportsJoin { - var err error - ret.RootKeys = &ttnpb.RootKeys{AppKey: &ttnpb.KeyEnvelope{}} - ret.RootKeys.AppKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, d.AppKey) - if err != nil { - return nil, err - } - } // Copy session information if available. hasSession := d.DevAddr != "NULL" && d.NwkSKey != "NULL" && d.AppKey != "" @@ -249,11 +252,14 @@ func (d Device) EndDevice(fpStore *frequencyplans.Store, applicationID, frequenc return nil, err } ret.MacState.CurrentParameters = ret.MacState.DesiredParameters - s, err = strconv.ParseUint(d.Rx1Delay, 16, 32) - if err != nil { - return nil, err + ret.MacState.CurrentParameters.Rx1Delay = ttnpb.RxDelay_RX_DELAY_1 // Fallback + if d.Rx1Delay != "NULL" { + s, err = strconv.ParseUint(d.Rx1Delay, 16, 32) + if err != nil { + return nil, err + } + ret.MacState.CurrentParameters.Rx1Delay = ttnpb.RxDelay(s) } - ret.MacState.CurrentParameters.Rx1Delay = ttnpb.RxDelay(s) } return ret, nil diff --git a/pkg/source/wanesy/source.go b/pkg/source/wanesy/source.go index 0f43f03..0e7a041 100644 --- a/pkg/source/wanesy/source.go +++ b/pkg/source/wanesy/source.go @@ -66,7 +66,7 @@ func (s Source) Iterator(isApplication bool) iterator.Iterator { // ExportDevice implements the source.Source interface. func (s Source) ExportDevice(devEUIString string) (*ttnpb.EndDevice, error) { - var devEUI, joinEUI types.EUI64 + var devEUI types.EUI64 if err := devEUI.UnmarshalText([]byte(devEUIString)); err != nil { return nil, err } @@ -74,10 +74,6 @@ func (s Source) ExportDevice(devEUIString string) (*ttnpb.EndDevice, error) { if !ok { return nil, errNoDeviceFound.WithAttributes("eui", devEUIString) } - - if err := joinEUI.UnmarshalText([]byte(wmcdev.AppEui)); err != nil { - return nil, err - } v3dev, err := wmcdev.EndDevice(s.fpStore, s.appID, s.frequencyPlanID) if err != nil { return nil, err From 25d3d8e19c49125b3b81348187ac2b8444bfe092 Mon Sep 17 00:00:00 2001 From: Krishna Iyer Easwaran Date: Fri, 26 Jan 2024 13:44:03 +0530 Subject: [PATCH 5/8] util: Update README.md --- README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/README.md b/README.md index 6d1ad08..8c9a815 100644 --- a/README.md +++ b/README.md @@ -307,6 +307,45 @@ $ ttn-lw-migrate firefly application --all --verbose > devices.json $ ttn-lw-migrate firefly application --all --invalidate-keys > devices.json ``` +## Wanesy + +### Configuration + +Configure with environment variables, or command-line arguments. + +See `ttn-lw-migrate wanesy {device|application} --help` for more details. + +The following example shows how to set options via environment variables. + +```bash +$ export APP_ID=my-test-app # Application ID for the exported devices +$ export FREQUENCY_PLAN_ID=EU_863_870 # Frequency Plan ID for the exported devices +$ export CSV_PATH= # Local path to the exported CSV file. +``` + +### Notes + +- The export process will halt if any error occurs. +- Since the migration tool uses a CSV file that's exported from WMC and does not interact with the API, make sure to remove/clean up the devices on WMC once the migration is completed. + +### Export Devices + +To export a single device using its Device EUI (e.g. `1111111111111112`): + +```bash +# Export a device from the CSV to TTS format. +$ ttn-lw-migrate wanesy device 1111111111111112 > devices.json +``` + +### Export All Devices + +In order to export all devices from the CSV file, use the `application` command. + +```bash +# Export all devices from the CSV. +$ ttn-lw-migrate wanesy application --all +``` + ## Development Environment Requires Go version 1.16 or higher. [Download Go](https://golang.org/dl/). From d2be08afe0ac75632f176494b7c591a83342b747 Mon Sep 17 00:00:00 2001 From: Krishna Iyer Easwaran Date: Mon, 12 Feb 2024 11:11:51 +0100 Subject: [PATCH 6/8] dev: Finalize parsing of fields --- pkg/source/wanesy/messages.go | 76 +++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/pkg/source/wanesy/messages.go b/pkg/source/wanesy/messages.go index a528cf9..623bd9d 100644 --- a/pkg/source/wanesy/messages.go +++ b/pkg/source/wanesy/messages.go @@ -16,6 +16,7 @@ package wanesy import ( "encoding/json" + "fmt" "strconv" "github.com/TheThingsNetwork/go-utils/random" @@ -41,6 +42,10 @@ func (d Devices) UnmarshalJSON(b []byte) error { if err := devEUI.UnmarshalText([]byte(dev.DevEui)); err != nil { return err } + if len(dev.DevAddr)%2 != 0 { + // Prepend 0 to make the length 8. + dev.DevAddr = fmt.Sprintf("0%s", dev.DevAddr) + } d[devEUI] = dev } return nil @@ -48,45 +53,46 @@ func (d Devices) UnmarshalJSON(b []byte) error { // Device is a LoRaWAN end device exported from Wanesy Management Center. type Device struct { - Activation string `json:"activation"` - AdrEnabled string `json:"adrEnabled"` - Altitude string `json:"altitude"` - AppEui string `json:"appEui"` - AppKey string `json:"appKey"` - AppSKey string `json:"appSKey"` - CfList string `json:"cfList"` - ClassType string `json:"classType"` + DevEui string `json:"devEui"` ClusterID string `json:"clusterId"` ClusterName string `json:"clusterName"` + Name string `json:"name"` + ClassType string `json:"classType"` + RfRegion string `json:"rfRegion"` Country string `json:"country"` - DevAddr string `json:"devAddr"` - DevEui string `json:"devEui"` - DevNonceCounter string `json:"devNonceCounter"` - DwellTime string `json:"dwellTime"` - FNwkSIntKey string `json:"fNwkSIntKey"` - FcntDown string `json:"fcntDown"` - FcntUp string `json:"fcntUp"` - Geolocation string `json:"geolocation"` - LastDataDownMessage string `json:"lastDataDownMessage"` - LastDataUpDr string `json:"lastDataUpDr"` - LastDataUpMessage string `json:"lastDataUpMessage"` - Latitude string `json:"latitude"` - Longitude string `json:"longitude"` MacVersion string `json:"macVersion"` - Name string `json:"name"` - NwkSKey string `json:"nwkSKey"` - PingSlotDr string `json:"pingSlotDr"` - PingSlotFreq string `json:"pingSlotFreq"` - Profile string `json:"profile"` RegParamsRevision string `json:"regParamsRevision"` - RfRegion string `json:"rfRegion"` + Profile string `json:"profile"` + AdrEnabled string `json:"adrEnabled"` + Activation string `json:"activation"` + AppEui string `json:"appEui"` + AppKey string `json:"appKey"` + FCntDown string `json:"fcntDown"` + FCntUp string `json:"fcntUp"` + DevNonceCounter string `json:"devNonceCounter"` + FNwkSIntKey string `json:"fNwkSIntKey"` + SNwkSIntKey string `json:"sNwkSIntKey"` Rx1Delay string `json:"rx1Delay"` Rx1DrOffset string `json:"rx1DrOffset"` Rx2Dr string `json:"rx2Dr"` Rx2Freq string `json:"rx2Freq"` RxWindows string `json:"rxWindows"` - SNwkSIntKey string `json:"sNwkSIntKey"` + CfList string `json:"cfList"` + DwellTime string `json:"dwellTime"` + PingSlotDr string `json:"pingSlotDr"` + PingSlotFreq string `json:"pingSlotFreq"` + Geolocation string `json:"geolocation"` + Latitude string `json:"latitude"` + Longitude string `json:"longitude"` + Altitude string `json:"altitude"` Status string `json:"status"` + LastDataUpMessage string `json:"lastDataUpMessage"` + LastDataDownMessage string `json:"lastDataDownMessage"` + LastDataUpDr string `json:"lastDataUpDr"` + DeviceProfile string `json:"device_profile"` + DevAddr string `json:"dev_addr"` + NwkSKey string `json:"NwkSKey"` + AppSKey string `json:"AppSKey"` } // EndDevice converts a Wanesy device to a TTS device. @@ -122,7 +128,7 @@ func (d Device) EndDevice(fpStore *frequencyplans.Store, applicationID, frequenc } ret.Ids.JoinEui = joinEUI.Bytes() } - if d.Rx2Dr != "NULL" { + if d.Rx2Dr != "" { s, err := strconv.ParseUint(d.Rx2Dr, 16, 32) if err != nil { return nil, err @@ -178,7 +184,7 @@ func (d Device) EndDevice(fpStore *frequencyplans.Store, applicationID, frequenc return nil, errInvalidMACVersion.WithAttributes("mac_version", d.MacVersion) } - if d.Longitude != "NULL" && d.Latitude != "NULL" && d.Altitude != "NULL" { + if d.Longitude != "" && d.Latitude != "" && d.Altitude != "" { latitude, _ := strconv.ParseFloat(d.Latitude, 64) longitude, _ := strconv.ParseFloat(d.Longitude, 64) altitude, err := strconv.ParseUint(d.Rx2Dr, 16, 32) @@ -196,7 +202,7 @@ func (d Device) EndDevice(fpStore *frequencyplans.Store, applicationID, frequenc } // Copy session information if available. - hasSession := d.DevAddr != "NULL" && d.NwkSKey != "NULL" && d.AppKey != "" + hasSession := d.DevAddr != "" && d.NwkSKey != "" && d.AppKey != "" if hasSession || !ret.SupportsJoin { var err error ret.Session = &ttnpb.Session{Keys: &ttnpb.SessionKeys{AppSKey: &ttnpb.KeyEnvelope{}, FNwkSIntKey: &ttnpb.KeyEnvelope{}}} @@ -235,12 +241,12 @@ func (d Device) EndDevice(fpStore *frequencyplans.Store, applicationID, frequenc } // Set FrameCounters - s, err := strconv.ParseUint(d.FcntUp, 16, 32) + s, err := strconv.ParseUint(d.FCntUp, 10, 32) if err != nil { return nil, err } ret.Session.LastFCntUp = uint32(s) - s, err = strconv.ParseUint(d.FcntDown, 16, 32) + s, err = strconv.ParseUint(d.FCntDown, 10, 32) if err != nil { return nil, err } @@ -253,8 +259,8 @@ func (d Device) EndDevice(fpStore *frequencyplans.Store, applicationID, frequenc } ret.MacState.CurrentParameters = ret.MacState.DesiredParameters ret.MacState.CurrentParameters.Rx1Delay = ttnpb.RxDelay_RX_DELAY_1 // Fallback - if d.Rx1Delay != "NULL" { - s, err = strconv.ParseUint(d.Rx1Delay, 16, 32) + if d.Rx1Delay != "" { + s, err = strconv.ParseUint(d.Rx1Delay, 10, 32) if err != nil { return nil, err } From 0a9a67fa107d3757d50f34223a23eb0c084539b3 Mon Sep 17 00:00:00 2001 From: Krishna Iyer Easwaran Date: Mon, 12 Feb 2024 13:45:20 +0100 Subject: [PATCH 7/8] dev: Add unit tests --- .gitignore | 3 ++ README.md | 2 + go.mod | 1 + pkg/source/wanesy/config.go | 6 +-- pkg/source/wanesy/messages_test.go | 71 +++++++++++++++++++++++++++++ pkg/source/wanesy/source.go | 2 +- pkg/source/wanesy/testdata/test.csv | 2 + pkg/source/wanesy/wanesy.go | 2 +- 8 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 pkg/source/wanesy/messages_test.go create mode 100644 pkg/source/wanesy/testdata/test.csv diff --git a/.gitignore b/.gitignore index 57d2a94..4fcbbe9 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ # Build files /build/ /dist/ + +# macOS related +*.DS_store diff --git a/README.md b/README.md index 8c9a815..ee73b54 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,8 @@ $ ttn-lw-migrate firefly application --all --invalidate-keys > devices.json ## Wanesy +Migration from Kerlink's Wanesy requires exporting the device data into a [CSV](./pkg/source/wanesy/testdata/test.csv) file and fed into this tool. Please reach out Kerlink to get an export of the devices that need to be migrated. + ### Configuration Configure with environment variables, or command-line arguments. diff --git a/go.mod b/go.mod index 6137dbb..d6154fc 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/brocaar/chirpstack-api/go/v3 v3.12.5 github.com/mdempsky/unconvert v0.0.0-20230125054757-2661c2c99a9b github.com/mgechev/revive v1.3.6 + github.com/smarty/assertions v1.15.1 github.com/smartystreets/assertions v1.13.1 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/pkg/source/wanesy/config.go b/pkg/source/wanesy/config.go index 3b38820..b649783 100644 --- a/pkg/source/wanesy/config.go +++ b/pkg/source/wanesy/config.go @@ -97,8 +97,8 @@ func (c *Config) Flags() *pflag.FlagSet { } // ImportDevices imports the devices from the provided CSV file. -func (c *Config) ImportDevices() (Devices, error) { - raw, err := os.ReadFile(c.csvPath) +func ImportDevices(csvPath string) (Devices, error) { + raw, err := os.ReadFile(csvPath) if err != nil { return nil, err } @@ -108,7 +108,7 @@ func (c *Config) ImportDevices() (Devices, error) { return nil, err } if len(readValues) < 2 { - return nil, errInvalidCSV.New() + return nil, errNoValuesInCSV.New() } values := make([]map[string]string, 0) for i := 1; i < len(readValues); i++ { diff --git a/pkg/source/wanesy/messages_test.go b/pkg/source/wanesy/messages_test.go new file mode 100644 index 0000000..55fd114 --- /dev/null +++ b/pkg/source/wanesy/messages_test.go @@ -0,0 +1,71 @@ +// Copyright © 2024 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wanesy_test + +import ( + "net/http" + "testing" + + "github.com/smarty/assertions" + "github.com/smarty/assertions/should" + . "go.thethings.network/lorawan-stack-migrate/pkg/source/wanesy" + "go.thethings.network/lorawan-stack/v3/pkg/fetch" + "go.thethings.network/lorawan-stack/v3/pkg/frequencyplans" + "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" + "go.thethings.network/lorawan-stack/v3/pkg/types" +) + +func TestImportAndExport(t *testing.T) { + a := assertions.New(t) + devices, err := ImportDevices("./testdata/test.csv") + a.So(err, should.BeNil) + a.So(len(devices), should.Equal, 1) + var devEUI types.EUI64 + if err = devEUI.UnmarshalText([]byte("1111111111111111")); err != nil { + t.Fatalf("Failed to unmarshal EUI: %v", err) + } + dev, ok := devices[devEUI] + if !ok { + t.FailNow() + } + a.So(dev.AppEui, should.Equal, "1111111111111111") + a.So(dev.FCntUp, should.Equal, "20") + a.So(dev.FCntDown, should.Equal, "10") + a.So(dev.DevAddr, should.Equal, "01234567") + + fpFetcher, err := fetch.FromHTTP( + http.DefaultClient, + "https://raw.githubusercontent.com/TheThingsNetwork/lorawan-frequency-plans/master", + ) + if err != nil { + t.Fatalf("Failed to create fetcher: %v", err) + } + + v3Device, err := dev.EndDevice(frequencyplans.NewStore(fpFetcher), "test-app", "EU_863_870") + if err != nil { + t.Fatalf("Failed to convert device: %v", err) + } + a.So(v3Device, should.NotBeNil) + + // Check converted fields. + a.So(v3Device.Ids.DevEui, should.Resemble, devEUI.Bytes()) + a.So(v3Device.Session, should.NotBeNil) + a.So(v3Device.Session.DevAddr, should.Resemble, []byte{0x01, 0x23, 0x45, 0x67}) + a.So(v3Device.Session.LastFCntUp, should.Equal, 20) + a.So(v3Device.Session.LastAFCntDown, should.Equal, 10) + a.So(v3Device.LorawanPhyVersion, should.Equal, ttnpb.PHYVersion_PHY_V1_0_2_REV_A) + a.So(v3Device.LorawanVersion, should.Equal, ttnpb.MACVersion_MAC_V1_0_2) + a.So(v3Device.Session.Keys, should.NotBeNil) +} diff --git a/pkg/source/wanesy/source.go b/pkg/source/wanesy/source.go index 0e7a041..c2ba4e4 100644 --- a/pkg/source/wanesy/source.go +++ b/pkg/source/wanesy/source.go @@ -38,7 +38,7 @@ func createNewSource(cfg *Config) source.CreateSource { if err := cfg.Initialize(src); err != nil { return nil, err } - devs, err := cfg.ImportDevices() + devs, err := ImportDevices(cfg.csvPath) if err != nil { return nil, err } diff --git a/pkg/source/wanesy/testdata/test.csv b/pkg/source/wanesy/testdata/test.csv new file mode 100644 index 0000000..212121f --- /dev/null +++ b/pkg/source/wanesy/testdata/test.csv @@ -0,0 +1,2 @@ +devEui,clusterId,clusterName,name,classType,rfRegion,country,macVersion,regParamsRevision,profile,adrEnabled,activation,appEui,appKey,fcntDown,fcntUp,devNonceCounter,fNwkSIntKey,sNwkSIntKey,rx1Delay,rx1DrOffset,rx2Dr,rx2Freq,rxWindows,cfList,dwellTime,pingSlotDr,pingSlotFreq,geolocation,latitude,longitude,altitude,status,lastDataUpMessage,lastDataDownMessage,lastDataUpDr,device_profile,dev_addr,NwkSKey,AppSKey +1111111111111111,0001,Test,Test,A,EU868,,1.0.2,A,STATIC,True,OTAA,1111111111111111,22222222222222222222222222222222,10,20,False,,,,,,,,,,,,,,,,,,,SF7BW125,,1234567,33333333333333333333333333333333,44444444444444444444444444444444 diff --git a/pkg/source/wanesy/wanesy.go b/pkg/source/wanesy/wanesy.go index e0b1c46..d5976f9 100644 --- a/pkg/source/wanesy/wanesy.go +++ b/pkg/source/wanesy/wanesy.go @@ -21,7 +21,7 @@ import ( ) var ( - errInvalidCSV = errors.DefineInvalidArgument("invalid_csv", "invalid csv file") + errNoValuesInCSV = errors.DefineInvalidArgument("no_values_in_csv", "no values in CSV file") errNoAppID = errors.DefineInvalidArgument("no_app_id", "no app id") errNoCSVFileProvided = errors.DefineInvalidArgument("no_csv_file_provided", "no csv file provided") errNoJoinEUI = errors.DefineInvalidArgument("no_join_eui", "no join eui") From 7a175394db7aa797560a4c412443bb60267a706e Mon Sep 17 00:00:00 2001 From: Krishna Iyer Easwaran Date: Mon, 12 Feb 2024 14:12:51 +0100 Subject: [PATCH 8/8] dev: Add changelog --- CHANGELOG.md | 2 ++ README.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca3fb59..ee009c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added +- Support to convert Kerlink Wanesy CSV format to The Things Stack JSON. + ### Changed ### Deprecated diff --git a/README.md b/README.md index ee73b54..b58e9f7 100644 --- a/README.md +++ b/README.md @@ -309,7 +309,7 @@ $ ttn-lw-migrate firefly application --all --invalidate-keys > devices.json ## Wanesy -Migration from Kerlink's Wanesy requires exporting the device data into a [CSV](./pkg/source/wanesy/testdata/test.csv) file and fed into this tool. Please reach out Kerlink to get an export of the devices that need to be migrated. +Migration from Kerlink's Wanesy requires exporting the device data into a [CSV](./pkg/source/wanesy/testdata/test.csv) file and feeding it into this tool. Please reach out to Kerlink to get an export of the devices that need to be migrated. ### Configuration