Merge pull request #108 from TheThingsNetwork/feature/wanesy

Support Migration from Wanesy

.gitignore

@@ -17,3 +17,6 @@
 # Build files
 /build/
 /dist/
+
+# macOS related
+*.DS_store

CHANGELOG.md

@@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Added
 
+- Support to convert Kerlink Wanesy CSV format to The Things Stack JSON.
+
 ### Changed
 
 ### Deprecated

README.md

@@ -307,6 +307,47 @@ $ ttn-lw-migrate firefly application --all --verbose > devices.json
 $ ttn-lw-migrate firefly application --all --invalidate-keys > devices.json
 ```
 
+## Wanesy
+
+Migration from Kerlink's Wanesy requires exporting the device data into a [CSV](./pkg/source/wanesy/testdata/test.csv) file and feeding it into this tool. Please reach out to Kerlink to get an export of the devices that need to be migrated.
+
+### Configuration
+
+Configure with environment variables, or command-line arguments.
+
+See `ttn-lw-migrate wanesy {device|application} --help` for more details.
+
+The following example shows how to set options via environment variables.
+
+```bash
+$ export APP_ID=my-test-app             # Application ID for the exported devices
+$ export FREQUENCY_PLAN_ID=EU_863_870   # Frequency Plan ID for the exported devices
+$ export CSV_PATH=<path>                # Local path to the exported CSV file.
+```
+
+### Notes
+
+- The export process will halt if any error occurs.
+- Since the migration tool uses a CSV file that's exported from WMC and does not interact with the API, make sure to remove/clean up the devices on WMC once the migration is completed.
+
+### Export Devices
+
+To export a single device using its Device EUI (e.g. `1111111111111112`):
+
+```bash
+# Export a device from the CSV to TTS format.
+$ ttn-lw-migrate wanesy device 1111111111111112 > devices.json
+```
+
+### Export All Devices
+
+In order to export all devices from the CSV file, use the `application` command.
+
+```bash
+# Export all devices from the CSV.
+$ ttn-lw-migrate wanesy application --all
+```
+
 ## Development Environment
 
 Requires Go version 1.16 or higher. [Download Go](https://golang.org/dl/).

cmd/root.go

@@ -23,6 +23,7 @@ import (
 	"go.thethings.network/lorawan-stack-migrate/cmd/firefly"
 	"go.thethings.network/lorawan-stack-migrate/cmd/ttnv2"
 	"go.thethings.network/lorawan-stack-migrate/cmd/tts"
+	"go.thethings.network/lorawan-stack-migrate/cmd/wanesy"
 	"go.thethings.network/lorawan-stack-migrate/pkg/export"
 	"go.thethings.network/lorawan-stack-migrate/pkg/source"
 )
@@ -81,4 +82,5 @@ func init() {
 	rootCmd.AddCommand(tts.TTSCmd)
 	rootCmd.AddCommand(chirpstack.ChirpStackCmd)
 	rootCmd.AddCommand(firefly.FireflyCmd)
+	rootCmd.AddCommand(wanesy.WanesyCmd)
 }

cmd/wanesy/wanesy.go

@@ -0,0 +1,27 @@
+// Copyright © 2023 The Things Network Foundation, The Things Industries B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wanesy
+
+import (
+	"go.thethings.network/lorawan-stack-migrate/pkg/commands"
+	_ "go.thethings.network/lorawan-stack-migrate/pkg/source/wanesy"
+)
+
+const sourceName = "wanesy"
+
+// WanesyCmd represents the Wanesy source.
+var WanesyCmd = commands.Source(sourceName,
+	"Migrate from Wanesy Management Center",
+)

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/brocaar/chirpstack-api/go/v3 v3.12.5
 	github.com/mdempsky/unconvert v0.0.0-20230125054757-2661c2c99a9b
 	github.com/mgechev/revive v1.3.6
+	github.com/smarty/assertions v1.15.1
 	github.com/smartystreets/assertions v1.13.1
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5

pkg/source/wanesy/config.go

@@ -0,0 +1,137 @@
+// Copyright © 2024 The Things Network Foundation, The Things Industries B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wanesy
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	"go.thethings.network/lorawan-stack-migrate/pkg/source"
+	"go.thethings.network/lorawan-stack/v3/pkg/fetch"
+	"go.thethings.network/lorawan-stack/v3/pkg/frequencyplans"
+	"go.thethings.network/lorawan-stack/v3/pkg/ttnpb"
+)
+
+type Config struct {
+	src source.Config
+
+	appID           string
+	frequencyPlanID string
+	csvPath         string
+	all             bool
+
+	derivedMacVersion ttnpb.MACVersion
+	derivedPhyVersion ttnpb.PHYVersion
+
+	flags   *pflag.FlagSet
+	fpStore *frequencyplans.Store
+}
+
+// NewConfig returns a new Wanesy configuration.
+func NewConfig() *Config {
+	config := &Config{
+		flags: &pflag.FlagSet{},
+	}
+	config.flags.StringVar(&config.frequencyPlanID,
+		"frequency-plan-id",
+		"",
+		"Frequency Plan ID for the exported devices")
+	config.flags.StringVar(&config.appID,
+		"app-id",
+		"",
+		"Application ID for the exported devices")
+	config.flags.StringVar(&config.csvPath,
+		"csv-path",
+		"",
+		"Path to the CSV file exported from Wanesy Management Center")
+	config.flags.BoolVar(&config.all,
+		"all",
+		false,
+		"Export all devices in the CSV. This is only used by the application command")
+	return config
+}
+
+// Initialize the configuration.
+func (c *Config) Initialize(src source.Config) error {
+	c.src = src
+
+	if c.appID = os.Getenv("APP_ID"); c.appID == "" {
+		return errNoAppID.New()
+	}
+	if c.frequencyPlanID = os.Getenv("FREQUENCY_PLAN_ID"); c.frequencyPlanID == "" {
+		return errNoFrequencyPlanID.New()
+	}
+	if c.csvPath = os.Getenv("CSV_PATH"); c.csvPath == "" {
+		return errNoCSVFileProvided.New()
+	}
+
+	fpFetcher, err := fetch.FromHTTP(http.DefaultClient, src.FrequencyPlansURL)
+	if err != nil {
+		return err
+	}
+	c.fpStore = frequencyplans.NewStore(fpFetcher)
+
+	return nil
+}
+
+// Flags returns the flags for the configuration.
+func (c *Config) Flags() *pflag.FlagSet {
+	return c.flags
+}
+
+// ImportDevices imports the devices from the provided CSV file.
+func ImportDevices(csvPath string) (Devices, error) {
+	raw, err := os.ReadFile(csvPath)
+	if err != nil {
+		return nil, err
+	}
+	reader := csv.NewReader(strings.NewReader(string(raw)))
+	readValues, err := reader.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+	if len(readValues) < 2 {
+		return nil, errNoValuesInCSV.New()
+	}
+	values := make([]map[string]string, 0)
+	for i := 1; i < len(readValues); i++ {
+		keys := readValues[0]
+		value := make(map[string]string)
+		for j := 0; j < len(keys); j++ {
+			noOfcolumns := len(readValues[i])
+			if j >= noOfcolumns {
+				value[keys[j]] = "" // Fill empty columns.
+				continue
+			}
+			value[keys[j]] = readValues[i][j]
+		}
+		values = append(values, value)
+	}
+	j, err := json.Marshal(values)
+	if err != nil {
+		return nil, err
+	}
+	devices := make(Devices)
+	err = devices.UnmarshalJSON(j)
+	if err != nil {
+		return nil, err
+	}
+	return devices, nil
+}