diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d1368b..99c3731 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added +- Firefly source. + ### Changed ### Deprecated diff --git a/README.md b/README.md index db887b1..6d1ad08 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Binaries are available on [GitHub](https://github.com/TheThingsNetwork/lorawan-s - [x] The Things Network Stack V2 - [x] [ChirpStack Network Server](https://www.chirpstack.io/) - [x] [The Things Stack](https://www.github.com/TheThingsNetwork/lorawan-stack/) -- [ ] [Firefly](https://fireflyiot.com/) +- [x] [Firefly](https://fireflyiot.com/) - [ ] [LORIOT Network Server](https://www.loriot.io/) Support for different sources is done by creating Source plugins. List available sources with: @@ -238,6 +238,75 @@ $ ttn-lw-migrate tts application 'my-app-id' --dry-run --verbose > devices.json $ ttn-lw-migrate tts application 'my-app-id' > devices.json ``` +## Firefly + +### Configuration + +Configure with environment variables, or command-line arguments. + +See `ttn-lw-migrate firefly {device|application} --help` for more details. + +The following example shows how to set options via environment variables. + +```bash +$ export FIREFLY_HOST=example.com # Host of the Firefly API +$ export FIREFLY_API_KEY=abcdefgh # Firefly API Key +$ export APP_ID=my-test-app # Application ID for the exported devices +$ export JOIN_EUI=1111111111111111 # JoinEUI for the exported devices +$ export FREQUENCY_PLAN_ID=EU_863_870 # Frequency Plan ID for the exported devices +$ export MAC_VERSION=1.0.2b # LoRaWAN MAC version for the exported devices +``` + +### Notes + +- The export process will halt if any error occurs. +- Use the `--invalidate-keys` option to invalidate the root and/or session keys of the devices on the Firefly server. This is necessary to prevent both networks from communicating with the same device. The last byte of the keys will be incremented by 0x01. This enables an easy rollback if necessary. Setting this flag to false (default) would result in a "dry run", where the devices are exported but they will still be able to communicate with the Firefly server. + +### Export Devices + +To export a single device using its Device EUI (e.g. `1111111111111112`): + +```bash +# dry run first, verify that no errors occur +$ ttn-lw-migrate firefly device 1111111111111112 --verbose > devices.json +# export device +$ ttn-lw-migrate firefly device 1111111111111112 --invalidate-keys > devices.json +``` + +In order to export a large number of devices, create a file named `device_euis.txt` with one device EUI per line: + +```txt +1111111111111112 +FF11111111111134 +ABCD111111111100 +``` + +And then export with: + +```bash +# dry run first, verify that no errors occur +$ ttn-lw-migrate firefly device --verbose < device_ids.txt > devices.json +# export devices +$ ttn-lw-migrate firefly device --invalidate-keys < device_ids.txt > devices.json +``` + +### Export All Devices + +The Firefly LNS does not strictly enforce device to application relationships. + +Setting the `--all` flag will export **all devices that are accessible by the API key**. The `application` command without the `--all` flag does nothing. + +> Note: Please be cautious while using this command as this might invalidate all the keys of all the devices. + +To export all devices accessible by the API Key, + +```bash +# dry run first, verify that no errors occur +$ ttn-lw-migrate firefly application --all --verbose > devices.json +# export all devices +$ ttn-lw-migrate firefly application --all --invalidate-keys > devices.json +``` + ## Development Environment Requires Go version 1.16 or higher. [Download Go](https://golang.org/dl/). diff --git a/cmd/firefly/firefly.go b/cmd/firefly/firefly.go new file mode 100644 index 0000000..acfbcf2 --- /dev/null +++ b/cmd/firefly/firefly.go @@ -0,0 +1,27 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package firefly + +import ( + "go.thethings.network/lorawan-stack-migrate/pkg/commands" + _ "go.thethings.network/lorawan-stack-migrate/pkg/source/firefly" +) + +const sourceName = "firefly" + +// FireflyCmd represents the firefly source. +var FireflyCmd = commands.Source(sourceName, + "Export devices from Digimondo's Firefly", +) diff --git a/cmd/root.go b/cmd/root.go index b37910c..05750a4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,16 +20,14 @@ import ( "github.com/spf13/cobra" "go.thethings.network/lorawan-stack-migrate/cmd/chirpstack" + "go.thethings.network/lorawan-stack-migrate/cmd/firefly" "go.thethings.network/lorawan-stack-migrate/cmd/ttnv2" "go.thethings.network/lorawan-stack-migrate/cmd/tts" "go.thethings.network/lorawan-stack-migrate/pkg/export" "go.thethings.network/lorawan-stack-migrate/pkg/source" - "go.thethings.network/lorawan-stack/v3/pkg/log" - "go.thethings.network/lorawan-stack/v3/pkg/rpcmiddleware/rpclog" ) var ( - logger *log.Logger ctx = context.Background() exportCfg = export.Config{} rootCfg = &source.RootConfig @@ -39,26 +37,8 @@ var ( SilenceUsage: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - logLevel := log.InfoLevel - if verbose, _ := cmd.Flags().GetBool("verbose"); verbose { - logLevel = log.DebugLevel - } - logHandler, err := log.NewZap("console") - if err != nil { - return err - } - logger = log.NewLogger( - logHandler, - log.WithLevel(logLevel), - ) - rpclog.ReplaceGrpcLogger(logger) - ctx = log.NewContext(ctx, logger) - exportCfg.DevIDPrefix, _ = cmd.Flags().GetString("dev-id-prefix") - exportCfg.EUIForID, _ = cmd.Flags().GetBool("set-eui-as-id") - ctx = export.NewContext(ctx, exportCfg) - - cmd.SetContext(ctx) + cmd.SetContext(export.NewContext(ctx, exportCfg)) return nil }, } @@ -86,6 +66,11 @@ func init() { "frequency-plans-url", "https://raw.githubusercontent.com/TheThingsNetwork/lorawan-frequency-plans/master", "URL for fetching frequency plans") + rootCmd.PersistentFlags().String( + "dev-id-prefix", + "", + "(optional) value to be prefixed to the resulting device IDs", + ) rootCmd.AddGroup(&cobra.Group{ ID: "sources", @@ -95,4 +80,5 @@ func init() { rootCmd.AddCommand(ttnv2.TTNv2Cmd) rootCmd.AddCommand(tts.TTSCmd) rootCmd.AddCommand(chirpstack.ChirpStackCmd) + rootCmd.AddCommand(firefly.FireflyCmd) } diff --git a/pkg/commands/export.go b/pkg/commands/export.go index 4423111..2bfc66d 100644 --- a/pkg/commands/export.go +++ b/pkg/commands/export.go @@ -16,23 +16,15 @@ package commands import ( "io" - "os" "github.com/spf13/cobra" "go.thethings.network/lorawan-stack-migrate/pkg/export" + "go.thethings.network/lorawan-stack-migrate/pkg/iterator" "go.thethings.network/lorawan-stack-migrate/pkg/source" "go.thethings.network/lorawan-stack/v3/pkg/log" ) func Export(cmd *cobra.Command, args []string, f func(s source.Source, item string) error) error { - var iter Iterator - switch len(args) { - case 0: - iter = NewReaderIterator(os.Stdin, '\n') - default: - iter = NewListIterator(args) - } - s, err := source.NewSource(cmd.Context()) if err != nil { return err @@ -43,6 +35,14 @@ func Export(cmd *cobra.Command, args []string, f func(s source.Source, item stri } }() + var iter iterator.Iterator + switch len(args) { + case 0: + iter = s.Iterator(cmd.Name() == "application") + default: + iter = iterator.NewListIterator(args) + } + for { item, err := iter.Next() switch err { diff --git a/pkg/commands/source.go b/pkg/commands/source.go index 2283100..54a8d92 100644 --- a/pkg/commands/source.go +++ b/pkg/commands/source.go @@ -127,7 +127,6 @@ func ExecuteParentPersistentPreRun(cmd *cobra.Command, args []string) error { return nil } p := cmd.Parent() - if f := p.PersistentPreRunE; f != nil { if err := f(p, args); err != nil { return err @@ -135,6 +134,7 @@ func ExecuteParentPersistentPreRun(cmd *cobra.Command, args []string) error { } else if f := p.PersistentPreRun; f != nil { f(p, args) } + cmd.SetContext(p.Context()) return nil } diff --git a/pkg/export/errors.go b/pkg/export/errors.go index dc416da..6a15f7b 100644 --- a/pkg/export/errors.go +++ b/pkg/export/errors.go @@ -22,4 +22,5 @@ var ( errInvalidFields = errors.DefineInvalidArgument("invalid_fields", "invalid fields for device `{device_id}`") errDevIDExceedsMaxLength = errors.Define("dev_id_exceeds_max_length", "device ID `{id}` exceeds max length") errAppIDExceedsMaxLength = errors.Define("app_id_exceeds_max_length", "application ID `{id}` exceeds max length") + errNoExportedIDorEUI = errors.Define("no_exported_id_or_eui", "device `{device_id}` has no exported ID or EUI") ) diff --git a/pkg/export/export.go b/pkg/export/export.go index cb9b99a..7a08cee 100644 --- a/pkg/export/export.go +++ b/pkg/export/export.go @@ -15,14 +15,14 @@ package export import ( + "encoding/hex" "fmt" "os" "strings" + "go.thethings.network/lorawan-stack-migrate/pkg/source" "go.thethings.network/lorawan-stack/v3/pkg/jsonpb" "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" - - "go.thethings.network/lorawan-stack-migrate/pkg/source" ) const ( @@ -36,7 +36,6 @@ func toJSON(dev *ttnpb.EndDevice) ([]byte, error) { } type Config struct { - EUIForID bool DevIDPrefix string } @@ -46,9 +45,13 @@ func (cfg Config) ExportDev(s source.Source, devID string) error { return errExport.WithAttributes("device_id", devID).WithCause(err) } oldID := dev.Ids.DeviceId + eui := dev.Ids.DevEui - if eui := dev.Ids.DevEui; cfg.EUIForID && eui != nil { - dev.Ids.DeviceId = strings.ToLower(string(eui)) + if oldID == "" { + if eui == nil { + return errNoExportedIDorEUI.WithAttributes("device_id", devID) + } + dev.Ids.DeviceId = strings.ToLower(hex.EncodeToString(eui)) } if cfg.DevIDPrefix != "" { dev.Ids.DeviceId = fmt.Sprintf("%s-%s", cfg.DevIDPrefix, dev.Ids.DeviceId) @@ -59,7 +62,7 @@ func (cfg Config) ExportDev(s source.Source, devID string) error { return errDevIDExceedsMaxLength.WithAttributes("id", id) } - if dev.Ids.DeviceId != oldID { + if dev.Ids.DeviceId != oldID && oldID != "" { if dev.Attributes == nil { dev.Attributes = make(map[string]string) } diff --git a/pkg/commands/iterator.go b/pkg/iterator/iterator.go similarity index 84% rename from pkg/commands/iterator.go rename to pkg/iterator/iterator.go index e5658b8..6fea334 100644 --- a/pkg/commands/iterator.go +++ b/pkg/iterator/iterator.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package commands +package iterator import ( "bufio" @@ -61,3 +61,17 @@ func (r *readerIterator) Next() (string, error) { } return strings.TrimSpace(s), err } + +// noopIterator is a no-op iterator. +type noopIterator struct { +} + +// NewNoopIterator returns a new no-op iterator. +func NewNoopIterator() Iterator { + return &noopIterator{} +} + +// Next implements Iterator +func (n *noopIterator) Next() (string, error) { + return "", io.EOF +} diff --git a/pkg/commands/iterator_test.go b/pkg/iterator/iterator_test.go similarity index 55% rename from pkg/commands/iterator_test.go rename to pkg/iterator/iterator_test.go index 43809d5..175dc4d 100644 --- a/pkg/commands/iterator_test.go +++ b/pkg/iterator/iterator_test.go @@ -1,4 +1,18 @@ -package commands_test +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iterator_test import ( "bytes" @@ -8,11 +22,11 @@ import ( "github.com/smartystreets/assertions" "github.com/smartystreets/assertions/should" - "go.thethings.network/lorawan-stack-migrate/pkg/commands" + "go.thethings.network/lorawan-stack-migrate/pkg/iterator" ) func TestListIterator(t *testing.T) { - it := commands.NewListIterator([]string{"one", "two", "three"}) + it := iterator.NewListIterator([]string{"one", "two", "three"}) a := assertions.New(t) s, err := it.Next() @@ -36,7 +50,7 @@ func TestListIterator(t *testing.T) { func TestReaderIterator(t *testing.T) { for _, sep := range []string{"\n", "\r\n"} { buf := []byte(strings.Join([]string{"one", "two", "three"}, sep)) - it := commands.NewReaderIterator(bytes.NewBuffer(buf), '\n') + it := iterator.NewReaderIterator(bytes.NewBuffer(buf), '\n') a := assertions.New(t) s, err := it.Next() diff --git a/pkg/log/log.go b/pkg/log/log.go new file mode 100644 index 0000000..11f09b2 --- /dev/null +++ b/pkg/log/log.go @@ -0,0 +1,43 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "context" + + "go.uber.org/zap" +) + +type loggerKeyType string + +var loggerKey loggerKeyType = "logger" + +// NewContext returns a new context with a *zap.SugaredLogger and panics if the logger is nil. +func NewContext(parentCtx context.Context, logger *zap.SugaredLogger) context.Context { + if logger == nil { + panic("Nil logger") + } + return context.WithValue(parentCtx, loggerKey, logger) +} + +// FromContext retrieves a *zap.SugaredLogger from a context and panics if there isn't one. +func FromContext(ctx context.Context) *zap.SugaredLogger { + val := ctx.Value(loggerKey) + logger, ok := val.(*zap.SugaredLogger) + if !ok { + panic("No logger in context") + } + return logger +} diff --git a/pkg/source/chirpstack/source.go b/pkg/source/chirpstack/source.go index 526ed7b..b8c1a8e 100644 --- a/pkg/source/chirpstack/source.go +++ b/pkg/source/chirpstack/source.go @@ -18,12 +18,15 @@ import ( "context" "fmt" "math" + "os" "strings" "time" csapi "github.com/brocaar/chirpstack-api/go/v3/as/external/api" + "go.thethings.network/lorawan-stack-migrate/pkg/iterator" "go.thethings.network/lorawan-stack-migrate/pkg/source" "go.thethings.network/lorawan-stack-migrate/pkg/source/chirpstack/config" + "go.thethings.network/lorawan-stack-migrate/pkg/util" "go.thethings.network/lorawan-stack/v3/pkg/log" "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" "go.thethings.network/lorawan-stack/v3/pkg/types" @@ -77,6 +80,11 @@ func createNewSource(cfg *config.Config) source.CreateSource { } } +// Iterator implements source.Source. +func (s Source) Iterator(bool) iterator.Iterator { + return iterator.NewReaderIterator(os.Stdin, '\n') +} + // RangeDevices implements the Source interface. func (p *Source) RangeDevices(id string, f func(source.Source, string) error) error { app, err := p.getApplication(id) @@ -135,7 +143,7 @@ func (p *Source) ExportDevice(devEui string) (*ttnpb.EndDevice, error) { } // Identifiers - dev.Ids.DevEui, err = unmarshalTextToBytes(&types.EUI64{}, devEui) + dev.Ids.DevEui, err = util.UnmarshalTextToBytes(&types.EUI64{}, devEui) if err != nil { return nil, errInvalidDevEUI.WithAttributes("dev_eui", devEui).WithCause(err) } @@ -269,19 +277,19 @@ func (p *Source) ExportDevice(devEui string) (*ttnpb.EndDevice, error) { switch dev.LorawanVersion { case ttnpb.MACVersion_MAC_V1_1: dev.RootKeys.AppKey = &ttnpb.KeyEnvelope{} - dev.RootKeys.AppKey.Key, err = unmarshalTextToBytes(&types.AES128Key{}, rootKeys.AppKey) + dev.RootKeys.AppKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, rootKeys.AppKey) if err != nil { return nil, errInvalidKey.WithAttributes(rootKeys.AppKey).WithCause(err) } dev.RootKeys.NwkKey = &ttnpb.KeyEnvelope{} - dev.RootKeys.NwkKey.Key, err = unmarshalTextToBytes(&types.AES128Key{}, rootKeys.NwkKey) + dev.RootKeys.NwkKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, rootKeys.NwkKey) if err != nil { return nil, errInvalidKey.WithAttributes(rootKeys.NwkKey).WithCause(err) } case ttnpb.MACVersion_MAC_V1_0, ttnpb.MACVersion_MAC_V1_0_1, ttnpb.MACVersion_MAC_V1_0_2, ttnpb.MACVersion_MAC_V1_0_3, ttnpb.MACVersion_MAC_V1_0_4: // For LoRaWAN v1.0.x, ChirpStack stores AppKey as NwkKey dev.RootKeys.AppKey = &ttnpb.KeyEnvelope{} - dev.RootKeys.AppKey.Key, err = unmarshalTextToBytes(&types.AES128Key{}, rootKeys.NwkKey) + dev.RootKeys.AppKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, rootKeys.NwkKey) if err != nil { return nil, errInvalidKey.WithAttributes(rootKeys.NwkKey).WithCause(err) } @@ -327,24 +335,24 @@ func (p *Source) ExportDevice(devEui string) (*ttnpb.EndDevice, error) { dev.Session.StartedAt = timestamppb.Now() dev.Session.Keys.AppSKey = &ttnpb.KeyEnvelope{} - dev.Session.Keys.AppSKey.Key, err = unmarshalTextToBytes(&types.AES128Key{}, activation.AppSKey) + dev.Session.Keys.AppSKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, activation.AppSKey) if err != nil { return nil, errInvalidKey.WithAttributes(activation.AppSKey).WithCause(err) } dev.Session.Keys.FNwkSIntKey = &ttnpb.KeyEnvelope{} - dev.Session.Keys.FNwkSIntKey.Key, err = unmarshalTextToBytes(&types.AES128Key{}, activation.FNwkSIntKey) + dev.Session.Keys.FNwkSIntKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, activation.FNwkSIntKey) if err != nil { return nil, errInvalidKey.WithAttributes(activation.FNwkSIntKey).WithCause(err) } switch dev.LorawanVersion { case ttnpb.MACVersion_MAC_V1_1: dev.Session.Keys.NwkSEncKey = &ttnpb.KeyEnvelope{} - dev.Session.Keys.NwkSEncKey.Key, err = unmarshalTextToBytes(&types.AES128Key{}, activation.NwkSEncKey) + dev.Session.Keys.NwkSEncKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, activation.NwkSEncKey) if err != nil { return nil, errInvalidKey.WithAttributes(activation.NwkSEncKey).WithCause(err) } dev.Session.Keys.SNwkSIntKey = &ttnpb.KeyEnvelope{} - dev.Session.Keys.SNwkSIntKey.Key, err = unmarshalTextToBytes(&types.AES128Key{}, activation.SNwkSIntKey) + dev.Session.Keys.SNwkSIntKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, activation.SNwkSIntKey) if err != nil { return nil, errInvalidKey.WithAttributes(activation.SNwkSIntKey).WithCause(err) } diff --git a/pkg/source/chirpstack/util.go b/pkg/source/chirpstack/util.go index 0a43f71..33e62fd 100644 --- a/pkg/source/chirpstack/util.go +++ b/pkg/source/chirpstack/util.go @@ -163,14 +163,3 @@ func (p *Source) getActivation(devEui string) (*csapi.DeviceActivation, error) { } return resp.DeviceActivation, err } - -func unmarshalTextToBytes( - unmarshaller interface { - UnmarshalText([]byte) error - Bytes() []byte - }, - source string, -) ([]byte, error) { - err := unmarshaller.UnmarshalText([]byte(source)) - return unmarshaller.Bytes(), err -} diff --git a/pkg/source/firefly/client/client.go b/pkg/source/firefly/client/client.go new file mode 100644 index 0000000..3d7fb50 --- /dev/null +++ b/pkg/source/firefly/client/client.go @@ -0,0 +1,194 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "time" + + "go.thethings.network/lorawan-stack-migrate/pkg/log" + "go.thethings.network/lorawan-stack/v3/pkg/errors" +) + +const defaultTimeout = 10 * time.Second + +// Config is the Firefly client configuration. +type Config struct { + APIKey string + Host string + CACertPath string + UseHTTP bool +} + +// Client is a Firefly client. +type Client struct { + *Config + *http.Client + ctx context.Context +} + +// New creates a new Firefly client. +func (cfg *Config) NewClient(ctx context.Context) (*Client, error) { + httpTransport := &http.Transport{} + if cfg.CACertPath != "" && !cfg.UseHTTP { + pemBytes, err := os.ReadFile(cfg.CACertPath) + if err != nil { + return nil, err + } + rootCAs := http.DefaultTransport.(*http.Transport).TLSClientConfig.RootCAs + if rootCAs == nil { + if rootCAs, err = x509.SystemCertPool(); err != nil { + rootCAs = x509.NewCertPool() + } + } + rootCAs.AppendCertsFromPEM(pemBytes) + httpTransport.TLSClientConfig = &tls.Config{ + RootCAs: rootCAs, + } + } + return &Client{ + Config: cfg, + Client: &http.Client{ + Transport: httpTransport, + Timeout: defaultTimeout, + }, + ctx: ctx, + }, nil +} + +var ( + errResourceNotFound = errors.DefineNotFound("resource_not_found", "resource `{resource}` not found") + errServer = errors.Define("server", "server error with code `{code}`") + errUnexpectedStatusCode = errors.Define("unexpected_status_code", "unexpected status code `{code}`") + errMultiplePackets = errors.Define("multiple_packets", "multiple packets found for device") +) + +// do executes an HTTP request. +func (c *Client) do(resource, method string, body []byte, params string) ([]byte, error) { + scheme := "https" + if c.UseHTTP { + scheme = "http" + } + + authAndParams := fmt.Sprintf("auth=%s", c.APIKey) + if params != "" { + authAndParams += "&" + params + } + url := url.URL{ + Scheme: scheme, + Host: c.Host, + Path: fmt.Sprintf("/api/v1/%s", resource), + RawQuery: authAndParams, + } + + logger := log.FromContext(c.ctx).With("url", url.String()) + logger.Debug("Request resource") + req, err := http.NewRequest(method, url.String(), bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + if err != nil { + return nil, err + } + res, err := c.Client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + body, err = io.ReadAll(res.Body) + if err != nil { + return nil, err + } + switch { + case res.StatusCode == http.StatusOK: + return body, nil + case res.StatusCode == http.StatusNotFound: + return nil, errResourceNotFound.WithAttributes("resource", resource) + case res.StatusCode >= 500: + return nil, errServer.WithAttributes("code", res.StatusCode) + default: + return nil, errUnexpectedStatusCode.WithAttributes("code", res.StatusCode) + } +} + +// GetDeviceByEUI gets a device by the EUI. +func (c *Client) GetDeviceByEUI(eui string) (*Device, error) { + body, err := c.do(fmt.Sprintf("devices/eui/%s", eui), http.MethodGet, nil, "") + if err != nil { + return nil, err + } + var wrapper struct { + Device Device `json:"device"` + } + if err := json.Unmarshal(body, &wrapper); err != nil { + return nil, err + } + return &wrapper.Device, nil +} + +// UpdateDevice updates the device. +func (c *Client) UpdateDeviceByEUI(eui string, dev Device) error { + wrapper := struct { + Device Device `json:"device"` + }{ + Device: dev, + } + body, err := json.Marshal(wrapper) + if err != nil { + return err + } + _, err = c.do(fmt.Sprintf("devices/eui/%s", eui), http.MethodPut, body, "") + return err +} + +// GetLastPacket gets the last packet for a device. +func (c *Client) GetLastPacket(eui string) (*Packet, error) { + body, err := c.do(fmt.Sprintf("devices/eui/%s/packets", eui), http.MethodGet, nil, "limit_to_last=1") + if err != nil { + return nil, err + } + var wrapper struct { + Packets []Packet `json:"packets"` + } + if err := json.Unmarshal(body, &wrapper); err != nil { + return nil, err + } + if len(wrapper.Packets) != 1 { + return nil, errMultiplePackets.WithAttributes("len", len(wrapper.Packets)) + } + return &wrapper.Packets[0], nil +} + +// GetAllDevices gets all devices that the API key has access to. +func (c *Client) GetAllDevices() ([]Device, error) { + body, err := c.do("devices", http.MethodGet, nil, "") + if err != nil { + return nil, err + } + var wrapper struct { + Device []Device `json:"devices"` + } + if err := json.Unmarshal(body, &wrapper); err != nil { + return nil, err + } + return wrapper.Device, nil +} diff --git a/pkg/source/firefly/client/messages.go b/pkg/source/firefly/client/messages.go new file mode 100644 index 0000000..b8bd3ce --- /dev/null +++ b/pkg/source/firefly/client/messages.go @@ -0,0 +1,86 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import "encoding/hex" + +// Location is the location of a device. +type Location struct { + Latitude float64 `json:"lat"` + Longitude float64 `json:"lon"` +} + +// Device is a Firefly device. +type Device struct { + Address string `json:"address,omitempty"` + AdrLimit int `json:"adr_limit,omitempty"` + ApplicationKey string `json:"application_key,omitempty"` + ApplicationSessionKey string `json:"application_session_key,omitempty"` + ClassC bool `json:"class_c,omitempty"` + Deduplicate bool `json:"deduplicate,omitempty"` + Description string `json:"description,omitempty"` + DeviceClassID int `json:"device_class_id,omitempty"` + EUI string `json:"eui,omitempty"` + FrameCounter int `json:"frame_counter,omitempty"` + InsertedAt string `json:"inserted_at,omitempty"` + Location *Location `json:"location,omitempty"` + Name string `json:"name,omitempty"` + NetworkSessionKey string `json:"network_session_key,omitempty"` + OTAA bool `json:"otaa,omitempty"` + OrganizationID int `json:"organization_id,omitempty"` + OverrideLocation bool `json:"override_location,omitempty"` + Region string `json:"region,omitempty"` + Rx2DataRate int `json:"rx2_data_rate,omitempty"` + SkipFCntCheck bool `json:"skip_fcnt_check,omitempty"` + Tags []string `json:"tags,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` +} + +// WithIncrementedKeys returns the device with last byte of the keys incremented. +func (d Device) WithIncrementedKeys() Device { + var ret Device + // Increment last byte of AppKey and AppSKey + if d.ApplicationKey != "" { + k, err := hex.DecodeString(d.ApplicationKey) + if err != nil { + panic(err) + } + k[len(k)-1]++ + ret.ApplicationKey = hex.EncodeToString(k) + } + if d.ApplicationSessionKey != "" { + k, err := hex.DecodeString(d.ApplicationSessionKey) + if err != nil { + panic(err) + } + k[len(k)-1]++ + ret.ApplicationSessionKey = hex.EncodeToString(k) + } + if d.NetworkSessionKey != "" { + k, err := hex.DecodeString(d.NetworkSessionKey) + if err != nil { + panic(err) + } + k[len(k)-1]++ + ret.NetworkSessionKey = hex.EncodeToString(k) + } + + return ret +} + +// Packet is a LoRaWAN uplink packet. +type Packet struct { + FCnt int `json:"fcnt"` +} diff --git a/pkg/source/firefly/config.go b/pkg/source/firefly/config.go new file mode 100644 index 0000000..32edfd2 --- /dev/null +++ b/pkg/source/firefly/config.go @@ -0,0 +1,173 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package firefly + +import ( + "net/http" + "os" + + "github.com/spf13/pflag" + + "go.thethings.network/lorawan-stack-migrate/pkg/source" + "go.thethings.network/lorawan-stack-migrate/pkg/source/firefly/client" + "go.thethings.network/lorawan-stack/v3/pkg/fetch" + "go.thethings.network/lorawan-stack/v3/pkg/frequencyplans" + "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" +) + +type Config struct { + client.Config + src source.Config + + appID string + frequencyPlanID string + joinEUI string + macVersion string + invalidateKeys bool + all bool + + derivedMacVersion ttnpb.MACVersion + derivedPhyVersion ttnpb.PHYVersion + + flags *pflag.FlagSet + fpStore *frequencyplans.Store +} + +// NewConfig returns a new Firefly configuration. +func NewConfig() *Config { + config := &Config{ + flags: &pflag.FlagSet{}, + } + + config.flags.StringVar(&config.Host, + "host", + "", + "Host of the Firefly API. Don't use the scheme (http/https). Port is optional") + config.flags.StringVar(&config.CACertPath, + "ca-cert-path", + "", + "(optional) Path to the CA certificate for the Firefly API") + config.flags.StringVar(&config.APIKey, + "api-key", + "", + "Key to access the Firefly API") + config.flags.StringVar(&config.joinEUI, + "join-eui", + "", + "JoinEUI for the exported devices") + config.flags.StringVar(&config.frequencyPlanID, + "frequency-plan-id", + "", + "Frequency Plan ID for the exported devices") + config.flags.StringVar(&config.macVersion, + "mac-version", + "", + `LoRaWAN MAC version for the exported devices. +Supported options are 1.0.0, 1.0.1, 1.0.2a, 1.0.2b, 1.0.3, 1.1.0a, 1.1.0b`) + config.flags.StringVar(&config.appID, + "app-id", + "", + "Application ID for the exported devices") + config.flags.BoolVar(&config.invalidateKeys, + "invalidate-keys", + false, + `Invalidate the root and/or session keys of the devices on the Firefly server. +This is necessary to prevent both networks from communicating with the same device. +The last byte of the keys will be incremented by 0x01. This enables an easy rollback if necessary. +Setting this flag to false would result in a dry run, +where the devices are exported but they are still valid on the firefly server + `) + config.flags.BoolVar(&config.UseHTTP, + "use-http", + false, + "(optional) Use HTTP instead of HTTPS for the Firefly API. Only for testing") + config.flags.BoolVar(&config.all, + "all", + false, + "Export all devices that the API key has access to. This is only used by the application command") + return config +} + +// Initialize the configuration. +func (c *Config) Initialize(src source.Config) error { + c.src = src + + if c.appID = os.Getenv("APP_ID"); c.appID == "" { + return errNoAppID.New() + } + if c.frequencyPlanID = os.Getenv("FREQUENCY_PLAN_ID"); c.frequencyPlanID == "" { + return errNoFrequencyPlanID.New() + } + if c.joinEUI = os.Getenv("JOIN_EUI"); c.joinEUI == "" { + return errNoJoinEUI.New() + } + if invalidateKeys := os.Getenv("JOIN_EUI"); invalidateKeys == "true" { + c.invalidateKeys = true + } + if all := os.Getenv("ALL"); all == "true" { + c.all = true + } + + if c.Host = os.Getenv("FIREFLY_HOST"); c.Host == "" { + return errNoHost.New() + } + if c.APIKey = os.Getenv("FIREFLY_API_KEY"); c.APIKey == "" { + return errNoAPIKey.New() + } + c.CACertPath = os.Getenv("FIREFLY_CA_CERT_PATH") + if useHTTP := os.Getenv("FIREFLY_USE_HTTP"); useHTTP == "true" { + c.UseHTTP = true + } + + c.macVersion = os.Getenv("MAC_VERSION") + switch c.macVersion { + case "1.0.0": + c.derivedMacVersion = ttnpb.MACVersion_MAC_V1_0 + c.derivedPhyVersion = ttnpb.PHYVersion_TS001_V1_0 + case "1.0.1": + c.derivedMacVersion = ttnpb.MACVersion_MAC_V1_0_1 + c.derivedPhyVersion = ttnpb.PHYVersion_TS001_V1_0_1 + case "1.0.2a": + c.derivedMacVersion = ttnpb.MACVersion_MAC_V1_0_2 + c.derivedPhyVersion = ttnpb.PHYVersion_RP001_V1_0_2 + case "1.0.2b": + c.derivedMacVersion = ttnpb.MACVersion_MAC_V1_0_2 + c.derivedPhyVersion = ttnpb.PHYVersion_RP001_V1_0_2_REV_B + case "1.0.3": + c.derivedMacVersion = ttnpb.MACVersion_MAC_V1_0_3 + c.derivedPhyVersion = ttnpb.PHYVersion_RP001_V1_0_3_REV_A + case "1.1.0a": + c.derivedMacVersion = ttnpb.MACVersion_MAC_V1_1 + c.derivedPhyVersion = ttnpb.PHYVersion_RP001_V1_1_REV_A + case "1.1.0b": + c.derivedMacVersion = ttnpb.MACVersion_MAC_V1_1 + c.derivedPhyVersion = ttnpb.PHYVersion_RP001_V1_1_REV_B + default: + return errInvalidMACVersion.WithAttributes("mac_version", c.macVersion) + } + + fpFetcher, err := fetch.FromHTTP(http.DefaultClient, src.FrequencyPlansURL) + if err != nil { + return err + } + c.fpStore = frequencyplans.NewStore(fpFetcher) + + return nil +} + +// Flags returns the flags for the configuration. +func (c *Config) Flags() *pflag.FlagSet { + return c.flags +} diff --git a/pkg/source/firefly/firefly.go b/pkg/source/firefly/firefly.go new file mode 100644 index 0000000..2cce457 --- /dev/null +++ b/pkg/source/firefly/firefly.go @@ -0,0 +1,41 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package firefly + +import ( + "go.thethings.network/lorawan-stack-migrate/pkg/source" + "go.thethings.network/lorawan-stack/v3/pkg/errors" +) + +var ( + errNoAPIKey = errors.DefineInvalidArgument("no_api_key", "no api key") + errNoHost = errors.DefineInvalidArgument("no_host", "no host") + errNoAppID = errors.DefineInvalidArgument("no_app_id", "no app id") + errNoJoinEUI = errors.DefineInvalidArgument("no_join_eui", "no join eui") + errNoDeviceFound = errors.DefineInvalidArgument("no_device_found", "no device with eui `{eui}` found") + errNoFrequencyPlanID = errors.DefineInvalidArgument("no_frequency_plan_id", "no frequency plan ID") + errInvalidMACVersion = errors.DefineInvalidArgument("invalid_mac_version", "invalid MAC version `{mac_version}`") +) + +func init() { + cfg := NewConfig() + + source.RegisterSource(source.Registration{ + Name: "firefly", + Description: "Migrate from Digimondo's Firefly", + FlagSet: cfg.Flags(), + Create: createNewSource(cfg), + }) +} diff --git a/pkg/source/firefly/source.go b/pkg/source/firefly/source.go new file mode 100644 index 0000000..22d80f2 --- /dev/null +++ b/pkg/source/firefly/source.go @@ -0,0 +1,210 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package firefly + +import ( + "context" + "os" + + "github.com/TheThingsNetwork/go-utils/random" + "go.thethings.network/lorawan-stack/v3/pkg/networkserver/mac" + "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" + "go.thethings.network/lorawan-stack/v3/pkg/types" + "google.golang.org/protobuf/types/known/timestamppb" + + "go.thethings.network/lorawan-stack-migrate/pkg/iterator" + "go.thethings.network/lorawan-stack-migrate/pkg/log" + "go.thethings.network/lorawan-stack-migrate/pkg/source" + "go.thethings.network/lorawan-stack-migrate/pkg/source/firefly/client" + "go.thethings.network/lorawan-stack-migrate/pkg/util" +) + +type Source struct { + *Config + *client.Client +} + +func createNewSource(cfg *Config) source.CreateSource { + return func(ctx context.Context, src source.Config) (source.Source, error) { + if err := cfg.Initialize(src); err != nil { + return nil, err + } + client, err := cfg.NewClient(log.NewContext(ctx, src.Logger)) + if err != nil { + return nil, err + } + return Source{ + Config: cfg, + Client: client, + }, nil + } +} + +// Iterator implements source.Source. +func (s Source) Iterator(isApplication bool) iterator.Iterator { + if !isApplication { + return iterator.NewReaderIterator(os.Stdin, '\n') + } + if s.all { + // The Firefly LNS does not group devices by an application. + // When the "all" flag is set, we get all devices accessible by the API key. + // We use a dummy "all" App ID to fallthrough to the RangeDevices method, + // where the appID argument is unused. + return iterator.NewListIterator( + []string{"all"}, + ) + } + return iterator.NewNoopIterator() +} + +// ExportDevice implements the source.Source interface. +func (s Source) ExportDevice(devEUIString string) (*ttnpb.EndDevice, error) { + ffdev, err := s.GetDeviceByEUI(devEUIString) + if err != nil { + return nil, err + } + if ffdev == nil { + return nil, errNoDeviceFound.WithAttributes("eui", devEUIString) + } + + var ( + devEUI, joinEUI types.EUI64 + ) + if err := devEUI.UnmarshalText([]byte(devEUIString)); err != nil { + return nil, err + } + if err := joinEUI.UnmarshalText([]byte(s.joinEUI)); err != nil { + return nil, err + } + v3dev := &ttnpb.EndDevice{ + Name: ffdev.Name, + Description: ffdev.Description, + FrequencyPlanId: s.frequencyPlanID, + Ids: &ttnpb.EndDeviceIdentifiers{ + ApplicationIds: &ttnpb.ApplicationIdentifiers{ApplicationId: s.appID}, + DevEui: devEUI.Bytes(), + JoinEui: joinEUI.Bytes(), + }, + MacSettings: &ttnpb.MACSettings{ + DesiredRx2DataRateIndex: &ttnpb.DataRateIndexValue{Value: ttnpb.DataRateIndex(ffdev.Rx2DataRate)}, + }, + SupportsClassC: ffdev.ClassC, + SupportsJoin: ffdev.OTAA, + LorawanVersion: s.derivedMacVersion, + LorawanPhyVersion: s.derivedPhyVersion, + } + if ffdev.Location != nil { + v3dev.Locations = map[string]*ttnpb.Location{ + "user": { + Latitude: ffdev.Location.Latitude, + Longitude: ffdev.Location.Longitude, + Source: ttnpb.LocationSource_SOURCE_REGISTRY, + }, + } + s.src.Logger.Debugw("Set location", "location", v3dev.Locations) + } + if v3dev.SupportsJoin { + v3dev.RootKeys = &ttnpb.RootKeys{AppKey: &ttnpb.KeyEnvelope{}} + v3dev.RootKeys.AppKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, ffdev.ApplicationKey) + if err != nil { + return nil, err + } + } + hasSession := ffdev.Address != "" && ffdev.NetworkSessionKey != "" && ffdev.ApplicationSessionKey != "" + + if hasSession || !v3dev.SupportsJoin { + v3dev.Session = &ttnpb.Session{Keys: &ttnpb.SessionKeys{AppSKey: &ttnpb.KeyEnvelope{}, FNwkSIntKey: &ttnpb.KeyEnvelope{}}} + v3dev.Session.DevAddr, err = util.UnmarshalTextToBytes(&types.DevAddr{}, ffdev.Address) + if err != nil { + return nil, err + } + if v3dev.SupportsJoin { + v3dev.Session.StartedAt = timestamppb.Now() + v3dev.Session.Keys.SessionKeyId = random.Bytes(16) + } + v3dev.Session.Keys.AppSKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, ffdev.ApplicationSessionKey) + if err != nil { + return nil, err + } + v3dev.Session.Keys.FNwkSIntKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, ffdev.NetworkSessionKey) + if err != nil { + return nil, err + } + switch v3dev.LorawanVersion { + case ttnpb.MACVersion_MAC_V1_1: + v3dev.Session.Keys.NwkSEncKey = &ttnpb.KeyEnvelope{} + v3dev.Session.Keys.NwkSEncKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, ffdev.ApplicationSessionKey) + if err != nil { + return nil, err + } + v3dev.Session.Keys.SNwkSIntKey = &ttnpb.KeyEnvelope{} + v3dev.Session.Keys.SNwkSIntKey.Key, err = util.UnmarshalTextToBytes(&types.AES128Key{}, ffdev.NetworkSessionKey) + if err != nil { + return nil, err + } + } + + // Set FrameCounters + packet, err := s.GetLastPacket(devEUIString) + if err != nil { + return nil, err + } + v3dev.Session.LastFCntUp = uint32(packet.FCnt) + v3dev.Session.LastAFCntDown = uint32(ffdev.FrameCounter) + v3dev.Session.LastNFCntDown = uint32(ffdev.FrameCounter) + + // Create a MACState. + if v3dev.MacState, err = mac.NewState(v3dev, s.fpStore, &ttnpb.MACSettings{}); err != nil { + return nil, err + } + v3dev.MacState.CurrentParameters = v3dev.MacState.DesiredParameters + v3dev.MacState.CurrentParameters.Rx1Delay = ttnpb.RxDelay_RX_DELAY_1 + } + + if s.invalidateKeys { + s.src.Logger.Debugw("Increment the last byte of the device keys", "device_id", ffdev.Name, "device_eui", ffdev.EUI) + // Increment the last byte of the device keys. + // This makes it easier to rollback a migration if needed. + updated := ffdev.WithIncrementedKeys() + err := s.UpdateDeviceByEUI(devEUIString, updated) + if err != nil { + return nil, err + } + } + + return v3dev, nil +} + +// RangeDevices implements the source.Source interface. +func (s Source) RangeDevices(_ string, f func(source.Source, string) error) error { + var ( + devs []client.Device + err error + ) + s.src.Logger.Debugw("Firefly LNS does not group devices by an application. Get all devices accessible by the API key") + devs, err = s.GetAllDevices() + if err != nil { + return err + } + for _, d := range devs { + if err := f(s, d.EUI); err != nil { + return err + } + } + return nil +} + +// Close implements the Source interface. +func (s Source) Close() error { return nil } diff --git a/pkg/source/source.go b/pkg/source/source.go index e8ec2b6..428416b 100644 --- a/pkg/source/source.go +++ b/pkg/source/source.go @@ -20,13 +20,17 @@ import ( "strings" "github.com/spf13/pflag" + "go.thethings.network/lorawan-stack-migrate/pkg/iterator" "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" + "go.uber.org/zap" ) type Config struct { DryRun, Verbose bool FrequencyPlansURL string + Logger *zap.SugaredLogger + source string } @@ -52,6 +56,8 @@ type Source interface { RangeDevices(appID string, f func(s Source, devID string) error) error // Close cleans up and terminates any open connections. Close() error + // Iterator returns an iterator for the source. + Iterator(isApplication bool) iterator.Iterator } // CreateSource is a function that constructs a new Source. @@ -82,6 +88,15 @@ func NewSource(ctx context.Context) (Source, error) { if RootConfig.Source() == "" { return nil, ErrNoSource.New() } + cfg := zap.NewProductionConfig() + if RootConfig.Verbose { + cfg.Level = zap.NewAtomicLevelAt(zap.DebugLevel) + } + zapLogger, err := cfg.Build() + if err != nil { + return nil, err + } + RootConfig.Logger = zapLogger.Sugar() if registration, ok := registeredSources[RootConfig.Source()]; ok { return registration.Create(ctx, RootConfig) } diff --git a/pkg/source/ttnv2/source.go b/pkg/source/ttnv2/source.go index b46454d..ebd666e 100644 --- a/pkg/source/ttnv2/source.go +++ b/pkg/source/ttnv2/source.go @@ -16,6 +16,7 @@ package ttnv2 import ( "context" + "os" ttnsdk "github.com/TheThingsNetwork/go-app-sdk" ttntypes "github.com/TheThingsNetwork/ttn/core/types" @@ -28,6 +29,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "google.golang.org/protobuf/types/known/wrapperspb" + "go.thethings.network/lorawan-stack-migrate/pkg/iterator" "go.thethings.network/lorawan-stack-migrate/pkg/source" ) @@ -191,6 +193,11 @@ func (s *Source) ExportDevice(devID string) (*ttnpb.EndDevice, error) { return v3dev, nil } +// Iterator implements source.Source. +func (s *Source) Iterator(bool) iterator.Iterator { + return iterator.NewReaderIterator(os.Stdin, '\n') +} + // RangeDevices implements the source.Source interface. func (s *Source) RangeDevices(appID string, f func(source.Source, string) error) error { s.config.appID = appID diff --git a/pkg/source/tts/config/config.go b/pkg/source/tts/config/config.go index d8bd478..3e221eb 100644 --- a/pkg/source/tts/config/config.go +++ b/pkg/source/tts/config/config.go @@ -23,11 +23,8 @@ import ( "github.com/spf13/pflag" "go.thethings.network/lorawan-stack-migrate/pkg/source" "go.thethings.network/lorawan-stack-migrate/pkg/source/tts/api" - "go.uber.org/zap" ) -var logger *zap.SugaredLogger - type serverConfig struct { defaultGRPCAddress, ApplicationServerGRPCAddress, @@ -150,12 +147,6 @@ type Config struct { func (c *Config) Initialize(rootConfig source.Config) error { c.Config = rootConfig - var err error - logger, err = NewLogger(c.Verbose) - if err != nil { - return err - } - if c.appAPIKey == "" { return errNoAppAPIKey.New() } @@ -164,7 +155,7 @@ func (c *Config) Initialize(rootConfig source.Config) error { switch { case c.insecure: api.SetInsecure(true) - logger.Warn("Using insecure connection to API") + c.Logger.Warn("Using insecure connection to API") default: if c.caPath != "" { @@ -179,25 +170,13 @@ func (c *Config) Initialize(rootConfig source.Config) error { // DeleteSourceDevice is not allowed during a dry run if c.DryRun && c.DeleteSourceDevice { - logger.Warn("Cannot delete source devices during a dry run.") + c.Logger.Warn("Cannot delete source devices during a dry run.") c.DeleteSourceDevice = false } return nil } -func NewLogger(verbose bool) (*zap.SugaredLogger, error) { - cfg := zap.NewProductionConfig() - if verbose { - cfg.Level = zap.NewAtomicLevelAt(zap.DebugLevel) - } - logger, err := cfg.Build() - if err != nil { - return nil, err - } - return logger.Sugar(), nil -} - func setCustomCA(path string) error { pemBytes, err := os.ReadFile(path) if err != nil { diff --git a/pkg/source/tts/source.go b/pkg/source/tts/source.go index 16ed0f2..ae2940a 100644 --- a/pkg/source/tts/source.go +++ b/pkg/source/tts/source.go @@ -16,16 +16,15 @@ package tts import ( "context" + "os" + "go.thethings.network/lorawan-stack-migrate/pkg/iterator" "go.thethings.network/lorawan-stack-migrate/pkg/source" "go.thethings.network/lorawan-stack-migrate/pkg/source/tts/api" "go.thethings.network/lorawan-stack-migrate/pkg/source/tts/config" "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" - "go.uber.org/zap" ) -var logger *zap.SugaredLogger - // Source implements the Source interface. type Source struct { ctx context.Context @@ -124,6 +123,11 @@ func (s Source) ExportDevice(devID string) (*ttnpb.EndDevice, error) { return dev, nil } +// Iterator implements source.Source. +func (s Source) Iterator(bool) iterator.Iterator { + return iterator.NewReaderIterator(os.Stdin, '\n') +} + // RangeDevices implements the source.Source interface. func (s Source) RangeDevices(appID string, f func(source.Source, string) error) error { s.config.AppID = appID @@ -166,7 +170,7 @@ func (s Source) getEndDevice(ids *ttnpb.EndDeviceIdentifiers, nsPaths, asPaths, res := &ttnpb.EndDevice{} if len(jsPaths) > 0 { if s.config.ServerConfig.JoinServerGRPCAddress == "" { - logger.With("paths", jsPaths).Warn("Join Server disabled but fields specified to get") + s.config.Logger.With("paths", jsPaths).Warn("Join Server disabled but fields specified to get") } else { js, err := api.Dial(s.ctx, s.config.ServerConfig.JoinServerGRPCAddress) if err != nil { @@ -177,7 +181,7 @@ func (s Source) getEndDevice(ids *ttnpb.EndDeviceIdentifiers, nsPaths, asPaths, FieldMask: ttnpb.FieldMask(jsPaths...), }) if err != nil { - logger.With("error", err).Warn("Could not get end device from Join Server") + s.config.Logger.With("error", err).Warn("Could not get end device from Join Server") } else { if err := validateDeviceIds(res.Ids, jsRes.Ids); err != nil { return nil, err @@ -191,7 +195,7 @@ func (s Source) getEndDevice(ids *ttnpb.EndDeviceIdentifiers, nsPaths, asPaths, } if len(asPaths) > 0 { if s.config.ServerConfig.ApplicationServerGRPCAddress == "" { - logger.With("paths", asPaths).Warn("Application Server disabled but fields specified to get") + s.config.Logger.With("paths", asPaths).Warn("Application Server disabled but fields specified to get") } else { as, err := api.Dial(s.ctx, s.config.ServerConfig.ApplicationServerGRPCAddress) if err != nil { @@ -215,7 +219,7 @@ func (s Source) getEndDevice(ids *ttnpb.EndDeviceIdentifiers, nsPaths, asPaths, } if len(nsPaths) > 0 { if s.config.ServerConfig.NetworkServerGRPCAddress == "" { - logger.With("paths", nsPaths).Warn("Network Server disabled but fields specified to get") + s.config.Logger.With("paths", nsPaths).Warn("Network Server disabled but fields specified to get") } else { ns, err := api.Dial(s.ctx, s.config.ServerConfig.NetworkServerGRPCAddress) if err != nil { @@ -253,7 +257,7 @@ func (s Source) setEndDevice(device *ttnpb.EndDevice, isPaths, nsPaths, asPaths, return nil, err } isDevice := &ttnpb.EndDevice{} - logger.With("paths", isPaths).Debug("Set end device on Identity Server") + s.config.Logger.With("paths", isPaths).Debug("Set end device on Identity Server") isDevice.SetFields(device, ttnpb.AddFields(ttnpb.ExcludeFields(isPaths, unsetPaths...), "ids")...) isRes, err := ttnpb.NewEndDeviceRegistryClient(is).Update(s.ctx, &ttnpb.UpdateEndDeviceRequest{ EndDevice: isDevice, @@ -269,14 +273,14 @@ func (s Source) setEndDevice(device *ttnpb.EndDevice, isPaths, nsPaths, asPaths, } if len(jsPaths) > 0 { if s.config.ServerConfig.JoinServerGRPCAddress == "" { - logger.With("paths", jsPaths).Warn("Join Server disabled but fields specified to set") + s.config.Logger.With("paths", jsPaths).Warn("Join Server disabled but fields specified to set") } else { js, err := api.Dial(s.ctx, s.config.ServerConfig.JoinServerGRPCAddress) if err != nil { return nil, err } jsDevice := &ttnpb.EndDevice{} - logger.With("paths", jsPaths).Debug("Set end device on Join Server") + s.config.Logger.With("paths", jsPaths).Debug("Set end device on Join Server") if err := jsDevice.SetFields(device, ttnpb.AddFields(ttnpb.ExcludeFields(jsPaths, unsetPaths...), "ids")...); err != nil { return nil, err } @@ -295,14 +299,14 @@ func (s Source) setEndDevice(device *ttnpb.EndDevice, isPaths, nsPaths, asPaths, } if len(nsPaths) > 0 { if s.config.ServerConfig.NetworkServerGRPCAddress == "" { - logger.With("paths", nsPaths).Warn("Network Server disabled but fields specified to set") + s.config.Logger.With("paths", nsPaths).Warn("Network Server disabled but fields specified to set") } else { ns, err := api.Dial(s.ctx, s.config.ServerConfig.NetworkServerGRPCAddress) if err != nil { return nil, err } nsDevice := &ttnpb.EndDevice{} - logger.With("paths", nsPaths).Debug("Set end device on Network Server") + s.config.Logger.With("paths", nsPaths).Debug("Set end device on Network Server") if err := nsDevice.SetFields(device, ttnpb.AddFields(ttnpb.ExcludeFields(nsPaths, unsetPaths...), "ids")...); err != nil { return nil, err } @@ -321,14 +325,14 @@ func (s Source) setEndDevice(device *ttnpb.EndDevice, isPaths, nsPaths, asPaths, } if len(asPaths) > 0 { if s.config.ServerConfig.ApplicationServerGRPCAddress == "" { - logger.With("paths", asPaths).Warn("Application Server disabled but fields specified to set") + s.config.Logger.With("paths", asPaths).Warn("Application Server disabled but fields specified to set") } else { as, err := api.Dial(s.ctx, s.config.ServerConfig.ApplicationServerGRPCAddress) if err != nil { return nil, err } asDevice := &ttnpb.EndDevice{} - logger.With("paths", asPaths).Debug("Set end device on Application Server") + s.config.Logger.With("paths", asPaths).Debug("Set end device on Application Server") if err := asDevice.SetFields(device, ttnpb.AddFields(ttnpb.ExcludeFields(asPaths, unsetPaths...), "ids")...); err != nil { return nil, err } diff --git a/pkg/source/tts/tts.go b/pkg/source/tts/tts.go index 8e12086..b6b756d 100644 --- a/pkg/source/tts/tts.go +++ b/pkg/source/tts/tts.go @@ -22,8 +22,6 @@ import ( func init() { cfg, flags := config.New() - logger, _ = config.NewLogger(cfg.Verbose) - source.RegisterSource(source.Registration{ Name: "tts", Description: "Migrate from The Things Stack", diff --git a/pkg/util/util.go b/pkg/util/util.go new file mode 100644 index 0000000..f252f26 --- /dev/null +++ b/pkg/util/util.go @@ -0,0 +1,27 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +// UnmarshalTextToBytes unmarshals the given source into the given unmarshaller and returns the bytes. +func UnmarshalTextToBytes( + unmarshaller interface { + UnmarshalText([]byte) error + Bytes() []byte + }, + source string, +) ([]byte, error) { + err := unmarshaller.UnmarshalText([]byte(source)) + return unmarshaller.Bytes(), err +}