Add support for record sets in Route53 (#309)

* Add support for multiple TXT records * Additional security to prevent unexpected DNS change * prevent merge issues * make the pylint happy * some more pylint fixes * Fix lint * Fix lint - 2 * Finish the new implementation * Fix lint Co-authored-by: Adrien Ferrand <[email protected]>
AnalogJ · May 6, 2020 · 7860d32 · 7860d32
1 parent 005e55d
commit 7860d32
Show file tree

Hide file tree

Showing 28 changed files with 2,304 additions and 288 deletions.
diff --git a/lexicon/providers/route53.py b/lexicon/providers/route53.py
@@ -136,7 +136,14 @@ def _authenticate(self):
 
     def _change_record_sets(self, action, rtype, name, content):
         ttl = self._get_lexicon_option('ttl')
-        value = '"{0}"'.format(content) if rtype in ['TXT', 'SPF'] else content
+        resource_records = []
+        if isinstance(content, list):
+            for i in content:
+                value = '"{0}"'.format(i) if rtype in ['TXT', 'SPF'] else i
+                resource_records.append({'Value': value})
+        else:
+            value = '"{0}"'.format(content) if rtype in ['TXT', 'SPF'] else content
+            resource_records.append({'Value': value})
         try:
             self.r53_client.change_resource_record_sets(
                 HostedZoneId=self.domain_id,
@@ -151,22 +158,31 @@ def _change_record_sets(self, action, rtype, name, content):
                                 'Name': self._fqdn_name(name),
                                 'Type': rtype,
                                 'TTL': ttl if ttl is not None else 300,
-                                'ResourceRecords': [
-                                    {
-                                        'Value': value
-                                    }
-                                ]
+                                'ResourceRecords': resource_records
                             }
                         }
                     ]
                 }
             )
             return True
         except botocore.exceptions.ClientError as error:
-            LOGGER.debug(str(error), exc_info=True)
+            if "Duplicate Resource Record" in error.response['Error']['Message']:
+                # Duplicate resource, that have been a noop. This is expected.
+                return True
+            LOGGER.error(str(error), exc_info=True)
+            return False
 
     def _create_record(self, rtype, name, content):
         """Create a record in the hosted zone."""
+        existing_records = self._list_record_sets(rtype, name)
+        if existing_records:
+            existing_record = existing_records[0]
+            if isinstance(existing_records[0]['content'], list):
+                return self._change_record_sets(
+                    'UPSERT', existing_record['type'], existing_record['name'],
+                    existing_record['content'] + [content])
+            return self._change_record_sets(
+                'UPSERT', rtype, name, [existing_record['content']] + [content])
         return self._change_record_sets('CREATE', rtype, name, content)
 
     def _update_record(self, identifier=None, rtype=None, name=None, content=None):
@@ -180,24 +196,76 @@ def _update_record(self, identifier=None, rtype=None, name=None, content=None):
             rtype = record['type']
             name = record['name']
 
-        return self._change_record_sets('UPSERT', rtype, name, content)
+        existing_records = self._list_record_sets(rtype, name)
+        if not existing_records:
+            raise ValueError('No matching record to update was found.')
+
+        for existing_record in existing_records:
+            if isinstance(existing_record['content'], list):
+                # Multiple values in record.
+                LOGGER.warning(
+                    'Warning, multiple records found for given parameters, '
+                    'only first entry will be updated: %s', existing_record)
+                new_content = existing_record['content'].copy()
+                new_content[0] = content
+            else:
+                new_content = content
+
+            self._change_record_sets('UPSERT', existing_record['type'],
+                                     existing_record['name'], new_content)
+
+        return True
 
     def _delete_record(self, identifier=None, rtype=None, name=None, content=None):
         """Delete a record from the hosted zone."""
         if identifier:
-            records = [record for record in self._list_records()
-                       if identifier == _identifier(record)]
-            if not records:
+            matching_records = [record for record in self._list_records()
+                                if identifier == _identifier(record)]
+            if not matching_records:
                 raise ValueError('No record found for identifier {0}'.format(identifier))
-            record = records[0]
-            rtype = record['type']
-            name = record['name']
-            content = record['content']
+            rtype = matching_records[0]['type']
+            name = matching_records[0]['name']
+            content = matching_records[0]['content']
+
+        existing_records = self._list_record_sets(rtype, name, content)
+        if not existing_records:
+            raise ValueError('No record found for the provided type, name and content')
+
+        for existing_record in existing_records:
+            if isinstance(existing_record['content'], list) and content is not None:
+                # multiple values in record, just remove one value and only if it actually exist
+                if content in existing_record['content']:
+                    existing_record['content'].remove(content)
+                    self._change_record_sets('UPSERT', existing_record['type'],
+                                             existing_record['name'], existing_record['content'])
+            else:
+                # if only one record exist, or if content is not specified, remove whole record
+                self._change_record_sets('DELETE', existing_record['type'],
+                                         existing_record['name'], existing_record['content'])
 
-        return self._change_record_sets('DELETE', rtype, name, content)
+        return True
 
     def _list_records(self, rtype=None, name=None, content=None):
         """List all records for the hosted zone."""
+        records = self._list_record_sets(rtype, name, content)
+
+        flatten_records = []
+        for record in records:
+            if isinstance(record['content'], list):
+                for one_content in record['content']:
+                    flatten_record = record.copy()
+                    flatten_record['content'] = one_content
+                    flatten_record['id'] = _identifier(flatten_record)
+                    flatten_records.append(flatten_record)
+            else:
+                record['id'] = _identifier(record)
+                flatten_records.append(record)
+
+        LOGGER.debug('list_records: %s', records)
+
+        return flatten_records
+
+    def _list_record_sets(self, rtype=None, name=None, content=None):
         records = []
         paginator = RecordSetPaginator(self.r53_client, self.domain_id)
         for record in paginator.all_record_sets():
@@ -213,16 +281,14 @@ def _list_records(self, rtype=None, name=None, content=None):
                                   in record['ResourceRecords']]
             if content is not None and content not in record_content:
                 continue
+
             LOGGER.debug('record: %s', record)
-            record = {
+            records.append({
                 'type': record['Type'],
                 'name': self._full_name(record['Name']),
                 'ttl': record.get('TTL', None),
                 'content': record_content[0] if len(record_content) == 1 else record_content,
-            }
-            record['id'] = _identifier(record)
-            records.append(record)
-        LOGGER.debug('list_records: %s', records)
+            })
         return records
 
     def _request(self, action='GET', url='/', data=None, query_params=None):

diff --git a/lexicon/tests/providers/test_route53.py b/lexicon/tests/providers/test_route53.py
@@ -11,7 +11,7 @@
 
 
 class Route53ProviderTests(TestCase, IntegrationTests):
-    """Route53 Proivder Tests."""
+    """Route53 Provider Tests."""
 
     provider_name = 'route53'
     domain = 'fullcr1stal.tk'
@@ -46,13 +46,3 @@ def _use_vcr(self, path):
                 filter_query_parameters=self._filter_query_parameters(),
                 filter_post_data_parameters=self._filter_post_data_parameters()):
             yield
-
-    # TODO: the following skipped suite and fixtures should be enabled
-    @pytest.mark.skip(reason="new test, missing recording")
-    def test_provider_when_calling_update_record_should_modify_record_name_specified(self):
-        return
-
-    @pytest.fixture(autouse=True)
-    def _skip_suite(self, request):  # pylint: disable=no-self-use
-        if request.node.get_closest_marker('ext_suite_1'):
-            pytest.skip('Skipping extended suite')
diff --git a/tests/fixtures/cassettes/route53/IntegrationTests/test_provider_authenticate.yaml b/tests/fixtures/cassettes/route53/IntegrationTests/test_provider_authenticate.yaml
@@ -7,7 +7,7 @@ interactions:
         Qm90bzMvMS4xMy4xIFB5dGhvbi8zLjguMCBXaW5kb3dzLzEwIEJvdG9jb3JlLzEuMTYuMQ==
       X-Amz-Date:
       - !!binary |
-        MjAyMDA1MDZUMDczOTA0Wg==
+        MjAyMDA1MDZUMDkyODM1Wg==
     method: GET
     uri: https://route53.amazonaws.com/2013-04-01/hostedzonesbyname
   response:
@@ -21,9 +21,9 @@ interactions:
       Content-Type:
       - text/xml
       Date:
-      - Wed, 06 May 2020 07:39:04 GMT
+      - Wed, 06 May 2020 09:28:36 GMT
       x-amzn-RequestId:
-      - 66360bc7-54f5-4318-8abe-c708320f9ad6
+      - f7f00a65-5bc0-43ac-9bbb-8a6abdc709eb
     status:
       code: 200
       message: OK

diff --git a/...oute53/IntegrationTests/test_provider_authenticate_with_unmanaged_domain_should_fail.yaml b/...oute53/IntegrationTests/test_provider_authenticate_with_unmanaged_domain_should_fail.yaml
@@ -7,7 +7,7 @@ interactions:
         Qm90bzMvMS4xMy4xIFB5dGhvbi8zLjguMCBXaW5kb3dzLzEwIEJvdG9jb3JlLzEuMTYuMQ==
       X-Amz-Date:
       - !!binary |
-        MjAyMDA1MDZUMDczOTA0Wg==
+        MjAyMDA1MDZUMDkyODM2Wg==
     method: GET
     uri: https://route53.amazonaws.com/2013-04-01/hostedzonesbyname
   response:
@@ -21,9 +21,9 @@ interactions:
       Content-Type:
       - text/xml
       Date:
-      - Wed, 06 May 2020 07:39:04 GMT
+      - Wed, 06 May 2020 09:28:37 GMT
       x-amzn-RequestId:
-      - 5d8460cd-b643-4ca0-89da-51b6a858ba01
+      - eaa498e3-cdd1-4221-a501-1cff683fd39b
     status:
       code: 200
       message: OK

diff --git a/...tionTests/test_provider_when_calling_create_record_for_A_with_valid_name_and_content.yaml b/...tionTests/test_provider_when_calling_create_record_for_A_with_valid_name_and_content.yaml
@@ -7,7 +7,7 @@ interactions:
         Qm90bzMvMS4xMy4xIFB5dGhvbi8zLjguMCBXaW5kb3dzLzEwIEJvdG9jb3JlLzEuMTYuMQ==
       X-Amz-Date:
       - !!binary |
-        MjAyMDA1MDZUMDczOTA1Wg==
+        MjAyMDA1MDZUMDkyODM3Wg==
     method: GET
     uri: https://route53.amazonaws.com/2013-04-01/hostedzonesbyname
   response:
@@ -21,9 +21,38 @@ interactions:
       Content-Type:
       - text/xml
       Date:
-      - Wed, 06 May 2020 07:39:05 GMT
+      - Wed, 06 May 2020 09:28:37 GMT
       x-amzn-RequestId:
-      - db5a5bef-a0e4-4909-92d6-fa7d5f5fa8d1
+      - ddae99b7-1caf-4f89-9e38-1f535fea5dfe
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      User-Agent:
+      - !!binary |
+        Qm90bzMvMS4xMy4xIFB5dGhvbi8zLjguMCBXaW5kb3dzLzEwIEJvdG9jb3JlLzEuMTYuMQ==
+      X-Amz-Date:
+      - !!binary |
+        MjAyMDA1MDZUMDkyODM3Wg==
+    method: GET
+    uri: https://route53.amazonaws.com/2013-04-01/hostedzone/Z0748009194T9V149ZJ4F/rrset
+  response:
+    body:
+      string: '<?xml version="1.0"?>
+
+        <ListResourceRecordSetsResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"><ResourceRecordSets><ResourceRecordSet><Name>fullcr1stal.tk.</Name><Type>NS</Type><TTL>172800</TTL><ResourceRecords><ResourceRecord><Value>ns-310.awsdns-38.com.</Value></ResourceRecord><ResourceRecord><Value>ns-714.awsdns-25.net.</Value></ResourceRecord><ResourceRecord><Value>ns-1610.awsdns-09.co.uk.</Value></ResourceRecord><ResourceRecord><Value>ns-1141.awsdns-14.org.</Value></ResourceRecord></ResourceRecords></ResourceRecordSet><ResourceRecordSet><Name>fullcr1stal.tk.</Name><Type>SOA</Type><TTL>900</TTL><ResourceRecords><ResourceRecord><Value>ns-310.awsdns-38.com.
+        awsdns-hostmaster.amazon.com. 1 7200 900 1209600 86400</Value></ResourceRecord></ResourceRecords></ResourceRecordSet></ResourceRecordSets><IsTruncated>false</IsTruncated><MaxItems>100</MaxItems></ListResourceRecordSetsResponse>'
+    headers:
+      Content-Length:
+      - '908'
+      Content-Type:
+      - text/xml
+      Date:
+      - Wed, 06 May 2020 09:28:37 GMT
+      x-amzn-RequestId:
+      - 06f772e7-81b9-443c-8846-25b1b047f0bb
     status:
       code: 200
       message: OK
@@ -38,24 +67,24 @@ interactions:
         Qm90bzMvMS4xMy4xIFB5dGhvbi8zLjguMCBXaW5kb3dzLzEwIEJvdG9jb3JlLzEuMTYuMQ==
       X-Amz-Date:
       - !!binary |
-        MjAyMDA1MDZUMDczOTA1Wg==
+        MjAyMDA1MDZUMDkyODM3Wg==
     method: POST
     uri: https://route53.amazonaws.com/2013-04-01/hostedzone/Z0748009194T9V149ZJ4F/rrset/
   response:
     body:
       string: '<?xml version="1.0"?>
 
-        <ChangeResourceRecordSetsResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"><ChangeInfo><Id>/change/C0659269253OS9CWIQ16Q</Id><Status>PENDING</Status><SubmittedAt>2020-05-06T07:39:06.122Z</SubmittedAt><Comment>CREATE
+        <ChangeResourceRecordSetsResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"><ChangeInfo><Id>/change/C02726741IBL0UADCO68R</Id><Status>PENDING</Status><SubmittedAt>2020-05-06T09:28:38.266Z</SubmittedAt><Comment>CREATE
         using lexicon Route 53 provider</Comment></ChangeInfo></ChangeResourceRecordSetsResponse>'
     headers:
       Content-Length:
       - '340'
       Content-Type:
       - text/xml
       Date:
-      - Wed, 06 May 2020 07:39:05 GMT
+      - Wed, 06 May 2020 09:28:38 GMT
       x-amzn-RequestId:
-      - 550b4715-2861-47b5-82a3-f712e457ff5b
+      - 7924168a-6fb1-4d6e-a598-4a408a50a389
     status:
       code: 200
       message: OK

diff --git a/...Tests/test_provider_when_calling_create_record_for_CNAME_with_valid_name_and_content.yaml b/...Tests/test_provider_when_calling_create_record_for_CNAME_with_valid_name_and_content.yaml
@@ -7,7 +7,7 @@ interactions:
         Qm90bzMvMS4xMy4xIFB5dGhvbi8zLjguMCBXaW5kb3dzLzEwIEJvdG9jb3JlLzEuMTYuMQ==
       X-Amz-Date:
       - !!binary |
-        MjAyMDA1MDZUMDczOTA2Wg==
+        MjAyMDA1MDZUMDkyODM4Wg==
     method: GET
     uri: https://route53.amazonaws.com/2013-04-01/hostedzonesbyname
   response:
@@ -21,9 +21,38 @@ interactions:
       Content-Type:
       - text/xml
       Date:
-      - Wed, 06 May 2020 07:39:06 GMT
+      - Wed, 06 May 2020 09:28:38 GMT
       x-amzn-RequestId:
-      - 6da3cb88-8773-4da5-97ce-f9b258bbf4a2
+      - 0282786a-fb37-4985-b545-bf5f7c5d812c
+    status:
+      code: 200
+      message: OK
+- request:
+    body: null
+    headers:
+      User-Agent:
+      - !!binary |
+        Qm90bzMvMS4xMy4xIFB5dGhvbi8zLjguMCBXaW5kb3dzLzEwIEJvdG9jb3JlLzEuMTYuMQ==
+      X-Amz-Date:
+      - !!binary |
+        MjAyMDA1MDZUMDkyODM4Wg==
+    method: GET
+    uri: https://route53.amazonaws.com/2013-04-01/hostedzone/Z0748009194T9V149ZJ4F/rrset
+  response:
+    body:
+      string: '<?xml version="1.0"?>
+
+        <ListResourceRecordSetsResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"><ResourceRecordSets><ResourceRecordSet><Name>fullcr1stal.tk.</Name><Type>NS</Type><TTL>172800</TTL><ResourceRecords><ResourceRecord><Value>ns-310.awsdns-38.com.</Value></ResourceRecord><ResourceRecord><Value>ns-714.awsdns-25.net.</Value></ResourceRecord><ResourceRecord><Value>ns-1610.awsdns-09.co.uk.</Value></ResourceRecord><ResourceRecord><Value>ns-1141.awsdns-14.org.</Value></ResourceRecord></ResourceRecords></ResourceRecordSet><ResourceRecordSet><Name>fullcr1stal.tk.</Name><Type>SOA</Type><TTL>900</TTL><ResourceRecords><ResourceRecord><Value>ns-310.awsdns-38.com.
+        awsdns-hostmaster.amazon.com. 1 7200 900 1209600 86400</Value></ResourceRecord></ResourceRecords></ResourceRecordSet><ResourceRecordSet><Name>localhost.fullcr1stal.tk.</Name><Type>A</Type><TTL>3600</TTL><ResourceRecords><ResourceRecord><Value>127.0.0.1</Value></ResourceRecord></ResourceRecords></ResourceRecordSet></ResourceRecordSets><IsTruncated>false</IsTruncated><MaxItems>100</MaxItems></ListResourceRecordSetsResponse>'
+    headers:
+      Content-Length:
+      - '1106'
+      Content-Type:
+      - text/xml
+      Date:
+      - Wed, 06 May 2020 09:28:38 GMT
+      x-amzn-RequestId:
+      - 86ca6c33-824f-4a43-ba55-778d4cca39cb
     status:
       code: 200
       message: OK
@@ -38,24 +67,24 @@ interactions:
         Qm90bzMvMS4xMy4xIFB5dGhvbi8zLjguMCBXaW5kb3dzLzEwIEJvdG9jb3JlLzEuMTYuMQ==
       X-Amz-Date:
       - !!binary |
-        MjAyMDA1MDZUMDczOTA2Wg==
+        MjAyMDA1MDZUMDkyODM4Wg==
     method: POST
     uri: https://route53.amazonaws.com/2013-04-01/hostedzone/Z0748009194T9V149ZJ4F/rrset/
   response:
     body:
       string: '<?xml version="1.0"?>
 
-        <ChangeResourceRecordSetsResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"><ChangeInfo><Id>/change/C0258066355AIBZ6RRFIY</Id><Status>PENDING</Status><SubmittedAt>2020-05-06T07:39:06.883Z</SubmittedAt><Comment>CREATE
+        <ChangeResourceRecordSetsResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"><ChangeInfo><Id>/change/C017689023E98RW18M7SE</Id><Status>PENDING</Status><SubmittedAt>2020-05-06T09:28:39.228Z</SubmittedAt><Comment>CREATE
         using lexicon Route 53 provider</Comment></ChangeInfo></ChangeResourceRecordSetsResponse>'
     headers:
       Content-Length:
       - '340'
       Content-Type:
       - text/xml
       Date:
-      - Wed, 06 May 2020 07:39:06 GMT
+      - Wed, 06 May 2020 09:28:39 GMT
       x-amzn-RequestId:
-      - 94956a77-90e5-44ec-9ef2-63d3e4af84cb
+      - f5d08c57-57f7-4f73-ae16-541b9066c2f9
     status:
       code: 200
       message: OK