Merge pull request #92 from opencontrol/references

Locally stored references [WIP]

fixtures/components/AWS/EC2.yaml

@@ -2,16 +2,16 @@ name: Amazon Elastic Compute Cloud # Name of the component
 documentation_complete: false # Manual check if the documentation is complete (for gap analysis)
 references:
   - name: Reference  # Name of the reference ie. EC2 website
-    url: Refernce URL  # Url of the reference
+    url: http://VerificationURL.com  # Url of the reference
     type: URL # type of reference (will affect how it's rendered in the documentation)
 verifications:
   EC2_Verification_1: # ID of verification
     name: EC2 Verification 1  # Name of verification
-    url: Verification 1 URL #  URL of the verification
+    url: http://VerificationURL.com #  URL of the verification
     type: URL # type of reference (will affect how it's rendered in the documentation)
   EC2_Verification_2:
     name: EC2 Governor 2
-    url: Verification 2 URL
+    url: artifact-ec2-1.png
     type: Image
 satisfies:
   NIST-800-53:

fixtures/components/CloudFoundry/UAA.yaml

@@ -2,16 +2,16 @@ name: User Account and Authentication (UAA) Server
 documentation_complete: false
 references:
   - name: Reference
-    url: Refernce URL
-    type: URL
+    url: artifact-uaa-1.png
+    type: Image
 verifications:
   UAA_Verification_1:
     name: UAA Verification 1
-    url: Verification 1 URL
+    url: http://Verification 1 URL
     type: URL
   UAA_Verification_2:
     name: UAA Verification 2
-    url: Verification 2 URL
+    url: artifact-uaa-2.png
     type: Image
 satisfies:
   NIST-800-53:

fixtures/docs/gitbook/content/AWS-EC2.md

@@ -1,10 +1,10 @@
 # Amazon Elastic Compute Cloud
 ### References  
 
-[Reference](Refernce URL)
+[Reference](http://VerificationURL.com)
 
 ### Verifications  
 
-[EC2 Verification 1](Verification 1 URL)
+[EC2 Verification 1](http://VerificationURL.com)
 
-[EC2 Governor 2](Verification 2 URL)
+![EC2 Governor 2](/artifacts/AWS/artifact-ec2-1.png)

fixtures/docs/gitbook/content/CloudFoundry-UAA.md

@@ -1,10 +1,10 @@
 # User Account and Authentication (UAA) Server
 ### References  
 
-[Reference](Refernce URL)
+![Reference](/artifacts/CloudFoundry/artifact-uaa-1.png)
 
 ### Verifications  
 
-[UAA Verification 1](Verification 1 URL)
+[UAA Verification 1](http://Verification 1 URL)
 
-[UAA Verification 2](Verification 2 URL)
+![UAA Verification 2](/artifacts/CloudFoundry/artifact-uaa-2.png)

fixtures/docs/gitbook/content/NIST-800-53-CM-2.md

@@ -4,6 +4,6 @@
 ## Amazon Elastic Compute Cloud
 Justification in narrative form  
 
-[EC2 Verification 1](Verification 1 URL)
+[EC2 Verification 1](http://VerificationURL.com)
 
-[UAA Verification 1](Verification 1 URL)
+[UAA Verification 1](http://Verification 1 URL)

fixtures/docs/gitbook/content/PCI-DSS-MAY-2015-1-1.md

@@ -4,6 +4,6 @@
 ## Amazon Elastic Compute Cloud
 Justification in narrative form  
 
-[EC2 Verification 1](Verification 1 URL)
+[EC2 Verification 1](http://VerificationURL.com)
 
-[UAA Verification 1](Verification 1 URL)
+[UAA Verification 1](http://Verification 1 URL)

fixtures/exports/certifications/LATO.yaml

@@ -6,33 +6,33 @@ components:
       references:
       - name: Reference
         type: URL
-        url: Refernce URL
+        url: http://VerificationURL.com
       verifications:
         EC2_Verification_1:
           name: EC2 Verification 1
           type: URL
-          url: Verification 1 URL
+          url: http://VerificationURL.com
         EC2_Verification_2:
           name: EC2 Governor 2
           type: Image
-          url: Verification 2 URL
+          url: AWS/artifact-ec2-1.png
   CloudFoundry:
     UAA:
       documentation_complete: false
       name: User Account and Authentication (UAA) Server
       references:
       - name: Reference
-        type: URL
-        url: Refernce URL
+        type: Image
+        url: CloudFoundry/artifact-uaa-1.png
       verifications:
         UAA_Verification_1:
           name: UAA Verification 1
           type: URL
-          url: Verification 1 URL
+          url: http://Verification 1 URL
         UAA_Verification_2:
           name: UAA Verification 2
           type: Image
-          url: Verification 2 URL
+          url: CloudFoundry/artifact-uaa-2.png
 name: LATO
 standards:
   NIST-800-53:
@@ -67,17 +67,6 @@ standards:
         family: CM
         name: Baseline Configuration
   PCI-DSS-MAY-2015:
-    1.1.1:
-      justifications:
-      - component: EC2
-        implementation_status: partial
-        narrative: Justification in narrative form
-        references: null
-        system: AWS
-      meta:
-        family: '1'
-        name: A formal process for approving and testing all network connections and
-          changes to the firewall and router configurations
     1.1:
       justifications:
       - component: EC2
@@ -105,3 +94,14 @@ standards:
         family: '2'
         name: Always change vendor-supplied defaults and remove or disable unnecessary
           default accounts before installing a system on the network.
+    1.1.1:
+      justifications:
+      - component: EC2
+        implementation_status: partial
+        narrative: Justification in narrative form
+        references: null
+        system: AWS
+      meta:
+        family: '1'
+        name: A formal process for approving and testing all network connections and
+          changes to the firewall and router configurations

src/renderers/certifications_to_gitbook.py

@@ -1,3 +1,6 @@
+""" Converts certification documentation to a gitbook """
+
+import filecmp
 import glob
 import os
 import shutil
@@ -15,17 +18,36 @@ def write_markdown(output_path, filename, text):
         md_file.write(text)
 
 
-def convert_element(element):
+def prepare_locally_stored_files(element, io_paths):
+    """ Prepare the files by moving locally stored files to the `artifacts` directory
+    and linking filepaths to that directory """
+    item_path = element['url']
+    if not ('http://' in item_path or 'https://' in item_path):
+        element['url'] = os.path.join('/artifacts', item_path).replace('\\', '/')
+        if io_paths:
+            output_path = os.path.join(io_paths['output'], 'artifacts', item_path)
+            input_path = os.path.join(io_paths['input'], item_path)
+            utils.create_dir(os.path.dirname(output_path))
+            if not os.path.exists(output_path) or not filecmp.cmp(input_path, output_path):
+                shutil.copy(input_path, output_path)
+
+
+def convert_element(element, io_paths=None):
     """ Converts a dict with a name url and type to markdown """
-    return '\n[{0}]({1})\n'.format(element['name'], element['url'])
+    prepare_locally_stored_files(element, io_paths)
+    if element['type'].lower() == 'image':
+        base_text = '\n![{0}]({1})\n'
+    else:
+        base_text = '\n[{0}]({1})\n'
+    return base_text.format(element['name'], element['url'])
 
 
 def generate_text_narative(narative):
     """ Checks if the narrative is in dict format or in string format.
     If the narrative is in dict format the script converts it to to a
     string """
     text = ''
-    if type(narative) == dict:
+    if isinstance(narative, dict):
         for key in sorted(narative):
             text += '{0}. {1} \n '.format(key, narative[key])
     else:
@@ -91,8 +113,8 @@ def concat_markdowns(markdown_path, output_path):
         utils.create_dir(ouput_dir)
         shutil.copy(filename, output_filepath)
     summary_path = os.path.join(markdown_path, 'SUMMARY.md')
-    with open(summary_path, 'r') as f:
-        main_summary = f.read()
+    with open(summary_path, 'r') as summary_file:
+        main_summary = summary_file.read()
     return main_summary
 
 
@@ -147,6 +169,7 @@ def fetch_component(reference, certification):
 
 
 def fetch_verification(verification_ref, certification):
+    """ Get the verfication component """
     component = fetch_component(verification_ref, certification)['verifications']
     return component[verification_ref['verification']]
 
@@ -185,14 +208,18 @@ def build_control_text(control, certification):
     return text
 
 
-def build_component_text(component):
+def build_component_text(component, io_paths):
     """ Create markdown output for component text """
     text = '\n### References  \n'
-    for reference in sorted(component.get('references', []), key=lambda k: k['name']):
-        text += convert_element(reference)
-    text += '\n### Verifications  \n'
-    for verification_key in sorted(component.get('verifications', [])):
-        text += convert_element(component['verifications'][verification_key])
+    references = component.get('references', [])
+    if references:
+        for reference in sorted(references, key=lambda k: k['name']):
+            text += convert_element(reference, io_paths)
+        text += '\n### Verifications  \n'
+    verifications = component.get('verifications', [])
+    if verifications:
+        for verification_key in sorted(verifications):
+            text += convert_element(component['verifications'][verification_key], io_paths)
     return text
 
 
@@ -205,13 +232,13 @@ def build_cert_page(page_dict, certification, output_path):
     write_markdown(output_path, file_name, text)
 
 
-def build_component_page(page_dict, certification, output_path):
+def build_component_page(page_dict, certification, io_paths):
     """ Write a page for the gitbook """
     text = '# {0}'.format(page_dict['component_name'])
     component = certification['components'][page_dict['system_key']][page_dict['component_key']]
-    text += build_component_text(component)
+    text += build_component_text(component, io_paths)
     file_name = 'content/' + page_dict['slug'] + '.md'
-    write_markdown(output_path, file_name, text)
+    write_markdown(io_paths['output'], file_name, text)
 
 
 def natural_sort(elements):
@@ -238,23 +265,27 @@ def build_standards_documentation(certification, output_path):
     return summary
 
 
-def build_components_documentation(certification, output_path):
+def build_components_documentation(certification, io_paths):
     """ Create the documentation for the components """
     summary = {}
     for system_key in sorted(certification['components']):
         summary[system_key] = {}
         for component_key in sorted(certification['components'][system_key]):
             page_dict = document_component_page(certification, system_key, component_key)
-            build_component_page(page_dict, certification, output_path)
+            build_component_page(page_dict, certification, io_paths)
             summary[system_key][component_key] = page_dict
     return summary
 
 
 def create_gitbook_documentation(certification_path, output_path, markdown_path=None):
     """ Convert certification to pages format """
     summaries = {}
+    io_paths = {
+        'output': output_path,
+        'input': os.path.dirname(certification_path)
+    }
     certification = utils.yaml_loader(certification_path)
+    summaries['components'] = build_components_documentation(certification, io_paths)
     summaries['standards'] = build_standards_documentation(certification, output_path)
-    summaries['components'] = build_components_documentation(certification, output_path)
     build_summary(summaries, output_path, markdown_path)
     return output_path