Make the location of "ui_results" configurable

apps/accounts/views.py

@@ -231,8 +231,8 @@ def registration(request):
                 # It is possible to use msg.add_alternative() to add HTML content too
                 html_content = ""
                 activation_url = settings.BASE_HOST + "/accounts/" + str(activation.url) + "/activate"
-                with open('/home/' + settings.LOCAL_USER +
-                          '/Datacube/data_cube_ui/static/assets/media/email_template.html') as f:
+                from os import path
+                with open(path.join(settings.BASE_DIR, 'static/assets/media/email_template.html')) as f:
                     for line in f:
                         if (line == "\t\t\tAVAILABLE_TOOLS\n"):
                             for app in Application.objects.all():
@@ -249,7 +249,7 @@ def registration(request):
                 html_content = html_content.replace("ACTIVATION_URL", activation_url)
                 msg.add_alternative(html_content, subtype='html')
                 # Attaching content:
-                fp = open('/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/static/assets/media/banner.png',
+                fp = open(path.join(settings.BASE_DIR, 'static/assets/media/banner.png'),
                           'rb')
                 att = MIMEImage(fp.read())  # Or use MIMEImage, etc
                 fp.close()

apps/cloud_coverage/models.py

@@ -18,6 +18,7 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import os
 
 from django.db import models
 from django.core.exceptions import ValidationError
@@ -61,8 +62,8 @@ class Query(BaseQuery):
 
     """
 
-    color_scale_path = '/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/utils/color_scales/cloud_coverage'
-    base_result_dir = '/datacube/ui_results/cloud_coverage'
+    color_scale_path = os.path.join(settings.BASE_DIR, 'utils/color_scales/cloud_coverage')
+    base_result_dir = os.path.join(settings.RESULTS_DATA_DIR, 'cloud_coverage')
 
     class Meta(BaseQuery.Meta):
         unique_together = (('satellite', 'area_id', 'time_start', 'time_end', 'latitude_max', 'latitude_min',

apps/cloud_coverage/tasks.py

@@ -367,6 +367,7 @@ def create_output_products(data, task_id=None):
             titles="Clean Pixel Percentage Per Acquisition")
 
     logger.info("All products created.")
+    task.rewrite_pathnames()
     # task.update_bounds_from_dataset(dataset)
     task.complete = True
     task.execution_end = datetime.now()

apps/coastal_change/models.py

@@ -18,7 +18,9 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import os
 
+from django.conf import settings
 from django.db import models
 from django.core.exceptions import ValidationError
 
@@ -74,7 +76,7 @@ class Query(BaseQuery):
 
     animated_product = models.ForeignKey(AnimationType)
 
-    base_result_dir = '/datacube/ui_results/coastal_change'
+    base_result_dir = os.path.join(settings.RESULTS_DATA_DIR, 'coastal_change')
 
     class Meta(BaseQuery.Meta):
         unique_together = (('satellite', 'area_id', 'time_start', 'time_end', 'latitude_max', 'latitude_min',

apps/coastal_change/tasks.py

@@ -422,6 +422,7 @@ def create_output_products(data, task_id=None):
                     writer.append_data(image)
 
     logger.info("All products created.")
+    task.rewrite_pathnames()
     # task.update_bounds_from_dataset(dataset)
     task.complete = True
     task.execution_end = datetime.now()

apps/custom_mosaic_tool/models.py

@@ -18,7 +18,9 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import os
 
+from django.conf import settings
 from django.db import models
 from django.core.exceptions import ValidationError
 
@@ -83,7 +85,7 @@ class Query(BaseQuery):
     animated_product = models.ForeignKey(AnimationType)
     compositor = models.ForeignKey(Compositor)
 
-    base_result_dir = '/datacube/ui_results/custom_mosaic_tool'
+    base_result_dir = os.path.join(settings.RESULTS_DATA_DIR, 'custom_mosaic_tool')
 
     class Meta(BaseQuery.Meta):
         unique_together = (

apps/custom_mosaic_tool/tasks.py

@@ -519,6 +519,7 @@ def create_output_products(data, task_id=None):
             titles="Clean Pixel Percentage Per Acquisition")
 
     logger.info("All products created.")
+    task.rewrite_pathnames()
     # task.update_bounds_from_dataset(dataset)
     task.complete = True
     task.execution_end = datetime.now()

apps/data_cube_manager/tasks.py

@@ -64,7 +64,7 @@ def update_data_cube_details(ingested_only=True):
     dataset_types = DatasetType.objects.using('agdc').filter(
         Q(definition__has_keys=['managed']) & Q(definition__has_keys=['measurements']))
 
-    dc = DataAccessApi(config='/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/config/.datacube.conf')
+    dc = DataAccessApi(config=os.path.join(os.getenv('HOME'), '.datacube.conf'))
 
     for dataset_type in dataset_types:
         ingestion_details, created = IngestionDetails.objects.get_or_create(
@@ -86,7 +86,7 @@ def run_ingestion(ingestion_definition):
     Returns:
         The primary key of the new dataset type.
     """
-    conf_path = '/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/config/.datacube.conf'
+    conf_path = os.path.join(os.getenv('HOME'), '.datacube.conf')
     index = index_connect(local_config=LocalConfig.find([conf_path]))
 
     source_type, output_type = ingest.make_output_type(index, ingestion_definition)
@@ -104,7 +104,7 @@ def ingestion_work(output_type, source_type, ingestion_definition):
         output_type, source_type: types produced by ingest.make_output_type
         ingestion_definition: dict representing a Data Cube ingestion def produced using the utils func.
     """
-    conf_path = '/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/config/.datacube.conf'
+    conf_path = os.path.join(os.getenv('HOME'), '.datacube.conf')
     index = index_connect(local_config=LocalConfig.find([conf_path]))
 
     tasks = ingest.create_task_list(index, output_type, None, source_type, ingestion_definition)

apps/data_cube_manager/views/dataset_type.py

@@ -108,7 +108,7 @@ def post(self, request):
         #since everything is valid, now create yaml from defs..
         product_def = utils.dataset_type_definition_from_forms(metadata_form, measurement_forms)
 
-        conf_path = '/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/config/.datacube.conf'
+        conf_path = os.path.join(os.getenv('HOME'), '.datacube.conf')
         index = index_connect(local_config=conf_path)
         try:
             type_ = index.products.from_doc(product_def)
@@ -154,7 +154,7 @@ def post(self, request):
         #since everything is valid, now create yaml from defs..
         product_def = utils.dataset_type_definition_from_forms(metadata_form, measurement_forms)
         try:
-            os.makedirs('/datacube/ui_results/data_cube_manager/product_defs/')
+            os.makedirs(os.path.join(settings.RESULTS_DATA_DIR, 'data_cube_manager/product_defs/'))
         except:
             pass
 
@@ -163,7 +163,8 @@ def _dict_representer(dumper, data):
 
         yaml.SafeDumper.add_representer(OrderedDict, _dict_representer)
 
-        yaml_url = '/datacube/ui_results/data_cube_manager/product_defs/' + str(uuid.uuid4()) + '.yaml'
+        yaml_url = os.path.join(settings.RESULTS_DATA_DIR, 'data_cube_manager/product_defs/') \
+            + str(uuid.uuid4()) + '.yaml'
         with open(yaml_url, 'w') as yaml_file:
             yaml.dump(product_def, yaml_file, Dumper=yaml.SafeDumper, default_flow_style=False, indent=4)
         return JsonResponse({'status': 'OK', 'url': yaml_url})

apps/data_cube_manager/views/ingestion.py

@@ -18,6 +18,7 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import os
 
 from django.shortcuts import render, redirect
 from django.http import HttpResponse, JsonResponse
@@ -103,7 +104,7 @@ def post(self, request):
         ingestion_def = utils.ingestion_definition_from_forms(metadata_form, storage_form, ingestion_bounds_form,
                                                               measurement_forms)
         try:
-            os.makedirs('/datacube/ui_results/data_cube_manager/ingestion_configurations/')
+            os.makedirs(os.path.join(settings.RESULTS_DATA_DIR, 'data_cube_manager/ingestion_configurations/'))
         except:
             pass
 
@@ -112,7 +113,8 @@ def _dict_representer(dumper, data):
 
         yaml.SafeDumper.add_representer(OrderedDict, _dict_representer)
 
-        yaml_url = '/datacube/ui_results/data_cube_manager/ingestion_configurations/' + str(uuid.uuid4()) + '.yaml'
+        yaml_url = os.path.join(settings.RESULTS_DATA_DIR, 'data_cube_manager/ingestion_configurations/') \
+            + str(uuid.uuid4()) + '.yaml'
         with open(yaml_url, 'w') as yaml_file:
             yaml.dump(ingestion_def, yaml_file, Dumper=yaml.SafeDumper, default_flow_style=False, indent=4)
         return JsonResponse({'status': 'OK', 'url': yaml_url})
@@ -289,7 +291,7 @@ def post(self, request):
             'description':
             "Sample subset of {} created for {}".format(dataset_type.name, request.user.username),
             'location':
-            "/datacube/ingested_data/{}".format(request.user.username),
+            os.path.join(os.path.dirname(settings.RESULTS_DATA_DIR), "ingested_data/{}".format(request.user.username)),
             'file_path_template':
             "SAMPLE_CUBE_4326_{tile_index[0]}_{tile_index[1]}_{start_time}.nc",
             'summary':

apps/dc_algorithm/management/commands/band_math_app/models.py

@@ -18,6 +18,7 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import os
 
 from django.db import models
 from django.core.exceptions import ValidationError
@@ -62,8 +63,8 @@ class Query(BaseQuery):
     compositor = models.ForeignKey(Compositor)
 
     #TODO: add color scale here
-    color_scale_path = '/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/utils/color_scales/default_color_scale'
-    base_result_dir = '/datacube/ui_results/band_math_app'
+    color_scale_path = os.path.join(settings.BASE_DIR, 'utils/color_scales/default_color_scale')
+    base_result_dir = os.path.join(settings.RESULTS_DATA_DIR, 'band_math_app')
 
     class Meta(BaseQuery.Meta):
         unique_together = (('satellite', 'area_id', 'time_start', 'time_end', 'latitude_max', 'latitude_min',

apps/dc_algorithm/management/commands/base_app/models.py

@@ -18,7 +18,9 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import os
 
+from django.conf import settings
 from django.db import models
 from django.core.exceptions import ValidationError
 
@@ -92,7 +94,7 @@ class Query(BaseQuery):
     compositor = models.ForeignKey(Compositor)
 
     # TODO: Fill out the configuration paths
-    base_result_dir = '/datacube/ui_results/app_name'
+    base_result_dir = os.path.join(settings.RESULTS_DATA_DIR, 'app_name')
 
     class Meta(BaseQuery.Meta):
         unique_together = (

apps/dc_algorithm/models/abstract_base_models.py

@@ -79,7 +79,7 @@ class Meta(Query.Meta):
     #false by default, only change is false-> true
     complete = models.BooleanField(default=False)
 
-    config_path = '/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/config/.datacube.conf'
+    config_path = os.path.join(os.getenv('HOME'), '.datacube.conf')
 
     class Meta:
         abstract = True
@@ -440,6 +440,22 @@ def get_progress(self):
         clamped_int = max(0, min(rounded_int, 100))
         return clamped_int
 
+    def rewrite_pathnames(self):
+        """Rewrites all paths to be stored in the database to something the web server can serve
+
+        Converts the absolute paths of all files created during a task into paths relative
+        to the RESULTS_DATA_DIR configured in the Datacube UI's Django settings. This is done
+        by iterating over all attributes of the task and applying the conversion to those attributes
+        that are of type "str" and have a name ending in "_path".
+        """
+        for attr in dir(self):
+            if attr.endswith("_path") and isinstance(getattr(self, attr), str):
+                path = getattr(self, attr)
+                if path.startswith(settings.RESULTS_DATA_DIR):
+                    stripped_path = path.replace(settings.RESULTS_DATA_DIR, '', 1).lstrip(os.path.sep)
+                    path = os.path.join(os.path.sep, "datacube", "ui_results", stripped_path)
+                    setattr(self, attr, path)
+
 
 class GenericTask(Query, Metadata, Result):
     """Serves as the model for an algorithm task containing a Query, Metadata, and Result

apps/dc_algorithm/views.py

@@ -18,6 +18,7 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import os
 
 from django.shortcuts import render
 from django.utils.decorators import method_decorator
@@ -26,6 +27,7 @@
 from django.forms.models import model_to_dict
 from django.views import View
 from django.apps import apps
+from django.conf import settings
 
 from .models import Application, Satellite, Area
 

apps/fractional_cover/models.py

@@ -18,7 +18,9 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import os
 
+from django.conf import settings
 from django.db import models
 from django.core.exceptions import ValidationError
 
@@ -60,7 +62,7 @@ class Query(BaseQuery):
     """
     compositor = models.ForeignKey(Compositor)
 
-    base_result_dir = '/datacube/ui_results/fractional_cover'
+    base_result_dir = os.path.join(settings.RESULTS_DATA_DIR, 'fractional_cover')
 
     class Meta(BaseQuery.Meta):
         unique_together = (('satellite', 'area_id', 'time_start', 'time_end', 'latitude_max', 'latitude_min',

apps/fractional_cover/tasks.py

@@ -504,6 +504,7 @@ def create_output_products(data, task_id=None):
             titles="Clean Pixel Percentage Per Acquisition")
 
     logger.info("All products created.")
+    task.rewrite_pathnames()
     # task.update_bounds_from_dataset(dataset)
     task.complete = True
     task.execution_end = datetime.now()

apps/ndvi_anomaly/models.py

@@ -18,6 +18,7 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import os
 
 from django.db import models
 from django.core.exceptions import ValidationError
@@ -61,16 +62,16 @@ class Query(BaseQuery):
     """
     baseline_selection = models.CharField(max_length=100, default="1,2,3,4,5,6,7,8,9,10,11,12")
 
-    base_result_dir = '/datacube/ui_results/ndvi_anomaly'
+    base_result_dir = os.path.join(settings.RESULTS_DATA_DIR, 'ndvi_anomaly')
     color_scales = {
         'baseline_ndvi':
-        '/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/utils/color_scales/ndvi',
+        os.path.join(settings.BASE_DIR, 'utils/color_scales/ndvi'),
         'scene_ndvi':
-        '/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/utils/color_scales/ndvi',
+        os.path.join(settings.BASE_DIR, 'utils/color_scales/ndvi'),
         'ndvi_difference':
-        '/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/utils/color_scales/ndvi_difference',
+        os.path.join(settings.BASE_DIR, 'utils/color_scales/ndvi_difference'),
         'ndvi_percentage_change':
-        '/home/' + settings.LOCAL_USER + '/Datacube/data_cube_ui/utils/color_scales/ndvi_percentage_change'
+        os.path.join(settings.BASE_DIR, 'utils/color_scales/ndvi_percentage_change')
     }
 
     class Meta(BaseQuery.Meta):

apps/ndvi_anomaly/tasks.py

@@ -428,6 +428,7 @@ def create_output_products(data, task_id=None):
             titles="Clean Pixel Percentage Per Acquisition")
 
     logger.info("All products created.")
+    task.rewrite_pathnames()
     # task.update_bounds_from_dataset(dataset)
     task.complete = True
     task.execution_end = datetime.now()

apps/slip/models.py

@@ -18,7 +18,9 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import os
 
+from django.conf import settings
 from django.db import models
 from django.core.exceptions import ValidationError
 
@@ -74,7 +76,7 @@ class Query(BaseQuery):
     baseline_method = models.ForeignKey(BaselineMethod)
     baseline_length = models.IntegerField(default=10)
 
-    base_result_dir = '/datacube/ui_results/slip'
+    base_result_dir = os.path.join(settings.RESULTS_DATA_DIR, 'slip')
 
     class Meta(BaseQuery.Meta):
         unique_together = (

apps/slip/tasks.py

@@ -461,6 +461,7 @@ def create_output_products(data, task_id=None):
             titles=["Clean Pixel Percentage Per Acquisition", "SLIP Pixels Percentage Per Acquisition"])
 
     logger.info("All products created.")
+    task.rewrite_pathnames()
     # task.update_bounds_from_dataset(dataset)
     task.complete = True
     task.execution_end = datetime.now()