build.gradle

buildscript {
  repositories {
    maven {
      url 'https://artifactory.wetransform.to/artifactory/local'
    }
    jcenter()
  }
  dependencies {
    // Gradle plugin
    classpath 'to.wetransform.hale:gradle-hale-plugin:1.2.0-SNAPSHOT'

    // for Config class
    classpath 'eu.esdihumboldt.hale:eu.esdihumboldt.util.config:5.3.0'
  }
}

import eu.esdihumboldt.util.config.*

plugins {
  id 'org.ajoberstar.grgit' version '2.3.0'
}

apply plugin: 'to.wetransform.hale'

hale {
  cliVersion = '5.3.0'
}

configurations.all {
  resolutionStrategy.cacheChangingModulesFor 5, 'minutes'
}

/*
 * Update projects on hale connect
 */

def projects = ConfigYaml.load(file('projects.yaml')).asMap()

projects.each { name, config ->
  def projectFile = file(config.file)
  assert projectFile.exists()

  def id = config.hcId.replaceAll('/', ':').replaceAll('\\.', ':')
  def urn = "hc:project:$id"

  def user = findProperty('hcUser')
  def password = findProperty('hcPassword') ?: ''

  // update task for each project
  task("update-$name", type: hale.cli()) {
    // project export --hc-user user --hc-password password --project file --target urn

    args << 'project'
    args << 'export'

    // credentials
    if (user) {
      args << '--hc-user'
      args << user
      args << '--hc-password'
      args << password
    }

    // source
    args << '--project'
    args << projectFile.absolutePath

    // target
    args << '--target'
    args << urn

    args << '--target-setting'
    args << 'excludedata=true'

    args << '--target-setting'
    args << 'excludecachedresources=true'

    // configuration

    logToConsole = true

    description "Update project ${name} on hale-connect."
    group 'hale-connect'
  }
}

task("updateAll", type: GradleBuild) {
  description "Updates all projects on hale-connect."
  group 'hale-connect'

  // retain buildId (to have combined summary)
  // startParameter.projectProperties.buildId = project.ext.buildId

  def taskList = []
  projects.each { name, config ->
    taskList << ("update-$name" as String)
  }
  tasks = taskList
}

def lastUpdateFile = file('hc-lastUpdate')
def baseCommit = lastUpdateFile.exists() ? lastUpdateFile.text : null
def isMainRun = !project.findProperty('updateRun')
if (!baseCommit && isMainRun) {
  println "Cannot determine changes, updating all projects"
}

/**
 * Helper for finding base alignment files
 * @param  alignmentFile the alignment file
 * @param  savePath      the save path of the project
 * @return               the list of base alignment file paths, with the path
 *                       relative to the Gradle project directory
 */
def determineBaseAlignmentFiles(File alignmentFile, def savePath) {
  def alignmentPath = alignmentFile.toPath().parent
  def result = []
  def xml = new groovy.util.XmlSlurper().parse(alignmentFile)

  xml.base.each { base ->
    def location = base.'@location'.toString()
    
    URI uri = URI.create(location)
    def relative

    if (uri.absolute) {
      // absolute file or other URI
      if (uri.scheme == 'file') {
        // resolve in relation to save location
        def rel = savePath.parent.relativize(new File(uri).toPath())
        relative = rel.toString()
      }
    }
    else {
      relative = location
    }

    //XXX decoding needed?

    if (relative) {
      // relative path (to alignment)
      def path = alignmentPath.resolve(relative).normalize()
      // relativize to Gradle project folder
      def baseFile = projectDir.toPath().relativize(path).toString()
      result << baseFile

      // do the same for the base alignment file (it may reference other base alignments)
      result.addAll(determineBaseAlignmentFiles(file(baseFile), savePath))
    }
  }

  result
}

/**
 * Helper for determining files associated to a project.
 * @param  fileName the file path of the project file
 * @return          the list of file paths, with the path relative to the
 *                  Gradle project directory
 */
def determineProjectFiles(String fileName) {
  // project file and alignment file
  def alignmentFileName = fileName + '.alignment.xml'
  def result = [fileName, alignmentFileName]

  // println "Project: " + fileName

  def projectPath = file(fileName).toPath()
  def xml = new groovy.util.XmlSlurper().parse(file(fileName))

  def saveLocation = xml.'save-config'.setting.find{ it.'@name' == 'target' }?.text()
  def savePath = new File(URI.create(saveLocation)).toPath()

  xml.resource.each { res ->
    def actionId = res.'@action-id'

    if (actionId != 'eu.esdihumboldt.hale.io.instance.read.source') {
      // ignore source data

      res.setting.each { setting ->
        if (setting.'@name' == 'source') {
          def source = setting.text()
          URI uri = URI.create(source)
          def relative

          if (uri.absolute) {
            // absolute file or other URI
            if (uri.scheme == 'file') {
              // resolve in relation to save location
              def rel = savePath.parent.relativize(new File(uri).toPath())
              relative = rel.toString()
            }
          }
          else {
            relative = source
          }

          //XXX decoding needed?

          if (relative) {
            // relative path (to project)
            def path = projectPath.resolve(relative).normalize()
            // relativize to Gradle project folder
            result << projectDir.toPath().relativize(path).toString()
          }
        }
      }
    }
  }

  // alignment file -> determine base alignments
  result.addAll(determineBaseAlignmentFiles(file(alignmentFileName), savePath))

  // result.each { println it }
  // println()

  result
}

task("updateChanged", type: GradleBuild) {
  description "Updates changed projects on hale-connect (based on Git log)."
  group 'hale-connect'

  startParameter.projectProperties.updateRun = true

  def taskList = []
  projects.each { name, config ->
    if (baseCommit) {
      def commits = grgit.log {
        paths = determineProjectFiles(config.file)
        range(baseCommit, 'HEAD')
      }

      if (!commits.empty) {
        if (isMainRun) {
          println "Found changes for project '$name'"
          println()
        }
        
        taskList << ("update-$name" as String)
      }
    }
    else {
      taskList << ("update-$name" as String)
    }

  }

  if (taskList.empty && isMainRun) {
    'No changes detected, no project is updated'
  }

  tasks = taskList
}.doLast {
  // save HEAD as last update
  lastUpdateFile.text = grgit.head().id
}


/*
 * Transformation
 */

def json = new groovy.json.JsonSlurper()
def transformations = json.parse(file(
  project.hasProperty('transformationsFile') ? project.getProperty('transformationsFile') : 'transformations.json'))

task('transform-all') {
  description "Runs all transformation tasks."
  group 'Transformation'
}

if (tasks.findByPath('clean') == null) {
  // create clean task
  task clean() {
    description 'Deletes all transformation results.'
    group 'Transformation'
  }
}

tasks.clean.doLast {
  // add actions to clean task
  delete project.file('transformiert')
}

transformations.each { name, config ->
  if (config.enabled != null && !config.enabled) {
    return
  }

  String targetName = name
  File targetFolder = new File(project.file('transformiert'), targetName)
  File targetFile = new File(targetFolder, 'result.gml')
  
  // add transformation task for each key
  task("transform-$name", type: hale.transform()) {
    // transformation project
    transformation = file(config.project)

    // project variables from config
    if (config.variables) {
      projectVariables << config.variables
    }

    // special case: Hauskoordinaten mapping
    boolean isHauskoordinaten = (name == 'ad-hk')
    // special case: CityGML
    boolean isCityGML = (config.model == 'CityGML')
    // special case: LB
    boolean isLB = (config.model == 'LB')
    // special case: LN
    boolean isLN = (config.model == 'LN')

    if (config.model && !isHauskoordinaten && !isCityGML && !isLB && !isLN) {
      // restrict to a specific model -> add generic filter
      filterArgs << '-filter'
      filterArgs << "CQL:modellart.AA_Modellart.advStandardModell = '${config.model}'"

      if (config.additionalModels) {
        // add filters for additional models (also accepting those)
        config.additionalModels.each { modelName ->
          filterArgs << '-filter'
          filterArgs << "CQL:modellart.AA_Modellart.advStandardModell = '${modelName}'"
        }
      }

      if (config.extentBundesland) {
        // If the transformation imports a Bundesland extent from a CSV file,
        // also accept the imported extent
        filterArgs << '-filter'
        filterArgs << "CQL:WKT <> ''"
      }

      // add project variable
      projectVariables['ADV_MODELLART'] = config.model
    }

    // add sources
    if (!isHauskoordinaten) {
      // project w/ XML/GML sources
      
      // source folder
      def sourceFolder = project.file('quelldaten')
      if (config.sourceFolder) {
        // prefer source folder from definition
        sourceFolder = project.file(config.sourceFolder)
      }
      else {
        String folder

        if (config.model && project.hasProperty('defaultSourceFolder_' + config.model)) {
          // use custom default source folder for model if present
          folder = project.getProperty('defaultSourceFolder_' + config.model)
        }

        if (!folder && project.hasProperty('defaultSourceFolder')) {
          // otherwise use custom default source folder if present
          folder = project.getProperty('defaultSourceFolder')
        }

        if (folder) {
          sourceFolder = project.file(folder)
        }
      }

      // source CRS
      def sourceCRS = null
      if (config.sourceCRS) {
        // prefer source CRS from definition
        sourceCRS = config.sourceCRS;
      }
      else {
        if (config.model && project.hasProperty('defaultSourceCRS_' + config.model)) {
          // use custom default source folder for model if present
          sourceCRS = project.getProperty('defaultSourceCRS_' + config.model)
        }

        if (!sourceCRS && project.hasProperty('defaultSourceCRS')) {
          // otherwise use custom default source CRS if present
          sourceCRS = project.getProperty('defaultSourceCRS')
        }
      }

      // source definitions

      source(sourceFolder) {
        provider 'eu.esdihumboldt.hale.io.gml.reader'
        include '*.gz'
        setting 'contentType', 'eu.esdihumboldt.hale.io.xml.gzip'
        if (sourceCRS) {
          setting 'defaultSrs', "code:${sourceCRS}"
        }
      }

      source(sourceFolder) {
        provider 'eu.esdihumboldt.hale.io.gml.reader'
        include '*.gml'
        include '*.xml'
        setting 'contentType', 'org.eclipse.core.runtime.xml'
        if (sourceCRS) {
          setting 'defaultSrs', "code:${sourceCRS}"
        }
      }

      if (config.extentBundesland && project.hasProperty('extentBundesland')) {
        source(project.getProperty('extentBundesland')) {
          provider 'eu.esdihumboldt.hale.io.csv.reader.instance'
          setting 'skip', false
          setting 'typename', 'extentBundesland'
          setting 'separator', ';'
          setting 'decimal', '.'
          setting 'quote', '"'
          setting 'contentType', 'eu.esdihumboldt.hale.io.csv'
          setting 'charset', 'UTF-8'
        }
      }

      // filter duplicate objects based on GML ID (= AAA-ObjektID)
      filterArgs << '-exclude'
      filterArgs << '''groovy:
        def id = instance.p.id.value();
        boolean rejected = false;
        if (id) {
          withContext {
            def collect = _.context.collector(it);
            if (collect.ids.values().contains(id)) {
              _log.warn(\'Rejecting feature with duplicate id \' + id);
              rejected = true;
            }
            else {
              collect.ids << id;
            }
          }
        }

        rejected;
      '''
      // enable global filter context (to allow filtering duplicates across files)
      overallFilterContext = true 
    }
    else {
      // Hauskoordinaten project
      
      // file locations
      def hkDatei = project.hasProperty('hkDatei') ? project.property('hkDatei') : project.file('testdaten/Hauskoordinaten/5.0/adressen-testdaten.csv')

      // common settings
      def hkQuote = '"'
      def hkCharset = 'UTF-8'
      def hkDecimal = ','
      def hkSeparator = ';'
      def hkEscape = '\\'

      // define sources
      
      source(hkDatei) {
        provider 'eu.esdihumboldt.hale.io.csv.reader.instance'

        setting 'typename', 'Hauskoordinaten'
        setting 'skip', project.hasProperty('hkSkipFirst') ? project.getProperty('hkSkipFirst') : 1

        setting 'charset', hkCharset
        setting 'quote', hkQuote
        setting 'decimal', hkDecimal
        setting 'separator', hkSeparator
        setting 'escape', hkEscape
      }
    }

    // transformation target
    target(targetFile) {
      provider 'eu.esdihumboldt.hale.io.wfs.fc.write-2.0' // WFS 2 FC
      setting 'xml.pretty', true
      setting 'crs.epsg.prefix', 'http://www.opengis.net/def/crs/EPSG/0/'
      if (project.hasProperty('targetCRS')) {
        setting 'crs', 'code:' + project.getProperty('targetCRS')
      }
      setting 'skipFeatureCount', true
    }
    
    // XML schema validation
    validate('eu.esdihumboldt.hale.io.xml.validator')
    
    // folder for output and reports
    logFolder = targetFolder

    // general options
    printStacktrace = true
    trustGroovy = true

    // activate hale internal validation of transformed instances
    environment['HALE_TRANSFORMATION_INTERNAL_VALIDATION'] = 'true'
    
    // custom additional arguments
    if (config.additionalArgs && config.additionalArgs instanceof Collection) {
      additionalArgs.addAll(config.additionalArgs)
    }

    description "Runs a transformation based on the project ${transformation.name}."
    group 'Transformation'
  }

  tasks["transform-$name"].doFirst {
    targetFolder.mkdirs()
  }

  tasks['transform-all'].dependsOn("transform-$name")

  // validation
  
  task("validate-$name") {
    description "Validate the transformed data set $name (NOT IMPLEMENTED)"
    group 'Validation'
  }.doLast {
    // check if the target file exists
    // we don't add a dependency to the transform task by intention, so that
    // the validation can be run independently
    assert targetFile.exists()

    /*
     * XXX This task could be used to validate the result from the transform task
     *
     * - check reports
     *   - check if the transformation was successful and there were no errors
     *   - check if writing the result was successful and w/o errors
     *   - check if the internal validation report was successful and there were no warnings (e.g. Xlink references that were not resolved)
     *   - check if the schema validation was successful and there were no errors
     * - run a validator on the data (e.g. Schematron, INSPIRE validator, ...)
     */
    
    
  }

  // upload

  task("upload-$name", type: hale.cli()) {
    args = ['data', 'rewrite'];

    args << '--data'
    args << targetFile.absolutePath

    // use all feature types as mapping relevant types
    args << '--schema-setting'
    args << 'relevantMode=featureTypes'

    // suppress parsing geometries (no need to re-encode)
    args << '--data-setting'
    args << 'suppressParsingGeometry=true'

    args << '--target'
    args << project.findProperty('uploadWFS')
    args << '--target-writer'
    args << 'eu.esdihumboldt.hale.io.wfs.write.partitioned'
    // args << 'eu.esdihumboldt.hale.io.wfs.write'
    args << '--target-setting'
    args << 'wfsVersion=2.0.0'

    description "Upload the transformed data set $name to a WFS-T"
    group 'WFS-T upload'
  }
  tasks["upload-$name"].doFirst {
    // check if the target file exists
    // we don't add a dependency to the transform or validate task by intention,
    // so that the upload can be run independently
    assert targetFile.exists()

    // WFS-T address must be set
    assert project.hasProperty('uploadWFS')
  }

}

// add documentation tasks

task('doc') {
  description 'Generate mapping documentation (All variants).'
  group 'Documentation'
}

[
  [task: 'docHtml', cmd: 'export-doc', type: 'HTML'],
  [task: 'docExcel', cmd: 'export-table', type: 'Excel']
].each { item ->
  task(item.task, type: hale.cli()) {
    args = ['project', 'alignment', item.cmd]

    description "Generate mapping documentation (${item.type})."
    group 'Documentation'
  }

  tasks['doc'].dependsOn(item.task)
}

tasks.docHtml.doLast {
  // replace halejs URLs with one that supports https
  print 'Use https URLs to load halejs...'

  // find documentation files
  def files = fileTree(dir: projectDir, include: 'annex*/mappings/**/*.halex.svg.html')

  files.each { file ->
    def text = file.text
    file.text = text.replaceAll(
      java.util.regex.Pattern.quote('http://build-artifacts.wetransform.to'),
      'https://s3-eu-central-1.amazonaws.com/build-artifacts.wetransform.to')
  }

  println 'done'
}

// add tasks for creating derived projects

task('derive-all') {
  description "Creates all derived projects."
  group 'Derived transformation projects'
}

transformations.each { name, config ->
  if (config.enabled != null && !config.enabled) {
    return
  }

  // create a task to derive a project for every model
  // association of a project

  if (config.model && config.deriveProject != false) {
    // determine model definition
    def modelDefinition
    switch(config.model) {
    case 'DLKM':
      modelDefinition = file('dlkm.model.json')
      break
    case 'Basis-DLM':
      modelDefinition = file('basis-dlm.model.json')
      break
    case 'DLM50':
      modelDefinition = file('dlm-50.model.json')
      break
    case 'DLM250':
      modelDefinition = file('dlm-250.model.json')
      break
    case 'DLM1000':
      modelDefinition = file('dlm-1000.model.json')
      break
    }

    def projectFile = file(config.project)

    if (modelDefinition) {
      def filterDir = new File(buildDir, 'filter')
      def filterFile = new File(filterDir, name + '-' + modelDefinition.name)

      // task to prepare filter definition
      task("prepare-filter-$name").doLast {
        // copy model definition
        filterDir.mkdirs()
        copy {
          from modelDefinition
          into filterDir
          rename {
            filterFile.name
          }
        }

        if (config.deriveFilter) {
          // merge filter with model definition
          def filterDef = new groovy.json.JsonSlurper().parse(filterFile)
          filterDef.putAll(config.deriveFilter)
          filterFile.text = groovy.json.JsonOutput.prettyPrint(
            groovy.json.JsonOutput.toJson(filterDef))
        }
      }

      // task to derive project
      task("derive-$name", type: hale.cli(), dependsOn: "prepare-filter-$name") {
        args = [
          'project',
          'alignment',
          'filter',
          '--name',
          config.model,
          '--json-filter',
          filterFile,
          '--skip-empty',
          '--skip-no-type-cells',
          '--prepend-description',
          "***Automatisch generierte Variante des Projekts für das ${config.model} Modell***"
        ]

        // allow customizing the behavior (also from projects using this as sub-project)
        if (!project.ext.has('deriveProjectUseCopy') || !project.ext.get('deriveProjectUseCopy')) {
          args << '--use-base-alignment'
        }

        args << projectFile

        requiresHaleCli = true

        description "Generate derived project for model ${config.model} of project ${projectFile.name}"
        group 'Derived transformation projects'
      }

      tasks['derive-all'].dependsOn("derive-$name")
    }
  }
}

/*
 * Gradle wrapper
 */
wrapper {
  gradleVersion = '8.10.2'
}