chore: Compatibility for unfolded CSN (#381)

- [x] support `assoc.elements` (instead or `.foreignKeys`)
- [x] `localized` views are missing
- [ ] calculating the full name of a deeply structured element by
walking up its `.parent` yields faulty names for already unfolded
structs with a `.parent` in their prototype.
- [x]  Smart Wildcards / overwrites: Structures are already unfolded
```
const input = CQL`SELECT from bookshop.Bar { 'first' as structure, * }` //> leafs of structure will still be rendered
```
- [ ] ...

---------

Co-authored-by: Patrice Bender <[email protected]>

Author: danjoa

.gitignore

@@ -137,3 +137,6 @@ dist
 
 # cds-typer
 @cds-models/
+
+# cds temp folders
+**/_out

db-service/lib/cqn4sql.js

@@ -972,7 +972,11 @@ function cqn4sql(originalQuery, model = cds.context?.model || cds.model) {
      * @returns {boolean} true if the element is a managed association and the model is flat
      */
     function isManagedAssocInFlatMode(e) {
-      return model.meta.transformation === 'odata' && e.isAssociation && e.keys
+      return (
+        (model.meta.transformation === 'odata' || model.meta.unfolded?.some(u => u === 'assocs')) &&
+        e.isAssociation &&
+        e.keys
+      )
     }
   }
 
@@ -1025,25 +1029,32 @@ function cqn4sql(originalQuery, model = cds.context?.model || cds.model) {
     if (!column) return column
     if (column.val || column.func || column.SELECT) return [column]
 
+    const structsAreUnfoldedAlready = model.meta.unfolded?.some(u => u === 'structs')
     let { baseName, columnAlias, tableAlias } = names
     const { exclude, replace } = excludeAndReplace || {}
     const { $refLinks, flatName, isJoinRelevant } = column
     let leafAssoc
     let element = $refLinks ? $refLinks[$refLinks.length - 1].definition : column
     if (isWildcard && element.type === 'cds.LargeBinary') return []
-    if (element.on) return [] // unmanaged doesn't make it into columns
+    if (element.on && !element.keys) return [] // unmanaged doesn't make it into columns
     else if (element.virtual === true) return []
     else if (!isJoinRelevant && flatName) baseName = flatName
     else if (isJoinRelevant) {
       const leaf = column.$refLinks[column.$refLinks.length - 1]
       leafAssoc = [...column.$refLinks].reverse().find(link => link.definition.isAssociation)
-      const { foreignKeys } = leafAssoc.definition
-      if (foreignKeys && leaf.alias in foreignKeys) {
+      let elements
+      //> REVISIT: remove once UCSN is standard (no more .foreignKeys)
+      elements = leafAssoc.definition.elements || leafAssoc.definition.foreignKeys
+      if (elements && leaf.alias in elements) {
         element = leafAssoc.definition
         baseName = getFullName(leafAssoc.definition)
         columnAlias = column.ref.slice(0, -1).map(idOnly).join('_')
       } else baseName = getFullName(column.$refLinks[column.$refLinks.length - 1].definition)
-    } else baseName = baseName ? `${baseName}_${element.name}` : getFullName(element)
+    } else if (!baseName && structsAreUnfoldedAlready) {
+      baseName = element.name // name is already fully constructed
+    } else {
+      baseName = baseName ? `${baseName}_${element.name}` : getFullName(element)
+    }
 
     // now we have the name of the to be expanded column
     // it could be a structure, an association or a scalar
@@ -1621,7 +1632,7 @@ function cqn4sql(originalQuery, model = cds.context?.model || cds.model) {
    * @returns {boolean}
    */
   function isStructured(elt) {
-    return Boolean(elt?.elements && elt.kind === 'element')
+    return Boolean(elt?.kind !== 'entity' && elt?.elements && !elt.isAssociation)
   }
 
   /**
@@ -1994,7 +2005,8 @@ function cqn4sql(originalQuery, model = cds.context?.model || cds.model) {
    * @returns the flat name of the element
    */
   function getFullName(node, name = node.name) {
-    if (node.parent.kind === 'entity') return name
+    // REVISIT: this is an unfortunate implementation
+    if (!node.parent || node.parent.kind === 'entity') return name
 
     return getFullName(node.parent, `${node.parent.name}_${name}`)
   }

db-service/lib/infer/index.js

@@ -99,12 +99,12 @@ function infer(originalQuery, model = cds.context?.model || cds.model) {
       if (!target) throw new Error(`"${first}" not found in the definitions of your model`)
       if (ref.length > 1) {
         target = from.ref.slice(1).reduce((d, r) => {
-          const next = d.elements[r.id || r]?.elements ? d.elements[r.id || r] : d.elements[r.id || r]?._target
+          const next = d.elements[r.id || r]?._target || d.elements[r.id || r]
           if (!next) throw new Error(`No association “${r.id || r}” in ${d.kind} “${d.name}”`)
           return next
         }, target)
       }
-      if (target.kind !== 'entity' && !target._isAssociation)
+      if (target.kind !== 'entity' && !target.isAssociation)
         throw new Error('Query source must be a an entity or an association')
 
       attachRefLinksToArg(from) // REVISIT: remove
@@ -165,7 +165,7 @@ function infer(originalQuery, model = cds.context?.model || cds.model) {
         // we need to search for first step in ´model.definitions[infixAlias]`
         if ($baseLink) {
           const { definition } = $baseLink
-          const elements = definition.elements || definition._target?.elements
+          const elements = definition._target?.elements || definition.elements
           const e = elements?.[id] || cds.error`"${id}" not found in the elements of "${definition.name}"`
           if (e.target) {
             // only fk access in infix filter
@@ -176,7 +176,7 @@ function infer(originalQuery, model = cds.context?.model || cds.model) {
                 `"${e.name}" in path "${arg.ref.map(idOnly).join('.')}" must not be an unmanaged association`,
               )
             // no non-fk traversal in infix filter
-            if (!expandOrExists && nextStep && !(nextStep in e.foreignKeys))
+            if (!expandOrExists && nextStep && !isForeignKeyOf(nextStep, e))
               throw new Error(`Only foreign keys of "${e.name}" can be accessed in infix filter`)
           }
           arg.$refLinks.push({ definition: e, target: definition })
@@ -495,7 +495,7 @@ function infer(originalQuery, model = cds.context?.model || cds.model) {
             nameSegments.push(id)
           } else if ($baseLink) {
             const { definition, target } = $baseLink
-            const elements = definition.elements || definition._target?.elements
+            const elements = definition._target?.elements || definition.elements
             if (elements && id in elements) {
               const element = elements[id]
               rejectNonFkAccess(element)
@@ -530,7 +530,7 @@ function infer(originalQuery, model = cds.context?.model || cds.model) {
           }
         } else {
           const { definition } = column.$refLinks[i - 1]
-          const elements = definition.elements || definition._target?.elements
+          const elements = definition._target?.elements || definition.elements //> go for assoc._target first, instead of assoc as struct
           const element = elements?.[id]
 
           if (firstStepIsSelf && element?.isAssociation) {
@@ -648,25 +648,25 @@ function infer(originalQuery, model = cds.context?.model || cds.model) {
         }
 
         /**
-         * Check if the next step in the ref is foreign key of `element`
+         * Check if the next step in the ref is foreign key of `assoc`
          * if not, an error is thrown.
-         * 
-         * @param {CSN.Element} element if this is an association, the next step must be a foreign key of the element.
+         *
+         * @param {CSN.Element} assoc if this is an association, the next step must be a foreign key of the element.
          */
-        function rejectNonFkAccess(element) {
-          if (!inNestedProjection && !inCalcElement && element.target) {
+        function rejectNonFkAccess(assoc) {
+          if (!inNestedProjection && !inCalcElement && assoc.target) {
             // only fk access in infix filter
             const nextStep = column.ref[i + 1]?.id || column.ref[i + 1]
             // no unmanaged assoc in infix filter path
-            if (!inExists && element.on)
+            if (!inExists && assoc.on)
               throw new Error(
-                `"${element.name}" in path "${column.ref
+                `"${assoc.name}" in path "${column.ref
                   .map(idOnly)
                   .join('.')}" must not be an unmanaged association`
               )
-            // no non-fk traversal in infix filter
-            if (nextStep && element.foreignKeys && !(nextStep in element.foreignKeys))
-              throw new Error(`Only foreign keys of "${element.name}" can be accessed in infix filter`)
+            // no non-fk traversal in infix filter in non-exists path
+            if (nextStep && !assoc.on && !isForeignKeyOf(nextStep, assoc))
+              throw new Error(`Only foreign keys of "${assoc.name}" can be accessed in infix filter`)
           }
         }
       })
@@ -723,7 +723,7 @@ function infer(originalQuery, model = cds.context?.model || cds.model) {
           if (inlineCol === '*') {
             const wildCardElements = {}
             // either the `.elements´ of the struct or the `.elements` of the assoc target
-            const leafLinkElements = $leafLink.definition.elements || $leafLink.definition._target.elements
+            const leafLinkElements = $leafLink.definition._target?.elements || $leafLink.definition.elements
             Object.entries(leafLinkElements).forEach(([k, v]) => {
               const name = namePrefix ? `${namePrefix}_${k}` : k
               // if overwritten/excluded omit from wildcard elements
@@ -1130,6 +1130,20 @@ function infer(originalQuery, model = cds.context?.model || cds.model) {
     }, '')
   }
 }
+/**
+ * Returns true if e is a foreign key of assoc.
+ * this function is also compatible with unfolded csn (UCSN),
+ * where association do not have foreign keys anymore.
+ *
+ * @param {*} e 
+ * @param {*} assoc
+ * @returns 
+ */
+function isForeignKeyOf(e, assoc) {
+  if(!assoc.isAssociation) return false
+  if(assoc.foreignKeys) return e in assoc.foreignKeys
+  return assoc.elements && e in assoc.elements
+}
 const idOnly = ref => ref.id || ref
 
 module.exports = infer

db-service/lib/infer/join-tree.js

@@ -200,9 +200,11 @@ class JoinTree {
           }
           child.$refLink.alias = this.addNextAvailableTableAlias($refLink.alias, outerQueries)
         }
-
-        const foreignKeys = node.$refLink?.definition.foreignKeys
-        if (node.$refLink && (!foreignKeys || !(child.$refLink.alias in foreignKeys)))
+        //> REVISIT: remove fallback once UCSN is standard
+        const elements =
+          node.$refLink?.definition.isAssociation &&
+          (node.$refLink.definition.elements || node.$refLink.definition.foreignKeys)
+        if (node.$refLink && (!elements || !(child.$refLink.alias in elements)))
           // foreign key access
           node.$refLink.onlyForeignKeyAccess = false
 

db-service/test/cds-infer/negative.test.js

@@ -117,7 +117,7 @@ describe('negative', () => {
       let query = CQL`SELECT from bookshop.Books:name { * }` // name does not exist
       expect(() => _inferred(query)).to.throw(/No association “name” in entity “bookshop.Books”/)
       let fromEndsWithScalar = CQL`SELECT from bookshop.Books:title { * }`
-      expect(() => _inferred(fromEndsWithScalar)).to.throw(/No association “title” in entity “bookshop.Books”/)
+      expect(() => _inferred(fromEndsWithScalar)).to.throw(/Query source must be a an entity or an association/)
     })
 
     // queries with multiple sources are not supported for cqn4sql transformation  (at least for now)

db-service/test/cqn2sql/insert.test.js

@@ -5,7 +5,8 @@ const cds = require('@sap/cds/lib')
 const cqn2sql = require('../../lib/cqn2sql')
 
 beforeAll(async () => {
-  cds.model = await cds.load(__dirname + '/testModel').then(cds.compile.for.nodejs)
+  let model = await cds.load(__dirname + '/testModel').then(cds.linked)
+  cds.model = cds.compile.for.nodejs(JSON.parse(JSON.stringify(model)))
 })
 
 describe('insert', () => {

db-service/test/cqn4sql/DELETE.test.js

@@ -70,7 +70,7 @@ describe('DELETE', () => {
     expect(query.DELETE).to.deep.equal(expected.DELETE)
   })
   it('DELETE with where exists expansion and path expression', () => {
-    cds.model = cds.compile.for.nodejs(cds.model)
+    cds.model = cds.compile.for.nodejs(JSON.parse(JSON.stringify(cds.model)))
     const { DELETE } = cds.ql
     let d = DELETE.from('bookshop.Books:author').where(`books.title = 'Harry Potter'`)
     const query = cqn4sql(d)

db-service/test/cqn4sql/calculated-elements.test.js

@@ -680,7 +680,7 @@ describe('Unfolding calculated elements in select list', () => {
     // at the leaf of a where exists path, there must be an association
     // calc elements can't end in an association, hence this does not work, yet.
     expect(() => cqn4sql(CQL`SELECT from booksCalc.Books:youngAuthorName { ID }`, model)).to.throw(
-      'No association “youngAuthorName” in entity “booksCalc.Books”',
+      'Query source must be a an entity or an association',
     )
   })
 

db-service/test/cqn4sql/column.element.test.js

@@ -69,7 +69,7 @@ describe('assign element onto columns with flat model', () => {
   let model
   beforeAll(async () => {
     model = cds.model = await cds.load('db/schema').then(cds.linked)
-    model = cds.compile.for.nodejs(model)
+    model = cds.compile.for.nodejs(JSON.parse(JSON.stringify(model)))
   })
 
   it('foreign key is adjacent to its association in flat model', () => {
@@ -131,8 +131,14 @@ describe('assign element onto columns with flat model', () => {
     expect(query).to.deep.eql(expected)
     expect(query.SELECT.columns[0]).to.have.property('element').that.eqls(AssocWithStructuredKey.elements.ID)
     // foreign key is part of flat model
-    expect(query.SELECT.columns[1]).to.have.property('element').that.eqls(AssocWithStructuredKey.elements.toStructuredKey_struct_mid_leaf)
-    expect(query.SELECT.columns[2]).to.have.property('element').that.eqls(AssocWithStructuredKey.elements.toStructuredKey_struct_mid_anotherLeaf)
+    if(model.meta.unfolded) { //> REVISIT: Remove once unfolded csn is standard
+      expect(query.SELECT.columns[1]).to.have.property('element').that.eqls(AssocWithStructuredKey.elements.toStructuredKey_struct_mid_leaf.__proto__)
+      expect(query.SELECT.columns[2]).to.have.property('element').that.eqls(AssocWithStructuredKey.elements.toStructuredKey_struct_mid_anotherLeaf.__proto__)
+    } else {
+      expect(query.SELECT.columns[1]).to.have.property('element').that.eqls(AssocWithStructuredKey.elements.toStructuredKey_struct_mid_leaf)
+      expect(query.SELECT.columns[2]).to.have.property('element').that.eqls(AssocWithStructuredKey.elements.toStructuredKey_struct_mid_anotherLeaf)  
+    }
+
 
     expect(query.SELECT.columns[3]).to.have.property('element').that.eqls(AssocWithStructuredKey.elements.toStructuredKey_second)
   })

db-service/test/cqn4sql/expand.test.js

@@ -306,7 +306,7 @@ describe('Unfold expands on associations to special subselects', () => {
     `
     // seems to only happen with the `for.nodejs(…)` compiled model
     expandQuery.SELECT.localized = true
-    expect(JSON.parse(JSON.stringify(cqn4sql(expandQuery, cds.compile.for.nodejs(model))))).to.deep.equal(expected)
+    expect(JSON.parse(JSON.stringify(cqn4sql(expandQuery, cds.compile.for.nodejs(JSON.parse(JSON.stringify(model))))))).to.deep.equal(expected)
   })
 
   it('add where exists <assoc> shortcut to expand subquery where condition', () => {
@@ -1021,7 +1021,7 @@ describe('Unfold expands on associations to special subselects', () => {
         }
       }
     `
-    let transformed = cqn4sql(q, cds.compile.for.nodejs(model))
+    let transformed = cqn4sql(q, cds.compile.for.nodejs(JSON.parse(JSON.stringify(model))))
     expect(JSON.parse(JSON.stringify(transformed))).to.deep.eql(CQL`
       SELECT from Collaborations as Collaborations {
         Collaborations.id,
@@ -1121,7 +1121,7 @@ describe('expand on structure part II', () => {
         (SELECT department.id, department.name from Department as department where Employee.department_id = department.id) as department,
         (SELECT assets.id, assets.descr from Assets as assets where Employee.id = assets.owner_id) as assets
     }`
-    expect(JSON.parse(JSON.stringify(cqn4sql(expandQuery, cds.compile.for.nodejs(model))))).to.eql(expected)
+    expect(JSON.parse(JSON.stringify(cqn4sql(expandQuery, cds.compile.for.nodejs(JSON.parse(JSON.stringify(model))))))).to.eql(expected)
   })
 
   it('multi expand with star but foreign key does not survive in structured mode', () => {

db-service/test/cqn4sql/flattening.test.js

@@ -164,7 +164,7 @@ describe('Flattening', () => {
             genre,
             genre_ID
           }`,
-        cds.linked(cds.compile.for.odata(model)),
+        cds.linked(cds.compile.for.nodejs(JSON.parse(JSON.stringify(model)))),
       )
       expect(query).to.deep.eql(CQL`SELECT from bookshop.Books as Books {
             Books.ID,

db-service/test/cqn4sql/search.test.js

@@ -94,7 +94,7 @@ describe('Replace attribute search by search predicate', () => {
       CQL`
       SELECT from bookshop.Books as Books
         left join bookshop.Authors as author on author.ID = Books.author_ID
-        left join bookshop.Books as books2 on books2.author_ID = author.ID
+        left join bookshop.Books as books2 on  books2.author_ID = author.ID
       {
         Books.ID,
         books2.title as authorsBook

db-service/test/cqn4sql/table-alias.test.js

@@ -1002,7 +1002,7 @@ describe('table alias access', () => {
       }
       expect(error.message).to.match(/author_ID/)
 
-      const nodeModel = cds.compile.for.nodejs(model)
+      const nodeModel = cds.compile.for.nodejs(JSON.parse(JSON.stringify(model)))
       const repeat = cqn4sql(resultCopy, nodeModel)
 
       // Ensure sure that the where clause does not change

db-service/test/cqn4sql/tupleExpansion.test.js

@@ -262,9 +262,9 @@ describe('Structural comparison', () => {
   })
 
   it('expands comparison also in exists subquery', () => {
-      const queryString = `SELECT from bookshop.AssocWithStructuredKey[toStructuredKey = null]:accessGroup { ID }`
-      let query = cqn4sql(CQL(queryString), model)
-      const expectedQueryString = `
+    const queryString = `SELECT from bookshop.AssocWithStructuredKey[toStructuredKey = null]:accessGroup { ID }`
+    let query = cqn4sql(CQL(queryString), model)
+    const expectedQueryString = `
         SELECT from bookshop.AccessGroups as accessGroup
         { accessGroup.ID }
         where exists (
@@ -275,7 +275,7 @@ describe('Structural comparison', () => {
               AssocWithStructuredKey.toStructuredKey_second                 = null
         )
       `
-      expect(query).to.deep.equal(CQL(expectedQueryString))
+    expect(query).to.deep.equal(CQL(expectedQueryString))
   })
 
   it('compare assocs with multiple keys', () => {
@@ -405,7 +405,7 @@ describe('Structural comparison', () => {
     })
   })
   it('Struct needs to be unfolded in on-condition of join', () => {
-    const queryString = `SELECT from bookshop.Unmanaged {
+    const query = CQL`SELECT from bookshop.Unmanaged {
       toSelf.field
     }`
 
@@ -415,8 +415,16 @@ describe('Structural comparison', () => {
       toSelf.field as toSelf_field
     }
     `
-    // const unfolded = cds.compile.for.nodejs(JSON.parse(JSON.stringify(model)))
-    const res = cqn4sql(CQL(queryString), model)
-      expect(res).to.eql(expected)
+    const unfolded = cds.compile.for.nodejs(JSON.parse(JSON.stringify(model)))
+    const structuredRes = cqn4sql(query, model)
+    const unfoldedRes = cqn4sql(query, unfolded)
+    //> REVISIT: remove fallback once UCSN is the new standard
+    if(unfolded.meta.unfolded) {
+      expect(structuredRes).to.eql(expected).to.eql(unfoldedRes)
+    } else {
+      // with odata csn, the on condition of the assoc is wrapped in xpr
+      expect(structuredRes.SELECT.from.on).to.eql(expected.SELECT.from.on).to.eql(unfoldedRes.SELECT.from.on[0].xpr)
+      expect(structuredRes.SELECT.columns).to.eql(expected.SELECT.columns).to.eql(unfoldedRes.SELECT.columns)
+    }
   })
 })

db-service/test/cqn4sql/where-exists.test.js

@@ -715,7 +715,7 @@ describe('EXISTS predicate in infix filter', () => {
       )
     `
     expect(() => {
-      cqn4sql(q, cds.compile.for.nodejs(model))
+      cqn4sql(q, cds.compile.for.nodejs(JSON.parse(JSON.stringify(model))))
     }).to.throw(/Only foreign keys of "participant" can be accessed in infix filter/)
   })
 })
@@ -1294,11 +1294,12 @@ describe('Path expressions in from combined with `exists` predicate', () => {
   })
 })
 
+
 describe('cap issue', () => {
   let model
   beforeAll(async () => {
     model = cds.model = await cds.load(__dirname + '/model/cap_issue').then(cds.linked)
-    model = cds.compile.for.nodejs(model)
+    model = cds.compile.for.nodejs(JSON.parse(JSON.stringify(model)))
   })
   it('MUST ... two EXISTS both on same path in where with real life example', () => {
     // make sure that in a localized scenario, all aliases