Adding support for glossarist model V2

lib/liquid/drops/concepts_drop.rb

@@ -3,9 +3,9 @@
 module Liquid
   module Drops
     class ConceptsDrop < Liquid::Drop
-      # rubocop:disable Lint/MissingSuper
-      NON_LANGUAGE_FIELDS = %w[term termid groups].freeze
+      NON_LANGUAGE_FIELDS = %w[identifier localized_concepts groups term].freeze
 
+      # rubocop:disable Lint/MissingSuper
       def initialize(managed_concept_collection, filters = {})
         @concepts_collection = managed_concept_collection
         @concepts_map = {}
@@ -36,42 +36,63 @@ def filtered_concepts(concepts_collection, filters)
         sort_filter = concept_filters.delete('sort_by')
         group_filter = concept_filters.delete('group')
 
-        concepts = concepts_collection.to_h["managed_concepts"].map do |concept|
-          filtered_concept = concept.dup
-          filtered_concept.each do |field, concept_hash|
-            next if NON_LANGUAGE_FIELDS.include?(field)
-
-            unless allowed_language?(field, language_filter)
-              filtered_concept.delete(field)
-              next
-            end
+        concepts = concepts_collection.map do |concept|
+          filtered_concept = concept.to_h["data"]
+          filtered_concept["term"] = concept.default_designation
 
-            concept_filters.each do |name, value|
-              fields = extract_nested_field_names(name)
-              if fields.last.start_with?("start_with")
-                value = fields.last.gsub(/start_with\(([^\)]*)\)/, '\1')
-                fields = fields[0..-2]
-
-                unless filtered_concept.dig(*fields).start_with?(value)
-                  filtered_concept.delete(field)
-                end
-              elsif filtered_concept.dig(*fields) != value
-                filtered_concept.delete(field)
-              end
-            end
-          end
+          filtered_concept = filtered_concept.merge(
+            extract_localized_concepts(
+              concept,
+              language_filter,
+            ),
+          )
 
-          if filtered_concept.keys & NON_LANGUAGE_FIELDS == filtered_concept.keys
-            nil
-          else
+          if retain_concept?(filtered_concept, concept_filters)
             filtered_concept
+          else
+            nil
           end
         end.compact
 
         apply_group_filter(concepts, group_filter)
         apply_sort_filter(concepts, sort_filter)
       end
 
+      def extract_localized_concepts(concept, languages)
+        localized_concepts = {}
+
+        if !languages || languages.empty?
+          concept.localized_concepts.each do |lang, localized_concept_uuid|
+            localized_concepts[lang] = concept.localizations[lang].to_h["data"]
+          end
+        else
+          languages.split(",").each do |lang|
+            localization = concept.localizations[lang]&.to_h&.dig("data")
+            localized_concepts[lang] = localization if localization
+          end
+        end
+
+        localized_concepts
+      end
+
+      def retain_concept?(filtered_concept, concept_filters)
+        concept_filters.each do |name, value|
+          fields = extract_nested_field_names(name)
+          if fields.last.start_with?("start_with")
+            value = fields.last.gsub(/start_with\(([^\)]*)\)/, '\1')
+            fields = fields[0..-2]
+
+            unless filtered_concept.dig(*fields).start_with?(value)
+              return false
+            end
+          elsif filtered_concept.dig(*fields) != value
+            return false
+          end
+        end
+
+        filtered_concept.keys & NON_LANGUAGE_FIELDS != filtered_concept.keys
+      end
+
       def apply_sort_filter(concepts, sort_by)
         return concepts unless sort_by
 

metanorma-plugin-glossarist.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "asciidoctor", "~> 2.0.0"
-  spec.add_dependency "glossarist"
+  spec.add_dependency "glossarist", "~> 2.0.1"
   spec.add_dependency "liquid", "~> 5"
 
   spec.add_development_dependency "pry"

spec/fixtures/dataset-glossarist-v2/concept/1c29256d-b5a7-5de1-b3e1-da4db6199e70.yaml

@@ -0,0 +1,7 @@
+---
+data:
+  identifier: 3.1.1.5
+  localized_concepts:
+    eng: 13d148ec-e30a-5c0e-b726-f6093d72d519
+  groups:
+  - bar

spec/fixtures/dataset-glossarist-v2/concept/208f7ac5-d12a-582f-b475-6b9b9f92dbdc.yaml

@@ -0,0 +1,7 @@
+---
+data:
+  identifier: 3.1.1.1
+  localized_concepts:
+    eng: ccceb779-456f-5115-926d-44487324d4bb
+  groups:
+  - foo

spec/fixtures/dataset-glossarist-v2/concept/dcd4415f-a6e5-5173-8de5-9e8827eaeb01.yaml

@@ -0,0 +1,7 @@
+---
+data:
+  identifier: 3.1.1.3
+  localized_concepts:
+    eng: c8763b71-b695-5117-9309-c7d998f63ad7
+  groups:
+  - foo

spec/fixtures/dataset-glossarist-v2/concept/ef715dd8-3886-5833-bf4c-f9a70feb430b.yaml

@@ -0,0 +1,8 @@
+---
+data:
+  identifier: 3.1.1.6
+  localized_concepts:
+    eng: e797ce71-7aa5-5ac4-97be-9212be802765
+    deu: aa48658e-f44a-551d-a60e-d0c620ef6328
+  groups:
+  - bar

spec/fixtures/dataset1/concept-3.1.1.5.yaml → spec/fixtures/dataset-glossarist-v2/localized_concept/13d148ec-e30a-5c0e-b726-f6093d72d519.yaml

@@ -1,9 +1,4 @@
----
-termid: 3.1.1.5
-term: biological entity
-groups:
-- bar
-eng:
+data:
   terms:
   - type: expression
     normative_status: preferred

spec/fixtures/dataset-glossarist-v2/localized_concept/aa48658e-f44a-551d-a60e-d0c620ef6328.yaml

@@ -0,0 +1,18 @@
+data:
+  terms:
+  - type: expression
+    normative_status: preferred
+    designation: persoon
+  definition:
+  - content: "biologische entiteit dat is een mens
+      wezen"
+  notes: []
+  examples: []
+  language_code: deu
+  entry_status: valid
+  sources:
+  - type: authoritative
+    origin:
+      ref: ISO/TS 14812:2022
+      clause: 3.1.1.6
+      link: https://www.iso.org/standard/79779.html

spec/fixtures/dataset1/concept-3.1.1.3.yaml → spec/fixtures/dataset-glossarist-v2/localized_concept/c8763b71-b695-5117-9309-c7d998f63ad7.yaml

@@ -1,9 +1,4 @@
----
-termid: 3.1.1.3
-term: material entity
-groups:
-- foo
-eng:
+data:
   terms:
   - type: expression
     normative_status: preferred

spec/fixtures/dataset1/concept-3.1.1.1.yaml → spec/fixtures/dataset-glossarist-v2/localized_concept/ccceb779-456f-5115-926d-44487324d4bb.yaml

@@ -1,9 +1,4 @@
----
-termid: 3.1.1.1
-term: entity
-groups:
-- foo
-eng:
+data:
   terms:
   - type: expression
     normative_status: preferred

spec/fixtures/dataset1/concept-3.1.1.6.yaml → spec/fixtures/dataset-glossarist-v2/localized_concept/e797ce71-7aa5-5ac4-97be-9212be802765.yaml

@@ -1,9 +1,4 @@
----
-termid: 3.1.1.6
-term: person
-groups:
-- bar
-eng:
+data:
   terms:
   - type: expression
     normative_status: preferred

spec/fixtures/invalid_dataset/concept/concept-3.1.1.1.yaml

@@ -0,0 +1,10 @@
+---
+data:
+  identifier: 3.1.1.5
+  localized_concepts:
+    eng: lc3
+  groups:
+  - bar
+
+  [.source]
+  <<ISO/TS 21308-4:2007>>

spec/metanorma/plugin/glossarist/dataset_preprocessor_spec.rb

@@ -12,7 +12,7 @@
               some text before glossarist block
 
               === Section 1
-              [glossarist,./spec/fixtures/dataset1,concepts]
+              [glossarist,./spec/fixtures/dataset-glossarist-v2,concepts]
               ----
               ==== {{ concepts['entity'].term }}
 
@@ -50,7 +50,7 @@
                 some text before glossarist block
 
                 === Section 1
-                [glossarist,./spec/fixtures/dataset1,filter='lang=ara',concepts]
+                [glossarist,./spec/fixtures/dataset-glossarist-v2,filter='lang=ara',concepts]
                 ----
                 {% for concept in concepts %}
                 ==== {{ concept.term }}
@@ -72,6 +72,47 @@
 
 
 
+                some text after glossarist block
+              OUTPUT
+            end
+
+            it "should render correct output" do
+              expect(subject.process(document, reader).source).to eq(expected_output)
+            end
+          end
+
+          describe "filter='lang=deu'" do
+            let(:reader) do
+              Asciidoctor::Reader.new <<~TEMPLATE
+                some text before glossarist block
+
+                === Section 1
+                [glossarist,./spec/fixtures/dataset-glossarist-v2,filter='lang=deu',concepts]
+                ----
+                {% for concept in concepts %}
+                ==== {{ concept.term }}
+
+                {{ concept.deu.definition[0].content }}
+                {% endfor %}
+                ----
+
+                some text after glossarist block
+              TEMPLATE
+            end
+
+            let(:expected_output) do
+              <<~OUTPUT.strip
+                some text before glossarist block
+
+                === Section 1
+
+
+                ==== person
+
+                biologische entiteit dat is een mens wezen
+
+
+
                 some text after glossarist block
               OUTPUT
             end
@@ -87,7 +128,7 @@
                 some text before glossarist block
 
                 === Section 1
-                [glossarist,./spec/fixtures/dataset1,filter='sort_by=term',concepts]
+                [glossarist,./spec/fixtures/dataset-glossarist-v2,filter='sort_by=term',concepts]
                 ----
                 {% for concept in concepts %}
                 ==== {{ concept.term }}
@@ -147,7 +188,7 @@
                 some text before glossarist block
 
                 === Section 1
-                [glossarist,./spec/fixtures/dataset1,filter='group=foo;sort_by=term',concepts]
+                [glossarist,./spec/fixtures/dataset-glossarist-v2,filter='group=foo;sort_by=term',concepts]
                 ----
                 {% for concept in concepts %}
                 ==== {{ concept.term }}
@@ -192,7 +233,7 @@
                 some text before glossarist block
 
                 === Section 1
-                [glossarist,./spec/fixtures/dataset1,filter='lang=eng;eng.terms.0.designation=entity',concepts]
+                [glossarist,./spec/fixtures/dataset-glossarist-v2,filter='lang=eng;eng.terms.0.designation=entity',concepts]
                 ----
                 {%- for concept in concepts -%}
                 ==== {{ concept.term }}
@@ -229,7 +270,7 @@
                 some text before glossarist block
 
                 === Section 1
-                [glossarist,./spec/fixtures/dataset1,filter='eng.terms.0.designation.start_with(enti)',concepts]
+                [glossarist,./spec/fixtures/dataset-glossarist-v2,filter='eng.terms.0.designation.start_with(enti)',concepts]
                 ----
                 {%- for concept in concepts -%}
                 ==== {{ concept.term }}
@@ -265,7 +306,7 @@
       context "[load dataset]" do
         let(:reader) do
           Asciidoctor::Reader.new <<~TEMPLATE
-            :glossarist-dataset: dataset1:./spec/fixtures/dataset1
+            :glossarist-dataset: dataset1:./spec/fixtures/dataset-glossarist-v2
 
             === Render Section
             {{ dataset1['entity']['eng'].definition[0].content }}
@@ -287,7 +328,7 @@
       context "[render concept]" do
         let(:reader) do
           Asciidoctor::Reader.new <<~TEMPLATE
-            :glossarist-dataset: dataset1:./spec/fixtures/dataset1
+            :glossarist-dataset: dataset1:./spec/fixtures/dataset-glossarist-v2
 
             === Render Section
             glossarist::render[dataset1, entity]
@@ -326,7 +367,7 @@
       context "[render bibliography entry]" do
         let(:reader) do
           Asciidoctor::Reader.new <<~TEMPLATE
-            :glossarist-dataset: dataset1:./spec/fixtures/dataset1
+            :glossarist-dataset: dataset1:./spec/fixtures/dataset-glossarist-v2
 
             glossarist::render_bibliography_entry[dataset1, entity]
           TEMPLATE
@@ -346,7 +387,7 @@
       context "[render bibliography]" do
         let(:reader) do
           Asciidoctor::Reader.new <<~TEMPLATE
-            :glossarist-dataset: dataset1:./spec/fixtures/dataset1
+            :glossarist-dataset: dataset1:./spec/fixtures/dataset-glossarist-v2
 
             glossarist::render_bibliography[dataset1]
           TEMPLATE