From 419a8fd092e32413ebd8cdc88226f2251e922d2c Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Thu, 9 Jan 2020 15:00:19 -0600 Subject: [PATCH 001/163] moves code out of bin elasticsearch files and into module in order to achieve support for multiple ES schemas - adds configuration for different schema locations - moves code from executables into Datura::Elasticsearch module - Datura::Options combines settings into schema path --- bin/admin_es_create_index | 16 +- bin/admin_es_delete_index | 11 +- bin/es_alias_add | 25 +-- bin/es_alias_delete | 14 +- bin/es_alias_list | 9 +- bin/es_clear_index | 89 +--------- bin/es_get_schema | 16 +- bin/es_set_schema | 19 +- lib/config/api_schema.yml | 194 --------------------- lib/config/es_api_schemas/1.0.yml | 191 ++++++++++++++++++++ lib/config/es_api_schemas/2.0.yml | 279 ++++++++++++++++++++++++++++++ lib/config/public.yml | 18 +- lib/datura.rb | 1 + lib/datura/elasticsearch.rb | 22 +++ lib/datura/elasticsearch/alias.rb | 55 ++++++ lib/datura/elasticsearch/data.rb | 94 ++++++++++ lib/datura/elasticsearch/index.rb | 77 +++++++++ lib/datura/options.rb | 18 ++ lib/datura/requirer.rb | 4 +- 19 files changed, 777 insertions(+), 375 deletions(-) delete mode 100644 lib/config/api_schema.yml create mode 100644 lib/config/es_api_schemas/1.0.yml create mode 100644 lib/config/es_api_schemas/2.0.yml create mode 100644 lib/datura/elasticsearch.rb create mode 100644 lib/datura/elasticsearch/alias.rb create mode 100644 lib/datura/elasticsearch/data.rb create mode 100644 lib/datura/elasticsearch/index.rb diff --git a/bin/admin_es_create_index b/bin/admin_es_create_index index e27997e18..94bee8ea9 100755 --- a/bin/admin_es_create_index +++ b/bin/admin_es_create_index @@ -2,18 +2,10 @@ require "datura" -params = Datura::Parser.es_create_delete_index -options = Datura::Options.new(params).all - -put_url = File.join(options["es_path"], "#{options["es_index"]}?pretty=true") -get_url = File.join(options["es_path"], "_cat", "indices?v&pretty=true") - begin - # TODO if we want to add any default settings to the new index, - # we can do that with the payload and then use rest-client again instead of exec - # however, rest-client appears to require a payload and won't allow simple "PUT" with none - puts "Creating new ES index: #{put_url}" - exec("curl -XPUT #{put_url}") + es = Datura::Elasticsearch::Index.new + es.create + es.set_schema rescue => e - puts "Error: #{e.inspect}" + puts e end diff --git a/bin/admin_es_delete_index b/bin/admin_es_delete_index index 76299afd4..8de5fbb06 100755 --- a/bin/admin_es_delete_index +++ b/bin/admin_es_delete_index @@ -1,15 +1,10 @@ #!/usr/bin/env ruby require "datura" -require "rest-client" - -params = Datura::Parser.es_create_delete_index -options = Datura::Options.new(params).all - -url = File.join(options["es_path"], "#{options["es_index"]}?pretty=true") begin - puts JSON.parse(RestClient.delete(url)) + es = Datura::Elasticsearch::Index.new + es.delete rescue => e - puts "Error with request, check that index exists before deleting: #{e}" + puts e end diff --git a/bin/es_alias_add b/bin/es_alias_add index e9c3f74d3..7f028dfe8 100755 --- a/bin/es_alias_add +++ b/bin/es_alias_add @@ -2,29 +2,8 @@ require "datura" -require "json" -require "rest-client" - -params = Datura::Parser.es_alias_add -options = Datura::Options.new(params).all - -ali = options["alias"] -idx = options["index"] -url = File.join(options["es_path"], "_aliases") - -data = { - actions: [ - { remove: { alias: ali, index: "_all" } }, - { add: { alias: ali, index: idx } } - ] -} - begin - res = RestClient.post(url, data.to_json, { content_type: :json }) - puts "Results of setting alias #{ali} to index #{idx}" - puts res - list = JSON.parse(RestClient.get(url)) - puts "\nAll aliases: #{JSON.pretty_generate(list)}" + puts Datura::Elasticsearch::Alias.add rescue => e - puts "Error: #{e.response}" + puts e end diff --git a/bin/es_alias_delete b/bin/es_alias_delete index d12a574d2..1317c39dc 100755 --- a/bin/es_alias_delete +++ b/bin/es_alias_delete @@ -2,12 +2,8 @@ require "datura" -require "json" -require "rest-client" - -params = Datura::Parser.es_alias_delete -options = Datura::Options.new(params).all -url = File.join(options["es_path"], options["index"], "_alias", options["alias"]) - -res = JSON.parse(RestClient.delete(url)) -puts JSON.pretty_generate(res) +begin + puts Datura::Elasticsearch::Alias.delete +rescue => e + puts e +end diff --git a/bin/es_alias_list b/bin/es_alias_list index d37691626..ba6df4d7e 100755 --- a/bin/es_alias_list +++ b/bin/es_alias_list @@ -2,11 +2,4 @@ require "datura" -require "json" -require "rest-client" - -options = Datura::Options.new({}).all -url = File.join(options["es_path"], "_aliases") - -res = JSON.parse(RestClient.get(url)) -puts JSON.pretty_generate(res) +puts Datura::Elasticsearch::Alias.list diff --git a/bin/es_clear_index b/bin/es_clear_index index 2890f6230..6cd7b8740 100755 --- a/bin/es_clear_index +++ b/bin/es_clear_index @@ -2,89 +2,8 @@ require "datura" -require "json" -require "rest-client" - -def confirm_basic(options, url) - # verify that the user is really sure about the index they're about to wipe - puts "Are you sure that you want to remove entries from" - puts " #{options["collection"]}'s #{options['environment']} environment?" - puts "url: #{url}" - puts "y/N" - answer = STDIN.gets.chomp - # boolean - return !!(answer =~ /[yY]/) -end - -def main - - # run the parameters through the option parser - params = Datura::Parser.clear_index_params - options = Datura::Options.new(params).all - if options["collection"] == "all" - clear_all(options) - else - clear_index(options) - end -end - -def build_data(options) - if options["regex"] - field = options["field"] || "identifier" - return { - "query" => { - "bool" => { - "must" => [ - { "regexp" => { field => options["regex"] } }, - { "term" => { "collection" => options["collection"] } } - ] - } - } - } - else - return { - "query" => { "term" => { "collection" => options["collection"] } } - } - end -end - -def clear_all(options) - puts "Please verify that you want to clear EVERY ENTRY from the ENTIRE INDEX\n\n" - puts "== FIELD / REGEX FILTERS NOT AVAILABLE FOR THIS OPTION, YOU'LL WIPE EVERYTHING ==\n\n" - puts "Seriously, you probably do not want to do this" - puts "Are you running this on something other than your local machine? RETHINK IT." - puts "Type: 'Yes I'm sure'" - confirm = STDIN.gets.chomp - if confirm == "Yes I'm sure" - url = "#{options["es_path"]}/#{options["es_index"]}/_doc/_delete_by_query?pretty=true" - post url, { "query" => { "match_all" => {} } } - else - puts "You typed '#{confirm}'. This is incorrect, exiting program" - exit - end -end - -def clear_index(options) - url = "#{options["es_path"]}/#{options["es_index"]}/_doc/_delete_by_query?pretty=true" - confirmation = confirm_basic(options, url) - - if confirmation - data = build_data(options) - post(url, data) - else - puts "come back anytime!" - exit - end +begin + Datura::Elasticsearch::Data.clear +rescue => e + puts e end - -def post(url, data={}) - begin - puts "clearing from #{url}: #{data.to_json}" - res = RestClient.post(url, data.to_json, {:content_type => :json}) - puts res.body - rescue => e - puts "error posting to ES: #{e.response}" - end -end - -main diff --git a/bin/es_get_schema b/bin/es_get_schema index 14e41b847..1326d5e48 100755 --- a/bin/es_get_schema +++ b/bin/es_get_schema @@ -2,19 +2,9 @@ require "datura" -require "json" -require "rest-client" -require "yaml" - -params = Datura::Parser.es_set_schema_params -options = Datura::Options.new(params).all - begin - url = File.join(options["es_path"], options["es_index"], "_mapping", "_doc?pretty=true") - res = RestClient.get(url) - puts res.body - puts "environment: #{options["environment"]}" - puts "url: #{url}" + es = Datura::Elasticsearch::Index.new + es.get_schema rescue => e - puts "Error: #{e.response}" + puts e end diff --git a/bin/es_set_schema b/bin/es_set_schema index f40050016..6c461478d 100755 --- a/bin/es_set_schema +++ b/bin/es_set_schema @@ -2,22 +2,9 @@ require "datura" -require "json" -require "rest-client" -require "yaml" - -params = Datura::Parser.es_set_schema_params -options = Datura::Options.new(params).all -path = File.join(options["datura_dir"], options["es_schema_path"]) -schema = YAML.load_file(path) - begin - idx = options["es_index"] - - url = File.join(options["es_path"], options["es_index"], "_mapping", "_doc?pretty=true") - puts "environment: #{options["environment"]}" - puts "Setting schema: #{url}" - RestClient.put(url, schema.to_json, { :content_type => :json }) + es = Datura::Elasticsearch::Index.new + es.set_schema rescue => e - puts "Error: #{e.response}" + puts e end diff --git a/lib/config/api_schema.yml b/lib/config/api_schema.yml deleted file mode 100644 index 26f1e6ccb..000000000 --- a/lib/config/api_schema.yml +++ /dev/null @@ -1,194 +0,0 @@ -properties: - identifier: - type: keyword - identifier: - type: keyword - collection: - type: keyword - collection_desc: - type: keyword - uri: - type: keyword - uri_data: - type: keyword - uri_html: - type: keyword - data_type: - type: keyword - image_location: - type: keyword - image_id: - type: keyword - # TODO copy to text? - title: - type: keyword - title_sort: - type: keyword - # TODO copy to text? - alternative: - type: keyword - creator_sort: - type: keyword - creator: - type: nested - properties: - name: - # TODO copy into text? - type: keyword - id: - type: keyword - subjects: - type: keyword - # TODO not sure yet if for display or search - abstract: - type: keyword - # TODO copy to text? - description: - type: keyword - publisher: - type: keyword - contributor: - type: nested - properties: - name: - type: keyword - id: - type: keyword - role: - type: keyword - date: - type: date - format: "yyyy-MM-dd||epoch_millis" - # ignore_malformed: true - date_display: - type: keyword - date_not_before: - type: date - format: "yyyy-MM-dd||epoch_millis" - # ignore_malformed: true - date_not_after: - type: date - format: "yyyy-MM-dd||epoch_millis" - # ignore_malformed: true - type: - type: keyword - format: - type: keyword - medium: - type: keyword - extent: - type: keyword - language: - type: keyword - languages: - type: keyword - relation: - type: keyword - source: - type: keyword - recipient: - type: nested - properties: - name: - type: keyword - id: - type: keyword - role: - type: keyword - rights_holder: - type: keyword - rights: - type: keyword - rights_uri: - type: keyword - spatial: - type: nested - properties: - id: - type: keyword - # display title for entire location - title: - type: keyword - type: - type: keyword - # specific name of building, park, mountain, etc - place_name: - type: keyword - coordinates: - type: geo_point - city: - type: keyword - county: - type: keyword - country: - type: keyword - region: - type: keyword - state: - type: keyword - street: - type: keyword - postal_code: - type: keyword - person: - type: nested - properties: - name: - # TODO copy into text? - type: keyword - id: - type: keyword - role: - type: keyword - annotations_text: - type: text - analyzer: english - category: - type: keyword - subcategory: - type: keyword - topics: - type: keyword - keywords: - type: keyword - people: - type: keyword - places: - type: keyword - works: - type: keyword - text: - type: text - analyzer: english -dynamic_templates: - - date_fields: - match: "*_d" - mapping: - type: date - format: "yyyy-MM-dd||epoch_millis" - - integer_fields: - match: "*_i" - mapping: - type: integer - - keyword_fields: - match: "*_k" - mapping: - type: keyword - - text_fields: - match: "*_t" - mapping: - type: text - analyzer: english - # language fields are always text fields - # but specifying _t_ for clarity - # _t_en functionally the same as _t - - text_english: - match: "*_t_en" - mapping: - type: text - analyzer: english - - text_spanish: - match: "*_t_es" - mapping: - type: text - analyzer: spanish diff --git a/lib/config/es_api_schemas/1.0.yml b/lib/config/es_api_schemas/1.0.yml new file mode 100644 index 000000000..89747ddf5 --- /dev/null +++ b/lib/config/es_api_schemas/1.0.yml @@ -0,0 +1,191 @@ +# compatible with Apium v1.0 +mappings: + properties: + identifier: + type: keyword + identifier: + type: keyword + collection: + type: keyword + collection_desc: + type: keyword + uri: + type: keyword + uri_data: + type: keyword + uri_html: + type: keyword + data_type: + type: keyword + image_location: + type: keyword + image_id: + type: keyword + # TODO copy to text? + title: + type: keyword + title_sort: + type: keyword + # TODO copy to text? + alternative: + type: keyword + creator_sort: + type: keyword + creator: + type: nested + properties: + name: + # TODO copy into text? + type: keyword + id: + type: keyword + subjects: + type: keyword + # TODO not sure yet if for display or search + abstract: + type: keyword + # TODO copy to text? + description: + type: keyword + publisher: + type: keyword + contributor: + type: nested + properties: + name: + type: keyword + id: + type: keyword + role: + type: keyword + date: + type: date + format: "yyyy-MM-dd||epoch_millis" + # ignore_malformed: true + date_display: + type: keyword + date_not_before: + type: date + format: "yyyy-MM-dd||epoch_millis" + # ignore_malformed: true + date_not_after: + type: date + format: "yyyy-MM-dd||epoch_millis" + # ignore_malformed: true + type: + type: keyword + format: + type: keyword + medium: + type: keyword + extent: + type: keyword + language: + type: keyword + languages: + type: keyword + relation: + type: keyword + source: + type: keyword + recipient: + type: nested + properties: + name: + type: keyword + id: + type: keyword + role: + type: keyword + rights_holder: + type: keyword + rights: + type: keyword + rights_uri: + type: keyword + coverage-spatial: + type: nested + properties: + place_name: + # TODO copy into text? + type: keyword + coordinates: + type: geo_point + id: + type: keyword + city: + type: keyword + county: + type: keyword + country: + type: keyword + region: + type: keyword + state: + type: keyword + street: + type: keyword + postal_code: + type: keyword + person: + type: nested + properties: + name: + # TODO copy into text? + type: keyword + id: + type: keyword + role: + type: keyword + annotations_text: + type: text + analyzer: english + category: + type: keyword + subcategory: + type: keyword + topics: + type: keyword + keywords: + type: keyword + people: + type: keyword + places: + type: keyword + works: + type: keyword + text: + type: text + analyzer: english + dynamic_templates: + - date_fields: + match: "*_d" + mapping: + type: date + format: "yyyy-MM-dd||epoch_millis" + - integer_fields: + match: "*_i" + mapping: + type: integer + - keyword_fields: + match: "*_k" + mapping: + type: keyword + - text_fields: + match: "*_t" + mapping: + type: text + analyzer: english + # language fields are always text fields + # but specifying _t_ for clarity + # _t_en functionally the same as _t + - text_english: + match: "*_t_en" + mapping: + type: text + analyzer: english + - text_spanish: + match: "*_t_es" + mapping: + type: text + analyzer: spanish diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml new file mode 100644 index 000000000..c34155bad --- /dev/null +++ b/lib/config/es_api_schemas/2.0.yml @@ -0,0 +1,279 @@ +# compatible with Apium v2.0 +settings: + analysis: + char_filter: + escapes: + type: mapping + mappings: + - " => " + - " => " + - " => " + - " => " + - " => " + - " => " + - "- => " + - "& => " + - ": => " + - "; => " + - ", => " + - ". => " + - "$ => " + - "@ => " + - "~ => " + - "\" => " + - "' => " + - "[ => " + - "] => " + normalizer: + keyword_normalized: + type: custom + char_filter: + - escapes + filter: + - asciifolding + - lowercase +mappings: + properties: + identifier: + type: keyword + normalizer: keyword_normalized + collection: + type: keyword + normalizer: keyword_normalized + collection_desc: + type: keyword + normalizer: keyword_normalized + uri: + type: keyword + normalizer: keyword_normalized + uri_data: + type: keyword + normalizer: keyword_normalized + uri_html: + type: keyword + normalizer: keyword_normalized + data_type: + type: keyword + normalizer: keyword_normalized + image_location: + type: keyword + normalizer: keyword_normalized + image_id: + type: keyword + normalizer: keyword_normalized + # TODO copy to text? + title: + type: keyword + normalizer: keyword_normalized + title_sort: + type: keyword + normalizer: keyword_normalized + # TODO copy to text? + alternative: + type: keyword + normalizer: keyword_normalized + creator_sort: + type: keyword + normalizer: keyword_normalized + creator: + type: nested + properties: + name: + # TODO copy into text? + type: keyword + normalizer: keyword_normalized + id: + type: keyword + normalizer: keyword_normalized + subjects: + type: keyword + normalizer: keyword_normalized + # TODO not sure yet if for display or search + abstract: + type: keyword + normalizer: keyword_normalized + # TODO copy to text? + description: + type: keyword + normalizer: keyword_normalized + publisher: + type: keyword + normalizer: keyword_normalized + contributor: + type: nested + properties: + name: + type: keyword + normalizer: keyword_normalized + id: + type: keyword + normalizer: keyword_normalized + role: + type: keyword + normalizer: keyword_normalized + date: + type: date + format: "yyyy-MM-dd||epoch_millis" + # ignore_malformed: true + date_display: + type: keyword + normalizer: keyword_normalized + date_not_before: + type: date + format: "yyyy-MM-dd||epoch_millis" + # ignore_malformed: true + date_not_after: + type: date + format: "yyyy-MM-dd||epoch_millis" + # ignore_malformed: true + type: + type: keyword + normalizer: keyword_normalized + format: + type: keyword + normalizer: keyword_normalized + medium: + type: keyword + normalizer: keyword_normalized + extent: + type: keyword + normalizer: keyword_normalized + language: + type: keyword + normalizer: keyword_normalized + languages: + type: keyword + normalizer: keyword_normalized + relation: + type: keyword + normalizer: keyword_normalized + source: + type: keyword + normalizer: keyword_normalized + recipient: + type: nested + properties: + name: + type: keyword + normalizer: keyword_normalized + id: + type: keyword + normalizer: keyword_normalized + role: + type: keyword + normalizer: keyword_normalized + rights_holder: + type: keyword + normalizer: keyword_normalized + rights: + type: keyword + normalizer: keyword_normalized + rights_uri: + type: keyword + normalizer: keyword_normalized + coverage-spatial: + type: nested + properties: + place_name: + # TODO copy into text? + type: keyword + normalizer: keyword_normalized + coordinates: + type: geo_point + id: + type: keyword + normalizer: keyword_normalized + city: + type: keyword + normalizer: keyword_normalized + county: + type: keyword + normalizer: keyword_normalized + country: + type: keyword + normalizer: keyword_normalized + region: + type: keyword + normalizer: keyword_normalized + state: + type: keyword + normalizer: keyword_normalized + street: + type: keyword + normalizer: keyword_normalized + postal_code: + type: keyword + normalizer: keyword_normalized + person: + type: nested + properties: + name: + # TODO copy into text? + type: keyword + normalizer: keyword_normalized + id: + type: keyword + normalizer: keyword_normalized + role: + type: keyword + normalizer: keyword_normalized + annotations_text: + type: text + analyzer: english + category: + type: keyword + normalizer: keyword_normalized + subcategory: + type: keyword + normalizer: keyword_normalized + topics: + type: keyword + normalizer: keyword_normalized + keywords: + type: keyword + normalizer: keyword_normalized + people: + type: keyword + normalizer: keyword_normalized + places: + type: keyword + normalizer: keyword_normalized + works: + type: keyword + normalizer: keyword_normalized + text: + type: text + analyzer: english + dynamic_templates: + - date_fields: + match: "*_d" + mapping: + type: date + format: "yyyy-MM-dd||epoch_millis" + - integer_fields: + match: "*_i" + mapping: + type: integer + - keyword_fields: + match: "*_k" + mapping: + type: keyword + normalizer: keyword_normalized + - text_fields: + match: "*_t" + mapping: + type: text + analyzer: english + # language fields are always text fields + # but specifying _t_ for clarity + # _t_en functionally the same as _t + - text_english: + match: "*_t_en" + mapping: + type: text + analyzer: english + - text_spanish: + match: "*_t_es" + mapping: + type: text + analyzer: spanish diff --git a/lib/config/public.yml b/lib/config/public.yml index 3fff24731..486e85037 100644 --- a/lib/config/public.yml +++ b/lib/config/public.yml @@ -26,12 +26,18 @@ default: log_size: 32768000 # size of log file in bytes log_level: Logger::INFO # available levels: UNKNOWN, FATAL, ERROR, WARN, INFO, DEBUG - # SCHEMA LOCATION - # misleadingly, this is not currently overrideable per collection - # TODO make overrideable in es_set_schema and post - # or perhaps remove it from this config since it is not collection-specific - # in any sense of the word, except if working with an entirely separate ES index - es_schema_path: lib/config/api_schema.yml + # ELASTICSEARCH SCHEMA CONFIGURATION + # if es_schema_override is false, datura is base directory + # if es_schema_override is true, then host data repo is the base directory + # it is NOT recommended to set es_schema_override to true! + # if you need something outside of your data repo's directory, consider + # overridding the es_schema_path method in options.rb + es_schema_override: false + # path from base directory to schemas + es_schema_path: lib/config/es_api_schemas + # current version of the API (powered by Elasticsearch) + # this setting determines which of the schemas will be used + api_version: "override with value like '1.0' or '2.0'" # RESOURCE LOCATIONS data_base: https://cdrhmedia.unl.edu # xml, csv, html snippets, etc diff --git a/lib/datura.rb b/lib/datura.rb index 5f46a4d55..3ff9278b2 100644 --- a/lib/datura.rb +++ b/lib/datura.rb @@ -1,5 +1,6 @@ require "datura/version" require "datura/data_manager" +require "datura/elasticsearch" module Datura diff --git a/lib/datura/elasticsearch.rb b/lib/datura/elasticsearch.rb new file mode 100644 index 000000000..fecba15fc --- /dev/null +++ b/lib/datura/elasticsearch.rb @@ -0,0 +1,22 @@ +require_relative './helpers.rb' +require_relative './options.rb' + +require "json" +require "rest-client" +require "yaml" + +module Datura::Elasticsearch + + # clear data from the index (leaves index schema intact) + module Data + end + + # manage the aliases used to refer to specific indexes + module Alias + end + + # manage the creation / deletion / schema configuration of indexes + class Index + end + +end diff --git a/lib/datura/elasticsearch/alias.rb b/lib/datura/elasticsearch/alias.rb new file mode 100644 index 000000000..177ee14d1 --- /dev/null +++ b/lib/datura/elasticsearch/alias.rb @@ -0,0 +1,55 @@ +require "json" +require "rest-client" + +require_relative "./../elasticsearch.rb" + +module Datura::Elasticsearch::Alias + + def self.add + params = Datura::Parser.es_alias_add + options = Datura::Options.new(params).all + + ali = options["alias"] + idx = options["index"] + + base_url = File.join(options["es_path"], "_aliases") + + data = { + actions: [ + { remove: { alias: ali, index: "_all" } }, + { add: { alias: ali, index: idx } } + ] + } + RestClient.post(base_url, data.to_json, { content_type: :json }) { |res, req, result| + if result.code == "200" + puts res + puts "Successfully added alias #{ali}. Current alias list:" + puts list + else + raise "#{result.code} error managing aliases: #{res}" + end + } + end + + def self.delete + params = Datura::Parser.es_alias_add + options = Datura::Options.new(params).all + + ali = options["alias"] + idx = options["index"] + + url = File.join(options["es_path"], idx, "_alias", ali) + + res = JSON.parse(RestClient.delete(url)) + puts JSON.pretty_generate(res) + list + end + + def self.list + options = Datura::Options.new({}).all + + res = RestClient.get(File.join(options["es_path"], "_aliases")) + JSON.pretty_generate(JSON.parse(res)) + end + +end diff --git a/lib/datura/elasticsearch/data.rb b/lib/datura/elasticsearch/data.rb new file mode 100644 index 000000000..5deedadb1 --- /dev/null +++ b/lib/datura/elasticsearch/data.rb @@ -0,0 +1,94 @@ +require "json" +require "rest-client" + +require_relative "./../elasticsearch.rb" + +module Datura::Elasticsearch::Data + + def self.clear + # run the parameters through the option parser + params = Datura::Parser.clear_index_params + options = Datura::Options.new(params).all + if options["collection"] == "all" + self.clear_all(options) + else + self.clear_index(options) + end + end + + private + + def self.build_clear_data(options) + if options["regex"] + field = options["field"] || "identifier" + { + "query" => { + "bool" => { + "must" => [ + { "regexp" => { field => options["regex"] } }, + { "term" => { "collection" => options["collection"] } } + ] + } + } + } + else + { + "query" => { "term" => { "collection" => options["collection"] } } + } + end + end + + def self.clear_all(options) + puts "Please verify that you want to clear EVERY ENTRY from the ENTIRE INDEX\n\n" + puts "== FIELD / REGEX FILTERS NOT AVAILABLE FOR THIS OPTION, YOU'LL WIPE EVERYTHING ==\n\n" + puts "Running this on something other than your computer's localhost? DON'T." + puts "Type: 'Yes I'm sure'" + confirm = STDIN.gets.chomp + if confirm == "Yes I'm sure" + url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") + json = { "query" => { "match_all" => {} } } + RestClient.post(url, json.to_json, { content_type: :json }) { |res, req, result| + if result.code == "200" + puts res + else + raise "#{result.code} error when clearing entire index: #{res}" + end + } + else + puts "You typed '#{confirm}'. This is incorrect, exiting program" + exit + end + end + + def self.clear_index(options) + url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") + confirmation = self.confirm_clear(options, url) + + if confirmation + data = self.build_clear_data(options) + RestClient.post(url, data.to_json, { content_type: :json }) { |res, req, result| + if result.code == "200" + puts res + else + raise "#{result.code} error when clearing index: #{res}" + end + } + else + puts "come back anytime!" + exit + end + end + + def self.confirm_clear(options, url) + # verify that the user is really sure about the index they're about to wipe + puts "Are you sure that you want to remove entries from" + puts " #{options["collection"]}'s #{options['environment']} environment?" + puts "url: #{url}" + puts "y/N" + answer = STDIN.gets.chomp + # boolean + !!(answer =~ /[yY]/) + end + + +end diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb new file mode 100644 index 000000000..1065eeff2 --- /dev/null +++ b/lib/datura/elasticsearch/index.rb @@ -0,0 +1,77 @@ +require "json" +require "rest-client" +require "yaml" + +require_relative "./../elasticsearch.rb" + +class Datura::Elasticsearch::Index + + def initialize + params = Datura::Parser.es_create_delete_index + @options = Datura::Options.new(params).all + + @base_url = File.join(@options["es_path"], @options["es_index"]) + @mapping_url = File.join(@base_url, "_mapping", "_doc?pretty=true") + @index_url = "#{@base_url}?pretty=true" + + # yaml settings (if exist) and mappings + @schema = YAML.load_file(@options["es_schema"]) + end + + def create + json = @schema["settings"].to_json + puts "Creating ES index for API version #{@options["api_version"]}: #{@index_url}" + + if json && json != "null" + RestClient.put(@index_url, json, { content_type: :json }) { |res, req, result| + if result.code == "200" + puts res + else + raise "#{result.code} error creating Elasticsearch index: #{res}" + end + } + else + RestClient.put(@index_url, nil) { |res, req, result| + if result.code == "200" + puts res + else + raise "#{result.code} error creating Elasticsearch index: #{res}" + end + } + end + end + + def delete + puts "Deleting #{@options["es_index"]} via url #{@index_url}" + + RestClient.delete(@index_url) { |res, req, result| + if result.code != "200" + raise "#{result.code} error deleting Elasticsearch index: #{res}" + end + } + end + + def get_schema + RestClient.get(@mapping_url) { |res, req, result| + if result.code == "200" + puts res + else + raise "#{result.code} error getting Elasticsearch schema: #{res}" + end + } + end + + def set_schema + json = @schema["mappings"].to_json + + puts "Setting schema: #{@mapping_url}" + RestClient.put(@mapping_url, json, { content_type: :json }) { |res, req, result| + if result.code == "200" + puts res + else + raise "#{result.code} error setting Elasticsearch schema: #{res}" + end + } + end + +end diff --git a/lib/datura/options.rb b/lib/datura/options.rb index 36d4e47e2..c478ced42 100644 --- a/lib/datura/options.rb +++ b/lib/datura/options.rb @@ -22,6 +22,24 @@ def initialize(params) # include the collection and datura gem directories in the options @all["collection_dir"] = collection_dir @all["datura_dir"] = datura_dir + + other_configuration + end + + def es_schema_path + internal_path = File.join(@all["es_schema_path"], "#{@all["api_version"]}.yml") + if @all["es_schema_override"] + File.join(@all["collection_dir"], internal_path) + else + File.join(@all["datura_dir"], internal_path) + end + end + + # after all options have been flattened, create customization by + # combining the set options, etc + def other_configuration + # put together the elasticsearch schema path + @all["es_schema"] = es_schema_path end def print_message(variable, name) diff --git a/lib/datura/requirer.rb b/lib/datura/requirer.rb index 75c7bb247..b3048c758 100644 --- a/lib/datura/requirer.rb +++ b/lib/datura/requirer.rb @@ -12,4 +12,6 @@ Dir["#{current_dir}/to_es/**/*.rb"].each { |f| require f } # file types -Dir["#{current_dir}/file_types/*.rb"].each { |f| require f } +Dir["#{current_dir}/file_types/*.rb"].each {|f| require f } +# elasticsearch files +Dir["#{current_dir}/elasticsearch/*.rb"].each {|f| require f } From e452e3b2ab31f2a22377e82bbeb60ffa5995596b Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Thu, 9 Jan 2020 15:00:32 -0600 Subject: [PATCH 002/163] removes unnecessary dtd for french 17 --- lib/config/f17.dtd | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 lib/config/f17.dtd diff --git a/lib/config/f17.dtd b/lib/config/f17.dtd deleted file mode 100644 index e69de29bb..000000000 From b01397a9718ed3bfd3e4b53eac3fe811aed591f2 Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Thu, 9 Jan 2020 15:38:19 -0600 Subject: [PATCH 003/163] combines some parameter gathering files --- lib/datura/elasticsearch/alias.rb | 4 +- lib/datura/elasticsearch/data.rb | 2 +- lib/datura/elasticsearch/index.rb | 2 +- lib/datura/parser_options/clear_index.rb | 2 +- .../{es_alias_add.rb => es_alias.rb} | 4 +- lib/datura/parser_options/es_alias_delete.rb | 50 ------------------- ...{es_create_delete_index.rb => es_index.rb} | 4 +- lib/datura/parser_options/es_set_schema.rb | 26 ---------- 8 files changed, 9 insertions(+), 85 deletions(-) rename lib/datura/parser_options/{es_alias_add.rb => es_alias.rb} (92%) delete mode 100644 lib/datura/parser_options/es_alias_delete.rb rename lib/datura/parser_options/{es_create_delete_index.rb => es_index.rb} (86%) delete mode 100644 lib/datura/parser_options/es_set_schema.rb diff --git a/lib/datura/elasticsearch/alias.rb b/lib/datura/elasticsearch/alias.rb index 177ee14d1..aa7c2e4ad 100644 --- a/lib/datura/elasticsearch/alias.rb +++ b/lib/datura/elasticsearch/alias.rb @@ -6,7 +6,7 @@ module Datura::Elasticsearch::Alias def self.add - params = Datura::Parser.es_alias_add + params = Datura::Parser.es_alias options = Datura::Options.new(params).all ali = options["alias"] @@ -32,7 +32,7 @@ def self.add end def self.delete - params = Datura::Parser.es_alias_add + params = Datura::Parser.es_alias options = Datura::Options.new(params).all ali = options["alias"] diff --git a/lib/datura/elasticsearch/data.rb b/lib/datura/elasticsearch/data.rb index 5deedadb1..44b8bb848 100644 --- a/lib/datura/elasticsearch/data.rb +++ b/lib/datura/elasticsearch/data.rb @@ -7,7 +7,7 @@ module Datura::Elasticsearch::Data def self.clear # run the parameters through the option parser - params = Datura::Parser.clear_index_params + params = Datura::Parser.clear_index options = Datura::Options.new(params).all if options["collection"] == "all" self.clear_all(options) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 1065eeff2..1c1718d26 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -7,7 +7,7 @@ class Datura::Elasticsearch::Index def initialize - params = Datura::Parser.es_create_delete_index + params = Datura::Parser.es_index @options = Datura::Options.new(params).all @base_url = File.join(@options["es_path"], @options["es_index"]) diff --git a/lib/datura/parser_options/clear_index.rb b/lib/datura/parser_options/clear_index.rb index 75176dbd0..e7e0f9a4b 100644 --- a/lib/datura/parser_options/clear_index.rb +++ b/lib/datura/parser_options/clear_index.rb @@ -1,5 +1,5 @@ module Datura::Parser - def self.clear_index_params + def self.clear_index @usage = "Usage: (es|solr)_clear_index -[options]..." options = {} # will hold all the options passed in by user diff --git a/lib/datura/parser_options/es_alias_add.rb b/lib/datura/parser_options/es_alias.rb similarity index 92% rename from lib/datura/parser_options/es_alias_add.rb rename to lib/datura/parser_options/es_alias.rb index 03d88e2c5..bb36b420f 100644 --- a/lib/datura/parser_options/es_alias_add.rb +++ b/lib/datura/parser_options/es_alias.rb @@ -1,6 +1,6 @@ module Datura::Parser - def self.es_alias_add - @usage = "Usage: es_alias_add -a alias -i index -e environment" + def self.es_alias + @usage = "Usage: (command) -a alias -i index -e environment" options = {} optparse = OptionParser.new do |opts| diff --git a/lib/datura/parser_options/es_alias_delete.rb b/lib/datura/parser_options/es_alias_delete.rb deleted file mode 100644 index ea38038b8..000000000 --- a/lib/datura/parser_options/es_alias_delete.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Datura::Parser - def self.es_alias_delete - @usage = "Usage: es_alias_delete -a alias -i index -e environment" - options = {} - - optparse = OptionParser.new do |opts| - opts.banner = @usage - - opts.on( '-h', '--help', 'How does this work?') do - puts opts - exit - end - - options["alias"] = nil - opts.on( '-a', '--alias [input]', 'Alias (cdrhapi-v1)') do |input| - if input && input.length > 0 - options["alias"] = input - else - puts "Must specify an alias with -a flag" - exit - end - end - - options["environment"] = "development" - opts.on( '-e', '--environment [input]', 'Environment (development, production)') do |input| - if input && input.length > 0 - options["environment"] = input - end - end - - options["index"] = nil - opts.on( '-i', '--index [input]', 'Index (cdrhapi-v1.1)') do |input| - if input && input.length > 0 - options["index"] = input - else - puts "Must specify an index with -i flag" - exit - end - end - - end - - optparse.parse! - if options["alias"].nil? || options["index"].nil? - puts "must specify alias and index with -a and -i, respectively" - exit - end - options - end -end diff --git a/lib/datura/parser_options/es_create_delete_index.rb b/lib/datura/parser_options/es_index.rb similarity index 86% rename from lib/datura/parser_options/es_create_delete_index.rb rename to lib/datura/parser_options/es_index.rb index 22ed0d1cc..716835548 100644 --- a/lib/datura/parser_options/es_create_delete_index.rb +++ b/lib/datura/parser_options/es_index.rb @@ -1,6 +1,6 @@ module Datura::Parser - def self.es_create_delete_index - @usage = "Usage: admin_es_(create|delete)_index -e environment" + def self.es_index + @usage = "Usage: (command) -e environment" options = {} # will hold all the options passed in by user optparse = OptionParser.new do |opts| diff --git a/lib/datura/parser_options/es_set_schema.rb b/lib/datura/parser_options/es_set_schema.rb deleted file mode 100644 index 4f3d2388a..000000000 --- a/lib/datura/parser_options/es_set_schema.rb +++ /dev/null @@ -1,26 +0,0 @@ -module Datura::Parser - def self.es_set_schema_params - @usage = "Usage: es_set_schema -e environment" - options = {} - - optparse = OptionParser.new do |opts| - opts.banner = @usage - - opts.on( '-h', '--help', 'How does this work?') do - puts opts - exit - end - - options["environment"] = "development" - opts.on( '-e', '--environment [input]', 'Environment (development, production)') do |input| - if input && input.length > 0 - options["environment"] = input - end - end - - end - - optparse.parse! - options - end -end From 4d02bee88136017bc1aee6663eb55b1814d781c8 Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Thu, 9 Jan 2020 16:05:26 -0600 Subject: [PATCH 004/163] updates gems and fixes test suite had suffered from errors and from gem deprecation warnings --- Gemfile.lock | 8 +++++--- datura.gemspec | 4 ++-- test/options_test.rb | 4 +++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 133495b78..2361f7646 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,8 +3,8 @@ PATH specs: datura (0.2.0.pre.beta) colorize (~> 0.8.1) - nokogiri (~> 1.8) - rest-client (~> 2.0.2) + nokogiri (~> 1.10) + rest-client (~> 2.1) GEM remote: https://rubygems.org/ @@ -12,6 +12,7 @@ GEM colorize (0.8.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) + http-accept (1.7.0) http-cookie (1.0.3) domain_name (~> 0.5) mime-types (3.3.1) @@ -23,7 +24,8 @@ GEM nokogiri (1.10.10) mini_portile2 (~> 2.4.0) rake (13.0.1) - rest-client (2.0.2) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) diff --git a/datura.gemspec b/datura.gemspec index ef85aa47d..ef3099880 100644 --- a/datura.gemspec +++ b/datura.gemspec @@ -55,8 +55,8 @@ Gem::Specification.new do |spec| spec.required_ruby_version = "~> 2.5" spec.add_runtime_dependency "colorize", "~> 0.8.1" - spec.add_runtime_dependency "nokogiri", "~> 1.8" - spec.add_runtime_dependency "rest-client", "~> 2.0.2" + spec.add_runtime_dependency "nokogiri", "~> 1.10" + spec.add_runtime_dependency "rest-client", "~> 2.1" spec.add_development_dependency "bundler", ">= 1.16.0", "< 3.0" spec.add_development_dependency "minitest", "~> 5.0" spec.add_development_dependency "rake", "~> 13.0" diff --git a/test/options_test.rb b/test/options_test.rb index cd088785b..9b4d6dff2 100644 --- a/test/options_test.rb +++ b/test/options_test.rb @@ -6,7 +6,9 @@ class Datura::Options def read_all_configs fake1, fake2 @general_config_pub = { "default" => { - "a" => "general default public" + "a" => "general default public", + "es_schema_path" => "lib/config", + "api_version" => "2.0" } } @collection_config_pub = { From 3308c75edb231956e8e7d7dc0f7d04459f3e71fa Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Fri, 10 Jan 2020 10:20:22 -0600 Subject: [PATCH 005/163] in progress working on validator for es fields) --- bin/es_get_schema | 2 +- lib/datura/data_manager.rb | 31 ++++++++++++++---------- lib/datura/elasticsearch.rb | 4 ---- lib/datura/elasticsearch/index.rb | 40 +++++++++++++++++++++++++------ 4 files changed, 53 insertions(+), 24 deletions(-) diff --git a/bin/es_get_schema b/bin/es_get_schema index 1326d5e48..664988d32 100755 --- a/bin/es_get_schema +++ b/bin/es_get_schema @@ -4,7 +4,7 @@ require "datura" begin es = Datura::Elasticsearch::Index.new - es.get_schema + puts es.get_schema rescue => e puts e end diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 9ae304a43..7e71af4d2 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -17,6 +17,8 @@ class Datura::DataManager attr_accessor :options attr_accessor :collection + attr_accessor :es_schema_mapping + def self.format_to_class classes = { "csv" => FileCsv, @@ -49,6 +51,9 @@ def initialize # set up posting URLs @es_url = File.join(options["es_path"], options["es_index"]) @solr_url = File.join(options["solr_path"], options["solr_core"], "update") + + # retrieve the specified elasticsearch index's schema + set_schema_mappings end # NOTE: This step is what allows collection specific files to override ANY @@ -76,7 +81,7 @@ def run puts msg check_options - set_schema + get_schema_mappings pre_file_preparation @files = prepare_files @@ -135,6 +140,11 @@ def check_options if should_transform?("es") assert_option("es_path") assert_option("es_index") + # options used to obtain the mappings + assert_option("es_schema_override") + assert_option("es_schema_path") + assert_option("api_version") + assert_option("collection") end @@ -262,22 +272,19 @@ def prepare_xslt end end - def set_schema - # if ES is requested and not transform only, then set the schema - # to make sure that any new fields are stored with the correct fieldtype + # NOTE plural method name in order to accommodate other platforms + # we may be checking in the future + def set_schema_mappings + # only get the elasticsearch mapping if it is needed for post request if should_transform?("es") && !@options["transform_only"] - schema = YAML.load_file(File.join(@options["datura_dir"], @options["es_schema_path"])) - path, idx = ["es_path", "es_index"].map { |i| @options[i] } - url = "#{path}/#{idx}/_mapping/_doc?pretty=true" begin - RestClient.put(url, schema.to_json, { content_type: :json }) - msg = "Successfully set elasticsearch schema for index #{idx} _doc" - @log.info(msg) - puts msg.green + es = Datura::Elasticsearch::Index.new(@options) + @es_schema_mapping = es.get_schema_mapping rescue => e - raise("Something went wrong setting the elasticsearch schema for index #{idx} _doc:\n#{e.to_s}".red) + raise "Unable to get the elasticsearch schema: #{e}" end end + end def set_up_logger diff --git a/lib/datura/elasticsearch.rb b/lib/datura/elasticsearch.rb index fecba15fc..ffc4c710e 100644 --- a/lib/datura/elasticsearch.rb +++ b/lib/datura/elasticsearch.rb @@ -1,10 +1,6 @@ require_relative './helpers.rb' require_relative './options.rb' -require "json" -require "rest-client" -require "yaml" - module Datura::Elasticsearch # clear data from the index (leaves index schema intact) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 1c1718d26..e2cc9cbcc 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -6,20 +6,29 @@ class Datura::Elasticsearch::Index - def initialize - params = Datura::Parser.es_index - @options = Datura::Options.new(params).all + attr_reader :schema_mapping + + # if options are passed in, then commandline arguments + # do not need to be parsed + def initialize(options = nil) + if !options + params = Datura::Parser.es_index + @options = Datura::Options.new(params).all + else + @options = options + end @base_url = File.join(@options["es_path"], @options["es_index"]) @mapping_url = File.join(@base_url, "_mapping", "_doc?pretty=true") @index_url = "#{@base_url}?pretty=true" # yaml settings (if exist) and mappings - @schema = YAML.load_file(@options["es_schema"]) + @requested_schema = YAML.load_file(@options["es_schema"]) + @schema_mapping = nil end def create - json = @schema["settings"].to_json + json = @requested_schema["settings"].to_json puts "Creating ES index for API version #{@options["api_version"]}: #{@index_url}" if json && json != "null" @@ -54,15 +63,32 @@ def delete def get_schema RestClient.get(@mapping_url) { |res, req, result| if result.code == "200" - puts res + JSON.parse(res) else raise "#{result.code} error getting Elasticsearch schema: #{res}" end } end + def get_schema_mapping + # if mapping has not already been set, get the schema and manipulate + if !@schema_mapping + schema = get_schema[@options["es_index"]] + @schema_mapping["fields"] = schema["mappings"]["properties"].keys + @schema_mapping["dynamic"] = [] + schema["mappings"]["dynamic_templates"].each do |field_type, info| + es_match = info["match"].sub("*", ".*") + regex = /^#{es_match}$/ + @schema_mapping["dynamic"] << info["match"] + end + # dynamic fields are listed like *_k and will need + # to be converted to /_k$/ instead + end + @schema_mapping + end + def set_schema - json = @schema["mappings"].to_json + json = @requested_schema["mappings"].to_json puts "Setting schema: #{@mapping_url}" RestClient.put(@mapping_url, json, { content_type: :json }) { |res, req, result| From 8e995b49d552189fdccaf5422a9873741aa2df11 Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Wed, 15 Jan 2020 10:20:57 -0600 Subject: [PATCH 006/163] creates validator for elasticsearch postings the validator ensures that all fields have either an exact mapping OR match a dynamic template, at least as far as our current simple dynamic templates go. This may need to be adjusted in the future if we start using more complex templates refactors file_type post_es method to use Elasticsearch::Index object and to rely less on repeated "returns" vs a final line at the end also adds some helpful methods like should_post? to complement should_transform? for ease --- lib/datura/data_manager.rb | 53 +++++++++++------------- lib/datura/elasticsearch/index.rb | 67 ++++++++++++++++++++++--------- lib/datura/file_type.rb | 32 +++++++++------ 3 files changed, 90 insertions(+), 62 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 7e71af4d2..237cb1cf5 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -17,7 +17,6 @@ class Datura::DataManager attr_accessor :options attr_accessor :collection - attr_accessor :es_schema_mapping def self.format_to_class classes = { @@ -47,13 +46,6 @@ def initialize prepare_xslt load_collection_classes set_up_logger - - # set up posting URLs - @es_url = File.join(options["es_path"], options["es_index"]) - @solr_url = File.join(options["solr_path"], options["solr_core"], "update") - - # retrieve the specified elasticsearch index's schema - set_schema_mappings end # NOTE: This step is what allows collection specific files to override ANY @@ -81,7 +73,7 @@ def run puts msg check_options - get_schema_mappings + set_up_services pre_file_preparation @files = prepare_files @@ -115,7 +107,7 @@ def allowed_files(all_files) # TODO should this move to Options class? def assert_option(opt) - if !@options.has_key?(opt) + if !@options.key?(opt) puts "Option #{opt} was not found! Check config files and add #{opt} to continue".red raise "Missing configuration options" end @@ -137,7 +129,7 @@ def batch_process_files def check_options # verify that everything's all good before moving to per-file level processing - if should_transform?("es") + if should_post?("es") assert_option("es_path") assert_option("es_index") # options used to obtain the mappings @@ -148,7 +140,7 @@ def check_options assert_option("collection") end - if should_transform?("solr") + if should_post?("solr") assert_option("solr_core") assert_option("solr_path") end @@ -199,8 +191,8 @@ def options_msg msg << "Running script with following options:\n" msg << "collection: #{@options['collection']}\n" msg << "Environment: #{@options['environment']}\n" - msg << "Posting to: #{@es_url}\n\n" if should_transform?("es") - msg << "Posting to: #{@solr_url}\n\n" if should_transform?("solr") + msg << "Posting to: #{@es.index_url}\n\n" if should_post?("es") + msg << "Posting to: #{@solr_url}\n\n" if should_post?("solr") msg << "Format: #{@options['format']}\n" if @options["format"] msg << "Regex: #{@options['regex']}\n" if @options["regex"] msg << "Allowed Files: #{@options['allowed_files']}\n" if @options["allowed_files"] @@ -272,21 +264,6 @@ def prepare_xslt end end - # NOTE plural method name in order to accommodate other platforms - # we may be checking in the future - def set_schema_mappings - # only get the elasticsearch mapping if it is needed for post request - if should_transform?("es") && !@options["transform_only"] - begin - es = Datura::Elasticsearch::Index.new(@options) - @es_schema_mapping = es.get_schema_mapping - rescue => e - raise "Unable to get the elasticsearch schema: #{e}" - end - end - - end - def set_up_logger # make directory if one does not already exist log_dir = File.join(@options["collection_dir"], "logs") @@ -300,6 +277,22 @@ def set_up_logger ) end + def set_up_services + if should_post?("es") + # set up elasticsearch instance + @es = Datura::Elasticsearch::Index.new(@options, schema_mapping: true) + end + + if should_post?("solr") + # set up posting URLs + @solr_url = File.join(options["solr_path"], options["solr_core"], "update") + end + end + + def should_post?(type) + should_transform?(type) && !@options["transform_only"] + end + def should_transform?(type) # adjust default transformation type in params parser @options["transform_types"].include?(type) @@ -318,7 +311,7 @@ def transform_and_post(file) error_with_transform_and_post("#{e}", @error_es) end else - res_es = file.post_es(@es_url) + res_es = file.post_es(@es) if res_es && res_es.has_key?("error") error_with_transform_and_post(res_es["error"], @error_es) end diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index e2cc9cbcc..f759de4ba 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -7,10 +7,11 @@ class Datura::Elasticsearch::Index attr_reader :schema_mapping + attr_reader :index_url # if options are passed in, then commandline arguments # do not need to be parsed - def initialize(options = nil) + def initialize(options = nil, schema_mapping: false) if !options params = Datura::Parser.es_index @options = Datura::Options.new(params).all @@ -18,21 +19,23 @@ def initialize(options = nil) @options = options end - @base_url = File.join(@options["es_path"], @options["es_index"]) - @mapping_url = File.join(@base_url, "_mapping", "_doc?pretty=true") - @index_url = "#{@base_url}?pretty=true" + @index_url = File.join(@options["es_path"], @options["es_index"]) + @pretty_url = "#{@index_url}?pretty=true" + @mapping_url = File.join(@index_url, "_mapping", "_doc?pretty=true") # yaml settings (if exist) and mappings @requested_schema = YAML.load_file(@options["es_schema"]) - @schema_mapping = nil + # if requested, grab the mapping currently associated with this index + # otherwise wait until after the requested schema is loaded + get_schema_mapping if schema_mapping end def create json = @requested_schema["settings"].to_json - puts "Creating ES index for API version #{@options["api_version"]}: #{@index_url}" + puts "Creating ES index for API version #{@options["api_version"]}: #{@pretty_url}" if json && json != "null" - RestClient.put(@index_url, json, { content_type: :json }) { |res, req, result| + RestClient.put(@pretty_url, json, { content_type: :json }) { |res, req, result| if result.code == "200" puts res else @@ -40,7 +43,7 @@ def create end } else - RestClient.put(@index_url, nil) { |res, req, result| + RestClient.put(@pretty_url, nil) { |res, req, result| if result.code == "200" puts res else @@ -51,9 +54,9 @@ def create end def delete - puts "Deleting #{@options["es_index"]} via url #{@index_url}" + puts "Deleting #{@options["es_index"]} via url #{@pretty_url}" - RestClient.delete(@index_url) { |res, req, result| + RestClient.delete(@pretty_url) { |res, req, result| if result.code != "200" raise "#{result.code} error deleting Elasticsearch index: #{res}" end @@ -72,17 +75,26 @@ def get_schema def get_schema_mapping # if mapping has not already been set, get the schema and manipulate - if !@schema_mapping + if @schema_mapping.nil? + @schema_mapping = {} + @schema_mapping["dynamic"] = nil + schema = get_schema[@options["es_index"]] - @schema_mapping["fields"] = schema["mappings"]["properties"].keys - @schema_mapping["dynamic"] = [] - schema["mappings"]["dynamic_templates"].each do |field_type, info| - es_match = info["match"].sub("*", ".*") - regex = /^#{es_match}$/ - @schema_mapping["dynamic"] << info["match"] + @schema_mapping["fields"] = schema["mappings"]["_doc"]["properties"].keys + + regex_pieces = [] + schema["mappings"]["_doc"]["dynamic_templates"].each do |template| + mapping = template.map { |k,v| v["match"] }.first + # dynamic fields are listed like *_k and will need + # to be converted to ^.*_k$, then combined into a mega-regex + es_match = mapping.sub("*", ".*") + regex = "^#{es_match}$" + regex_pieces << regex + end + if !regex_pieces.empty? + regex_joined = regex_pieces.join("|") + @schema_mapping["dynamic"] = /#{regex_joined}/ end - # dynamic fields are listed like *_k and will need - # to be converted to /_k$/ instead end @schema_mapping end @@ -100,4 +112,21 @@ def set_schema } end + # doc: ruby hash corresponding with Elasticsearch document JSON + def valid_document?(doc) + get_schema_mapping if @schema_mapping.nil? + fields = @schema_mapping["fields"] + dynamic = @schema_mapping["dynamic"] + # NOTE: validation only checking the names of fields + # against the schema, NOT the contents of fields + # since Elasticsearch itself will know if you are sending it + # text instead of a date field, etc + + valid = doc.keys.all? do |doc_field| + # check if exact match for a field or if it matches dynamic field mapping + fields.include?(doc_field) || doc_field.match(dynamic) + end + valid + end + end diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 236369a30..68eb6b8b5 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -49,30 +49,36 @@ def parse_markup_lang_file CommonXml.create_xml_object(self.file_location) end - def post_es(url=nil) - url = url || "#{@options["es_path"]}/#{@options["es_index"]}" + def post_es(es) + error = nil begin transformed = transform_es rescue => e - return { "error" => "Error transforming ES for #{self.filename(false)}: #{e}" } + "Error transforming ES for #{self.filename(false)}: #{e}" end if transformed && transformed.length > 0 transformed.each do |doc| id = doc["identifier"] - puts "posting #{id}" - puts "PATH: #{url}/_doc/#{id}" if options["verbose"] - # NOTE: If you need to do partial updates rather than replacement of doc - # you will need to add _update at the end of this URL - begin - RestClient.put("#{url}/_doc/#{id}", doc.to_json, {:content_type => :json } ) - rescue => e - return { "error" => "Error transforming or posting to ES for #{self.filename(false)}: #{e.response}" } + # before a document is posted, we need to make sure that the fields validate against the schema + if es.valid_document?(doc) + + puts "posting #{id}" + puts "PATH: #{es.index_url}/_doc/#{id}" if options["verbose"] + # NOTE: If you need to do partial updates rather than replacement of doc + # you will need to add _update at the end of this URL + begin + RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, {:content_type => :json } ) + rescue => e + "Error transforming or posting to ES for #{self.filename(false)}: #{e.response}" + end + else + error = "Document #{id} did not validate against the elasticsearch schema" end end else - return { "error" => "No file was transformed" } + error = "No file was transformed" end - return { "docs" => transformed } + error ? { "error" => error } : { "docs" => transformed} end def post_solr(url=nil) From 9f7eebf2d5656fdb08845874550efc57ed1c8ac0 Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Fri, 17 Jan 2020 11:03:24 -0600 Subject: [PATCH 007/163] whoops missed one --- lib/datura/file_type.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 68eb6b8b5..0d4d755d8 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -69,7 +69,7 @@ def post_es(es) begin RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, {:content_type => :json } ) rescue => e - "Error transforming or posting to ES for #{self.filename(false)}: #{e.response}" + error = "Error transforming or posting to ES for #{self.filename(false)}: #{e.response}" end else error = "Document #{id} did not validate against the elasticsearch schema" From 44c3b8001dd19dd1c04b68197092bc410b3f18e2 Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Fri, 17 Jan 2020 11:16:42 -0600 Subject: [PATCH 008/163] refactored validator to handle nested field specific mapping that is, previously it was assuming that nested subfields all match a top level field or dynamic mapping actually, nested fields can specify their own specific mapping that the subfields may also access....sooooooo I had to redo stuff tests should be passing! thank goodness for unit tests and tdd --- lib/config/es_api_schemas/2.0.yml | 4 + lib/config/public.yml | 2 + lib/datura/data_manager.rb | 5 +- lib/datura/elasticsearch/index.rb | 64 ++++-- test/es_index_test.rb | 130 +++++++++++ test/fixtures/es_mapping_2.0.json | 345 ++++++++++++++++++++++++++++++ 6 files changed, 533 insertions(+), 17 deletions(-) create mode 100644 test/es_index_test.rb create mode 100644 test/fixtures/es_mapping_2.0.json diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index c34155bad..237c9cf8b 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -259,6 +259,10 @@ mappings: mapping: type: keyword normalizer: keyword_normalized + - nested_fields: + match: "*_n" + mapping: + type: nested - text_fields: match: "*_t" mapping: diff --git a/lib/config/public.yml b/lib/config/public.yml index 486e85037..5bee0d115 100644 --- a/lib/config/public.yml +++ b/lib/config/public.yml @@ -38,6 +38,8 @@ default: # current version of the API (powered by Elasticsearch) # this setting determines which of the schemas will be used api_version: "override with value like '1.0' or '2.0'" + # NOTE: es_schema option is set later as combination of above + # es_schema_override, es_schema_path, and api_version # RESOURCE LOCATIONS data_base: https://cdrhmedia.unl.edu # xml, csv, html snippets, etc diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 237cb1cf5..d35e2290a 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -68,12 +68,13 @@ def print_options def run @time = [Time.now] # log starting information for user + check_options + set_up_services + msg = options_msg @log.info(msg) puts msg - check_options - set_up_services pre_file_preparation @files = prepare_files diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index f759de4ba..6732552cb 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -75,15 +75,25 @@ def get_schema def get_schema_mapping # if mapping has not already been set, get the schema and manipulate - if @schema_mapping.nil? - @schema_mapping = {} - @schema_mapping["dynamic"] = nil + if !defined?(@schema_mapping) + @schema_mapping = { + "dyanmic" => nil, # /regex|regex/ + "fields" => [], # [ fields ] + "nested" => {} # { field: [ nested_fields ] } + } schema = get_schema[@options["es_index"]] - @schema_mapping["fields"] = schema["mappings"]["_doc"]["properties"].keys + doc = schema["mappings"]["_doc"] + + doc["properties"].each do |field, value| + @schema_mapping["fields"] << field + if value["type"] == "nested" + @schema_mapping["nested"][field] = value["properties"].keys + end + end regex_pieces = [] - schema["mappings"]["_doc"]["dynamic_templates"].each do |template| + doc["dynamic_templates"].each do |template| mapping = template.map { |k,v| v["match"] }.first # dynamic fields are listed like *_k and will need # to be converted to ^.*_k$, then combined into a mega-regex @@ -114,19 +124,43 @@ def set_schema # doc: ruby hash corresponding with Elasticsearch document JSON def valid_document?(doc) - get_schema_mapping if @schema_mapping.nil? - fields = @schema_mapping["fields"] - dynamic = @schema_mapping["dynamic"] + get_schema_mapping if !defined?(@schema_mapping) # NOTE: validation only checking the names of fields # against the schema, NOT the contents of fields - # since Elasticsearch itself will know if you are sending it - # text instead of a date field, etc - - valid = doc.keys.all? do |doc_field| - # check if exact match for a field or if it matches dynamic field mapping - fields.include?(doc_field) || doc_field.match(dynamic) + # Elasticsearch itself checks that you are sending date + # formats to date fields, etc + + doc.all? do |field, value| + if valid_field?(field) + # great, the field is valid, now check if it is a parent + nested = Array(value).map do |nested| + if nested.class == Hash + nested.keys.all? { |k| valid_field?(k, field) } + end + end + # if the array is empty, ignore it, otherwise find out if any + # nested fields failed the validate + nested.compact.all? { |t| t } + else + false + end end - valid + end + + # if a field, including those inside nested fields, + # matches a top level field mapping or a dynamic field, + # they are good to go + # further, if this is a nested field, they may check + # to see if the specific nesting mapping validates them + def valid_field?(field, parent=nil) + @schema_mapping["fields"].include?(field) || + field.match(@schema_mapping["dynamic"]) || + valid_nested_field?(field, parent) + end + + def valid_nested_field?(field, parent) + parent_mapping = @schema_mapping["nested"][parent] + parent_mapping.include?(field) if parent_mapping end end diff --git a/test/es_index_test.rb b/test/es_index_test.rb new file mode 100644 index 000000000..9bf6d6018 --- /dev/null +++ b/test/es_index_test.rb @@ -0,0 +1,130 @@ +require "test_helper" + +class Datura::ElasticsearchIndexTest < Minitest::Test + + @@options = { + "api_version" => "2.0", + "es_index" => "fake_index", + "es_path" => "fake_path", + "es_schema" => File.join( + File.expand_path(File.dirname(__FILE__)), + "../lib/config/es_api_schemas/2.0.yml" + ) + } + + # stub in get_schema so that we can test get_schema_mapping without + # worrying about integration with actual index + + class Datura::Elasticsearch::Index + def get_schema + raw = File.read( + File.join( + File.expand_path(File.dirname(__FILE__)), + "fixtures/es_mapping_2.0.json" + ) + ) + JSON.parse(raw) + end + end + + def test_initialize + # test that options populate if you pass existing ones in + es = Datura::Elasticsearch::Index.new(@@options) + path = File.join(@@options["es_path"], @@options["es_index"]) + assert_equal path, es.index_url + + # test that schema mapping occurs, although it will be with the stubbed + # in version of get_schema above, rather than index integration + es = Datura::Elasticsearch::Index.new(@@options, schema_mapping: true) + assert es.schema_mapping + end + + def test_get_schema_mapping + # let's just see what happens + es = Datura::Elasticsearch::Index.new(@@options) + es.get_schema_mapping + assert es.schema_mapping["fields"] + assert_equal 46, es.schema_mapping["fields"].length + assert_equal( + /^.*_d$|^.*_i$|^.*_k$|^.*_n$|^.*_t$|^.*_t_en$|^.*_t_es$/, + es.schema_mapping["dynamic"] + ) + end + + def test_valid_document? + es = Datura::Elasticsearch::Index.new(@@options) + + # basic fields + assert es.valid_document?({ "identifier" => "a" }) + assert es.valid_document?({ + "collection" => "a", + "date_not_before" => "2012-01-01", + "text" => "a", + }) + + # nested fields with child fields not matching top level field + assert es.valid_document?({ + "creator" => [ + { + "id" => "a", + "name" => "a" + } + ] + }) + + # nested fields with child fields matching top level / dynamic + assert es.valid_document?({ + "creator" => [ + { + "subcategory" => "a", + "data_type" => "a", + "keyword_k" => "a" + } + ] + }) + + # dynamic fields, each type + assert es.valid_document?({ "new_field_d" => "2012-01-1" }) + assert es.valid_document?({ "new_field_i" => "1" }) + assert es.valid_document?({ "new_field_k" => "a" }) + assert es.valid_document?({ "new_field_t" => "a" }) + assert es.valid_document?({ "new_field_t_en" => "a" }) + assert es.valid_document?({ "new_field_t_es" => "a" }) + + # test failures of basic and dynamic fields + refute es.valid_document?({ "bad_field" => "a" }) + refute es.valid_document?({ "dynamic_t_bad" => "a" }) + + # test failure of nested field with all bad subfields + refute es.valid_document?({ + "creator" => [ + { + "bad_field" => "a", + "another_one" => "a" + } + ] + }) + + # test feailure of nested field with mixture of good / bad + refute es.valid_document?({ + "creator" => [ + { + "id" => "a", + "keyword_k" => "a" + }, + { + "id" => "a", + "bad_field" => "a" + } + ] + }) + + # test that bad fields hidden with good still fail + refute es.valid_document?({ + "collection" => "a", + "keyword_k" => "a", + "bad_field" => "a" + }) + end + +end diff --git a/test/fixtures/es_mapping_2.0.json b/test/fixtures/es_mapping_2.0.json new file mode 100644 index 000000000..f82189503 --- /dev/null +++ b/test/fixtures/es_mapping_2.0.json @@ -0,0 +1,345 @@ +{ + "fake_index" : { + "mappings" : { + "_doc" : { + "dynamic_templates" : [ + { + "date_fields" : { + "match" : "*_d", + "mapping" : { + "format" : "yyyy-MM-dd||epoch_millis", + "type" : "date" + } + } + }, + { + "integer_fields" : { + "match" : "*_i", + "mapping" : { + "type" : "integer" + } + } + }, + { + "keyword_fields" : { + "match" : "*_k", + "mapping" : { + "normalizer" : "keyword_normalized", + "type" : "keyword" + } + } + }, + { + "nested_fields" : { + "match" : "*_n", + "mapping" : { + "type" : "nested" + } + } + }, + { + "text_fields" : { + "match" : "*_t", + "mapping" : { + "analyzer" : "english", + "type" : "text" + } + } + }, + { + "text_english" : { + "match" : "*_t_en", + "mapping" : { + "analyzer" : "english", + "type" : "text" + } + } + }, + { + "text_spanish" : { + "match" : "*_t_es", + "mapping" : { + "analyzer" : "spanish", + "type" : "text" + } + } + } + ], + "properties" : { + "abstract" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "alternative" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "annotations_text" : { + "type" : "text", + "analyzer" : "english" + }, + "category" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "collection" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "collection_desc" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "contributor" : { + "type" : "nested", + "properties" : { + "id" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "name" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "role" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + } + } + }, + "coverage-spatial" : { + "type" : "nested", + "properties" : { + "city" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "coordinates" : { + "type" : "geo_point" + }, + "country" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "county" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "id" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "place_name" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "postal_code" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "region" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "state" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "street" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + } + } + }, + "creator" : { + "type" : "nested", + "properties" : { + "id" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "name" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + } + } + }, + "creator_sort" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "data_type" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "date" : { + "type" : "date", + "format" : "yyyy-MM-dd||epoch_millis" + }, + "date_display" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "date_not_after" : { + "type" : "date", + "format" : "yyyy-MM-dd||epoch_millis" + }, + "date_not_before" : { + "type" : "date", + "format" : "yyyy-MM-dd||epoch_millis" + }, + "description" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "extent" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "format" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "identifier" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "image_id" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "image_location" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "keywords" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "language" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "languages" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "medium" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "people" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "person" : { + "type" : "nested", + "properties" : { + "id" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "name" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "role" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + } + } + }, + "places" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "publisher" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "recipient" : { + "type" : "nested", + "properties" : { + "id" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "name" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "role" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + } + } + }, + "relation" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "rights" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "rights_holder" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "rights_uri" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "source" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "subcategory" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "subjects" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "text" : { + "type" : "text", + "analyzer" : "english" + }, + "title" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "title_sort" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "topics" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "type" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "uri" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "uri_data" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "uri_html" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + }, + "works" : { + "type" : "keyword", + "normalizer" : "keyword_normalized" + } + } + } + } + } +} From 2deb3ca8bb69fd11e3298fba2942ad043f336224 Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Fri, 17 Jan 2020 15:03:20 -0600 Subject: [PATCH 009/163] get rid of unnecessary variable definitions --- lib/datura/file_types/file_csv.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/datura/file_types/file_csv.rb b/lib/datura/file_types/file_csv.rb index 65655a940..cd8a4e381 100644 --- a/lib/datura/file_types/file_csv.rb +++ b/lib/datura/file_types/file_csv.rb @@ -13,7 +13,6 @@ def build_html_from_csv # Note: if overriding this function, it's recommended to use # a more specific identifier for each row of the CSV # but since this is a generic version, simply using the current iteration number - id = index # using XML instead of HTML for simplicity's sake builder = Nokogiri::XML::Builder.new do |xml| xml.div(class: "main_content") { From c863b325d26edf8a8b177b93e4adeb7994f7246b Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 20 May 2022 10:10:35 -0500 Subject: [PATCH 010/163] put data methods in index class --- bin/es_clear_index | 2 +- lib/datura/elasticsearch/data.rb | 94 ------------------------------- lib/datura/elasticsearch/index.rb | 85 ++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 95 deletions(-) delete mode 100644 lib/datura/elasticsearch/data.rb diff --git a/bin/es_clear_index b/bin/es_clear_index index 6cd7b8740..c8534eba5 100755 --- a/bin/es_clear_index +++ b/bin/es_clear_index @@ -3,7 +3,7 @@ require "datura" begin - Datura::Elasticsearch::Data.clear + Datura::Elasticsearch::Index.clear rescue => e puts e end diff --git a/lib/datura/elasticsearch/data.rb b/lib/datura/elasticsearch/data.rb deleted file mode 100644 index 44b8bb848..000000000 --- a/lib/datura/elasticsearch/data.rb +++ /dev/null @@ -1,94 +0,0 @@ -require "json" -require "rest-client" - -require_relative "./../elasticsearch.rb" - -module Datura::Elasticsearch::Data - - def self.clear - # run the parameters through the option parser - params = Datura::Parser.clear_index - options = Datura::Options.new(params).all - if options["collection"] == "all" - self.clear_all(options) - else - self.clear_index(options) - end - end - - private - - def self.build_clear_data(options) - if options["regex"] - field = options["field"] || "identifier" - { - "query" => { - "bool" => { - "must" => [ - { "regexp" => { field => options["regex"] } }, - { "term" => { "collection" => options["collection"] } } - ] - } - } - } - else - { - "query" => { "term" => { "collection" => options["collection"] } } - } - end - end - - def self.clear_all(options) - puts "Please verify that you want to clear EVERY ENTRY from the ENTIRE INDEX\n\n" - puts "== FIELD / REGEX FILTERS NOT AVAILABLE FOR THIS OPTION, YOU'LL WIPE EVERYTHING ==\n\n" - puts "Running this on something other than your computer's localhost? DON'T." - puts "Type: 'Yes I'm sure'" - confirm = STDIN.gets.chomp - if confirm == "Yes I'm sure" - url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") - json = { "query" => { "match_all" => {} } } - RestClient.post(url, json.to_json, { content_type: :json }) { |res, req, result| - if result.code == "200" - puts res - else - raise "#{result.code} error when clearing entire index: #{res}" - end - } - else - puts "You typed '#{confirm}'. This is incorrect, exiting program" - exit - end - end - - def self.clear_index(options) - url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") - confirmation = self.confirm_clear(options, url) - - if confirmation - data = self.build_clear_data(options) - RestClient.post(url, data.to_json, { content_type: :json }) { |res, req, result| - if result.code == "200" - puts res - else - raise "#{result.code} error when clearing index: #{res}" - end - } - else - puts "come back anytime!" - exit - end - end - - def self.confirm_clear(options, url) - # verify that the user is really sure about the index they're about to wipe - puts "Are you sure that you want to remove entries from" - puts " #{options["collection"]}'s #{options['environment']} environment?" - puts "url: #{url}" - puts "y/N" - answer = STDIN.gets.chomp - # boolean - !!(answer =~ /[yY]/) - end - - -end diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 6732552cb..842d9316b 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -163,4 +163,89 @@ def valid_nested_field?(field, parent) parent_mapping.include?(field) if parent_mapping end + def self.clear + # run the parameters through the option parser + params = Datura::Parser.clear_index + options = Datura::Options.new(params).all + if options["collection"] == "all" + self.clear_all(options) + else + self.clear_index(options) + end + end + + private + + def self.build_clear_data(options) + if options["regex"] + field = options["field"] || "identifier" + { + "query" => { + "bool" => { + "must" => [ + { "regexp" => { field => options["regex"] } }, + { "term" => { "collection" => options["collection"] } } + ] + } + } + } + else + { + "query" => { "term" => { "collection" => options["collection"] } } + } + end + end + + def self.clear_all(options) + puts "Please verify that you want to clear EVERY ENTRY from the ENTIRE INDEX\n\n" + puts "== FIELD / REGEX FILTERS NOT AVAILABLE FOR THIS OPTION, YOU'LL WIPE EVERYTHING ==\n\n" + puts "Running this on something other than your computer's localhost? DON'T." + puts "Type: 'Yes I'm sure'" + confirm = STDIN.gets.chomp + if confirm == "Yes I'm sure" + url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") + json = { "query" => { "match_all" => {} } } + RestClient.post(url, json.to_json, { content_type: :json }) { |res, req, result| + if result.code == "200" + puts res + else + raise "#{result.code} error when clearing entire index: #{res}" + end + } + else + puts "You typed '#{confirm}'. This is incorrect, exiting program" + exit + end + end + + def self.clear_index(options) + url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") + confirmation = self.confirm_clear(options, url) + + if confirmation + data = self.build_clear_data(options) + RestClient.post(url, data.to_json, { content_type: :json }) { |res, req, result| + if result.code == "200" + puts res + else + raise "#{result.code} error when clearing index: #{res}" + end + } + else + puts "come back anytime!" + exit + end + end + + def self.confirm_clear(options, url) + # verify that the user is really sure about the index they're about to wipe + puts "Are you sure that you want to remove entries from" + puts " #{options["collection"]}'s #{options['environment']} environment?" + puts "url: #{url}" + puts "y/N" + answer = STDIN.gets.chomp + # boolean + !!(answer =~ /[yY]/) + end + end From 282dabf910b01a22d7146f5c66a8ca7cd9f7e6cb Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 20 May 2022 10:11:39 -0500 Subject: [PATCH 011/163] require_relative so that tests can be run from base directory --- test/common_xml_test.rb | 2 +- test/datura_test.rb | 2 +- test/es_index_test.rb | 2 +- test/helpers_test.rb | 2 +- test/options_test.rb | 2 +- test/tei_to_es_test.rb | 3 ++- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/common_xml_test.rb b/test/common_xml_test.rb index 765f85c97..6d5c22224 100644 --- a/test/common_xml_test.rb +++ b/test/common_xml_test.rb @@ -1,4 +1,4 @@ -require "test_helper" +require_relative "test_helper" require "nokogiri" class CommonXmlTest < Minitest::Test diff --git a/test/datura_test.rb b/test/datura_test.rb index 92db7ee99..f0e256d1d 100644 --- a/test/datura_test.rb +++ b/test/datura_test.rb @@ -1,4 +1,4 @@ -require "test_helper" +require_relative "test_helper" class DaturaTest < Minitest::Test def test_that_it_has_a_version_number diff --git a/test/es_index_test.rb b/test/es_index_test.rb index 9bf6d6018..5cee19c27 100644 --- a/test/es_index_test.rb +++ b/test/es_index_test.rb @@ -1,4 +1,4 @@ -require "test_helper" +require_relative "test_helper" class Datura::ElasticsearchIndexTest < Minitest::Test diff --git a/test/helpers_test.rb b/test/helpers_test.rb index fcc4f9c7d..88d740ecf 100644 --- a/test/helpers_test.rb +++ b/test/helpers_test.rb @@ -1,4 +1,4 @@ -require "test_helper" +require_relative "test_helper" require "nokogiri" class Datura::HelpersTest < Minitest::Test diff --git a/test/options_test.rb b/test/options_test.rb index 9b4d6dff2..1bf33b60f 100644 --- a/test/options_test.rb +++ b/test/options_test.rb @@ -1,4 +1,4 @@ -require "test_helper" +require_relative "test_helper" # override the Options class method so that we # can test without real config files diff --git a/test/tei_to_es_test.rb b/test/tei_to_es_test.rb index abf22a898..19d59c9d1 100644 --- a/test/tei_to_es_test.rb +++ b/test/tei_to_es_test.rb @@ -1,4 +1,5 @@ -require "test_helper" +require_relative "test_helper" +require "nokogiri" class TeiToEsTest < Minitest::Test def setup From d081aa0b4633481f16ce939e96e75e407245ba01 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 20 May 2022 11:22:47 -0500 Subject: [PATCH 012/163] move puts from bin methods into es classes --- bin/es_alias_add | 2 +- bin/es_alias_delete | 2 +- bin/es_alias_list | 2 +- bin/es_get_schema | 2 +- lib/datura/elasticsearch/alias.rb | 2 +- lib/datura/elasticsearch/index.rb | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/es_alias_add b/bin/es_alias_add index 7f028dfe8..6be2c564a 100755 --- a/bin/es_alias_add +++ b/bin/es_alias_add @@ -3,7 +3,7 @@ require "datura" begin - puts Datura::Elasticsearch::Alias.add + Datura::Elasticsearch::Alias.add rescue => e puts e end diff --git a/bin/es_alias_delete b/bin/es_alias_delete index 1317c39dc..6c6f2ade4 100755 --- a/bin/es_alias_delete +++ b/bin/es_alias_delete @@ -3,7 +3,7 @@ require "datura" begin - puts Datura::Elasticsearch::Alias.delete + Datura::Elasticsearch::Alias.delete rescue => e puts e end diff --git a/bin/es_alias_list b/bin/es_alias_list index ba6df4d7e..23ad183e8 100755 --- a/bin/es_alias_list +++ b/bin/es_alias_list @@ -2,4 +2,4 @@ require "datura" -puts Datura::Elasticsearch::Alias.list +Datura::Elasticsearch::Alias.list diff --git a/bin/es_get_schema b/bin/es_get_schema index 664988d32..1326d5e48 100755 --- a/bin/es_get_schema +++ b/bin/es_get_schema @@ -4,7 +4,7 @@ require "datura" begin es = Datura::Elasticsearch::Index.new - puts es.get_schema + es.get_schema rescue => e puts e end diff --git a/lib/datura/elasticsearch/alias.rb b/lib/datura/elasticsearch/alias.rb index aa7c2e4ad..3dbdefe02 100644 --- a/lib/datura/elasticsearch/alias.rb +++ b/lib/datura/elasticsearch/alias.rb @@ -49,7 +49,7 @@ def self.list options = Datura::Options.new({}).all res = RestClient.get(File.join(options["es_path"], "_aliases")) - JSON.pretty_generate(JSON.parse(res)) + puts JSON.pretty_generate(JSON.parse(res)) end end diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 842d9316b..09e89c8bd 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -66,7 +66,7 @@ def delete def get_schema RestClient.get(@mapping_url) { |res, req, result| if result.code == "200" - JSON.parse(res) + puts JSON.parse(res) else raise "#{result.code} error getting Elasticsearch schema: #{res}" end From ebbb8d2ca13864a552f1c03c0ecadbe4da4dcb20 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 20 May 2022 16:19:23 -0500 Subject: [PATCH 013/163] change get_schema to return rather than puts reverts my earlier change, it breaks other functions --- bin/es_get_schema | 2 +- lib/datura/elasticsearch/index.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/es_get_schema b/bin/es_get_schema index 1326d5e48..664988d32 100755 --- a/bin/es_get_schema +++ b/bin/es_get_schema @@ -4,7 +4,7 @@ require "datura" begin es = Datura::Elasticsearch::Index.new - es.get_schema + puts es.get_schema rescue => e puts e end diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 09e89c8bd..842d9316b 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -66,7 +66,7 @@ def delete def get_schema RestClient.get(@mapping_url) { |res, req, result| if result.code == "200" - puts JSON.parse(res) + JSON.parse(res) else raise "#{result.code} error getting Elasticsearch schema: #{res}" end From a91a3528aaa654efbeafdf5fe8439384530acede Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 20 May 2022 16:19:53 -0500 Subject: [PATCH 014/163] simplify regex --- lib/datura/elasticsearch/index.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 842d9316b..68ce6b805 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -98,12 +98,11 @@ def get_schema_mapping # dynamic fields are listed like *_k and will need # to be converted to ^.*_k$, then combined into a mega-regex es_match = mapping.sub("*", ".*") - regex = "^#{es_match}$" - regex_pieces << regex + regex_pieces << es_match end if !regex_pieces.empty? regex_joined = regex_pieces.join("|") - @schema_mapping["dynamic"] = /#{regex_joined}/ + @schema_mapping["dynamic"] = /^(?:#{regex_joined})$/ end end @schema_mapping From 1c4b4eb48ff03e7319a0d84ef469248bc8f8881f Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 20 May 2022 16:23:59 -0500 Subject: [PATCH 015/163] drop unnecessary conditional --- lib/datura/elasticsearch/index.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 68ce6b805..2219d1503 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -123,7 +123,7 @@ def set_schema # doc: ruby hash corresponding with Elasticsearch document JSON def valid_document?(doc) - get_schema_mapping if !defined?(@schema_mapping) + get_schema_mapping # NOTE: validation only checking the names of fields # against the schema, NOT the contents of fields # Elasticsearch itself checks that you are sending date From 37284241e370c1aa4cdbf9d141761c9656f746a6 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 20 May 2022 16:53:49 -0500 Subject: [PATCH 016/163] return early if invalid nested field found --- lib/datura/elasticsearch/index.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 2219d1503..a409ca550 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -132,14 +132,18 @@ def valid_document?(doc) doc.all? do |field, value| if valid_field?(field) # great, the field is valid, now check if it is a parent - nested = Array(value).map do |nested| + Array(value).each do |nested| if nested.class == Hash - nested.keys.all? { |k| valid_field?(k, field) } + if nested.keys.all? { |k| valid_field?(k, field) } + next + else + # if one of the nested hashes fails, it + return false + end end end - # if the array is empty, ignore it, otherwise find out if any - # nested fields failed the validate - nested.compact.all? { |t| t } + # all nested fields passed, so it is valid + true else false end From 4d2c2777e9ba6c2408f53b4269ccbd4b3af26c1d Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 23 May 2022 11:44:29 -0500 Subject: [PATCH 017/163] change coverage-spatial to spatial --- lib/config/es_api_schemas/1.0.yml | 2 +- lib/config/es_api_schemas/2.0.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/config/es_api_schemas/1.0.yml b/lib/config/es_api_schemas/1.0.yml index 89747ddf5..9d3a69650 100644 --- a/lib/config/es_api_schemas/1.0.yml +++ b/lib/config/es_api_schemas/1.0.yml @@ -103,7 +103,7 @@ mappings: type: keyword rights_uri: type: keyword - coverage-spatial: + spatial: type: nested properties: place_name: diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index 237c9cf8b..841da72a4 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -171,7 +171,7 @@ mappings: rights_uri: type: keyword normalizer: keyword_normalized - coverage-spatial: + spatial: type: nested properties: place_name: From 1b3feeb57764b8a4fc5f1048a7af41ac251e48fa Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 23 May 2022 11:52:09 -0500 Subject: [PATCH 018/163] Update CHANGELOG.md update documentation for changes to the schema --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e06d16f69..b90e11bf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,8 @@ Versioning](https://semver.org/spec/v2.0.0.html). - Tests and fixtures for all supported formats except CustomToEs - `get_elements` returns nodeset given xpath arguments - `spatial` nested fields `spatial.type` and `spatial.title` +- Versioning system to support multiple elasticsearch schemas +- Validator to check against the elasticsearch copy ### Changed - Arguments for `get_text`, `get_list`, and `get_xpaths` @@ -58,12 +60,14 @@ Versioning](https://semver.org/spec/v2.0.0.html). - Documentation updated - Changed Install instructions to include RVM and gemset naming conventions - API field `coverage_spatial` is now just `spatial` +- refactored executables into modules and classes ### Migration - Change `coverage_spatial` nested field to `spatial` - `get_text`, `get_list`, and `get_xpaths` require changing arguments to keyword (like `xml` and `keep_tags`) - Recommend checking xpaths and behavior of fields after updating to this version, as some defaults have changed - Possible to refactor previous FileCsv overrides to use new CsvToEs abilities, but not necessary +- Config files should specify `api_version` as 1.0 or 2.0 ## [v0.1.6](https://github.com/CDRH/datura/compare/v0.1.5...v0.1.6) - 2020-04-24 - Improvements to CSV, WEBS transformers and adds Custom transformer From 70f98a1b0f5ccc706f843e87b53e6be0e9a57bfd Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Thu, 9 Jan 2020 15:00:19 -0600 Subject: [PATCH 019/163] moves code out of bin elasticsearch files and into module in order to achieve support for multiple ES schemas - adds configuration for different schema locations - moves code from executables into Datura::Elasticsearch module - Datura::Options combines settings into schema path --- lib/datura/elasticsearch/alias.rb | 8 ++- lib/datura/elasticsearch/data.rb | 94 +++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 lib/datura/elasticsearch/data.rb diff --git a/lib/datura/elasticsearch/alias.rb b/lib/datura/elasticsearch/alias.rb index 3dbdefe02..adf1c5cf1 100644 --- a/lib/datura/elasticsearch/alias.rb +++ b/lib/datura/elasticsearch/alias.rb @@ -6,7 +6,11 @@ module Datura::Elasticsearch::Alias def self.add +<<<<<<< HEAD params = Datura::Parser.es_alias +======= + params = Datura::Parser.es_alias_add +>>>>>>> 01ed9e56d (moves code out of bin elasticsearch files and into module) options = Datura::Options.new(params).all ali = options["alias"] @@ -32,7 +36,7 @@ def self.add end def self.delete - params = Datura::Parser.es_alias + params = Datura::Parser.es_alias_add options = Datura::Options.new(params).all ali = options["alias"] @@ -49,7 +53,7 @@ def self.list options = Datura::Options.new({}).all res = RestClient.get(File.join(options["es_path"], "_aliases")) - puts JSON.pretty_generate(JSON.parse(res)) + JSON.pretty_generate(JSON.parse(res)) end end diff --git a/lib/datura/elasticsearch/data.rb b/lib/datura/elasticsearch/data.rb new file mode 100644 index 000000000..5deedadb1 --- /dev/null +++ b/lib/datura/elasticsearch/data.rb @@ -0,0 +1,94 @@ +require "json" +require "rest-client" + +require_relative "./../elasticsearch.rb" + +module Datura::Elasticsearch::Data + + def self.clear + # run the parameters through the option parser + params = Datura::Parser.clear_index_params + options = Datura::Options.new(params).all + if options["collection"] == "all" + self.clear_all(options) + else + self.clear_index(options) + end + end + + private + + def self.build_clear_data(options) + if options["regex"] + field = options["field"] || "identifier" + { + "query" => { + "bool" => { + "must" => [ + { "regexp" => { field => options["regex"] } }, + { "term" => { "collection" => options["collection"] } } + ] + } + } + } + else + { + "query" => { "term" => { "collection" => options["collection"] } } + } + end + end + + def self.clear_all(options) + puts "Please verify that you want to clear EVERY ENTRY from the ENTIRE INDEX\n\n" + puts "== FIELD / REGEX FILTERS NOT AVAILABLE FOR THIS OPTION, YOU'LL WIPE EVERYTHING ==\n\n" + puts "Running this on something other than your computer's localhost? DON'T." + puts "Type: 'Yes I'm sure'" + confirm = STDIN.gets.chomp + if confirm == "Yes I'm sure" + url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") + json = { "query" => { "match_all" => {} } } + RestClient.post(url, json.to_json, { content_type: :json }) { |res, req, result| + if result.code == "200" + puts res + else + raise "#{result.code} error when clearing entire index: #{res}" + end + } + else + puts "You typed '#{confirm}'. This is incorrect, exiting program" + exit + end + end + + def self.clear_index(options) + url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") + confirmation = self.confirm_clear(options, url) + + if confirmation + data = self.build_clear_data(options) + RestClient.post(url, data.to_json, { content_type: :json }) { |res, req, result| + if result.code == "200" + puts res + else + raise "#{result.code} error when clearing index: #{res}" + end + } + else + puts "come back anytime!" + exit + end + end + + def self.confirm_clear(options, url) + # verify that the user is really sure about the index they're about to wipe + puts "Are you sure that you want to remove entries from" + puts " #{options["collection"]}'s #{options['environment']} environment?" + puts "url: #{url}" + puts "y/N" + answer = STDIN.gets.chomp + # boolean + !!(answer =~ /[yY]/) + end + + +end From ab2a750e7598a8104ac61e0898e9e9c32286a5f1 Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Fri, 10 Jan 2020 10:20:22 -0600 Subject: [PATCH 020/163] in progress working on validator for es fields) --- lib/datura/data_manager.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index d35e2290a..a063b28d0 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -17,6 +17,10 @@ class Datura::DataManager attr_accessor :options attr_accessor :collection +<<<<<<< HEAD +======= + attr_accessor :es_schema_mapping +>>>>>>> 9c7eded0a (in progress working on validator for es fields)) def self.format_to_class classes = { From 8dc6ce814b0e30d5d50698f5db5556e20279cef5 Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Wed, 15 Jan 2020 10:20:57 -0600 Subject: [PATCH 021/163] creates validator for elasticsearch postings the validator ensures that all fields have either an exact mapping OR match a dynamic template, at least as far as our current simple dynamic templates go. This may need to be adjusted in the future if we start using more complex templates refactors file_type post_es method to use Elasticsearch::Index object and to rely less on repeated "returns" vs a final line at the end also adds some helpful methods like should_post? to complement should_transform? for ease --- lib/datura/data_manager.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index a063b28d0..9eac0d094 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -17,11 +17,6 @@ class Datura::DataManager attr_accessor :options attr_accessor :collection -<<<<<<< HEAD -======= - attr_accessor :es_schema_mapping ->>>>>>> 9c7eded0a (in progress working on validator for es fields)) - def self.format_to_class classes = { "csv" => FileCsv, From 2d8827600ef14df2bce6d97d4756ea3f1fe3e169 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 3 Nov 2021 14:04:21 -0500 Subject: [PATCH 022/163] add byebug and update gems --- Gemfile | 1 + Gemfile.lock | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Gemfile b/Gemfile index 88e0e14c5..afaf20dc0 100644 --- a/Gemfile +++ b/Gemfile @@ -2,5 +2,6 @@ source "https://rubygems.org" git_source(:github) {|repo_name| "https://github.com/#{repo_name}" } +gem "byebug" # Specify your gem's dependencies in datura.gemspec gemspec diff --git a/Gemfile.lock b/Gemfile.lock index 2361f7646..ae964e7c8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,21 +9,24 @@ PATH GEM remote: https://rubygems.org/ specs: + byebug (11.1.3) colorize (0.8.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) http-accept (1.7.0) - http-cookie (1.0.3) + http-cookie (1.0.4) domain_name (~> 0.5) mime-types (3.3.1) mime-types-data (~> 3.2015) - mime-types-data (3.2020.0512) - mini_portile2 (2.4.0) - minitest (5.14.1) + mime-types-data (3.2021.0901) + mini_portile2 (2.6.1) + minitest (5.14.4) netrc (0.11.0) - nokogiri (1.10.10) - mini_portile2 (~> 2.4.0) - rake (13.0.1) + nokogiri (1.12.5) + mini_portile2 (~> 2.6.1) + racc (~> 1.4) + racc (1.6.0) + rake (13.0.6) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) @@ -31,13 +34,14 @@ GEM netrc (~> 0.8) unf (0.1.4) unf_ext - unf_ext (0.0.7.7) + unf_ext (0.0.8) PLATFORMS ruby DEPENDENCIES bundler (>= 1.16.0, < 3.0) + byebug datura! minitest (~> 5.0) rake (~> 13.0) From 6d20c53f5cd145884a71dc43644b98db0e25df7e Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 3 Nov 2021 14:05:44 -0500 Subject: [PATCH 023/163] specify proper api_version, add xsl file for ead --- lib/config/public.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/config/public.yml b/lib/config/public.yml index 5bee0d115..11b42716c 100644 --- a/lib/config/public.yml +++ b/lib/config/public.yml @@ -37,7 +37,7 @@ default: es_schema_path: lib/config/es_api_schemas # current version of the API (powered by Elasticsearch) # this setting determines which of the schemas will be used - api_version: "override with value like '1.0' or '2.0'" + api_version: "1.0" # NOTE: es_schema option is set later as combination of above # es_schema_override, es_schema_path, and api_version @@ -67,6 +67,7 @@ default: html_html_xsl: scripts/.xslt-datura/html_to_html/html_to_html.xsl tei_html_xsl: scripts/.xslt-datura/tei_to_html/tei_to_html.xsl vra_html_xsl: scripts/.xslt-datura/vra_to_html/vra_to_html.xsl + ead_html_xsl: scripts/.xslt-datura/ead_to_html/ead_to_html.xsl # XSLT PARAMETERS # NOTE! If you are altering ANY of the variables you must From 6b633196ec6ee5621b5b7e1dcef3fd8f6abcfa15 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 3 Nov 2021 14:10:16 -0500 Subject: [PATCH 024/163] add ead to format_to_class --- lib/datura/data_manager.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 9eac0d094..96544e045 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -1,7 +1,7 @@ require "colorize" require "logger" require "yaml" - +require "byebug" require_relative "./requirer.rb" class Datura::DataManager @@ -20,6 +20,7 @@ class Datura::DataManager def self.format_to_class classes = { "csv" => FileCsv, + "ead" => FileEad, "html" => FileHtml, "tei" => FileTei, "vra" => FileVra, @@ -76,7 +77,6 @@ def run pre_file_preparation @files = prepare_files - pre_batch_processing batch_process_files post_batch_processing @@ -299,7 +299,6 @@ def should_transform?(type) end def transform_and_post(file) - # elasticsearch if should_transform?("es") if @options["transform_only"] From 6e281f63512f862f13e94cf64ee098691c0b397e Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 3 Nov 2021 14:15:08 -0500 Subject: [PATCH 025/163] add date helpers from newer Datura version --- lib/datura/helpers.rb | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index efa0001ff..831d148c3 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -45,6 +45,38 @@ def self.date_standardize(date, before=true) # params: directory (string) # returns: returns array of all files found ([] if none), # returns nil if no directory by that name exists + def self.date_display(date, nd_text="N.D.") + date_hyphen = self.date_standardize(date) + if date_hyphen + y, m, d = date_hyphen.split("-").map { |s| s.to_i } + date_obj = Date.new(y, m, d) + date_obj.strftime("%B %-d, %Y") + else + nd_text + end + end + + # date_standardize + # automatically defaults to setting incomplete dates to the earliest + # date (2016-07 becomes 2016-07-01) but pass in "false" in order + # to set it to the latest available date + def self.date_standardize(date, before=true) + if date + y, m, d = date.split(/-|\//) + if y && y.length == 4 + # use -1 to indicate that this will be the last possible + m_default = before ? "01" : "-1" + d_default = before ? "01" : "-1" + m = m_default if !m + d = d_default if !d + if Date.valid_date?(y.to_i, m.to_i, d.to_i) + date = Date.new(y.to_i, m.to_i, d.to_i) + date.strftime("%Y-%m-%d") + end + end + end + end + def self.get_directory_files(directory, verbose_flag=false) exists = File.directory?(directory) if exists From 37a7f07f7e72627842a5430a4f21aae108f9b2de Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 3 Nov 2021 14:17:32 -0500 Subject: [PATCH 026/163] add byebug to gemspec --- datura.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/datura.gemspec b/datura.gemspec index ef3099880..316a5e2a0 100644 --- a/datura.gemspec +++ b/datura.gemspec @@ -60,4 +60,5 @@ Gem::Specification.new do |spec| spec.add_development_dependency "bundler", ">= 1.16.0", "< 3.0" spec.add_development_dependency "minitest", "~> 5.0" spec.add_development_dependency "rake", "~> 13.0" + spec.add_development_dependency "byebug", "~> 11.0" end From 5bc84bbecdfbf9123c52e26212e0ad7e0ddd90cc Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 3 Nov 2021 14:21:05 -0500 Subject: [PATCH 027/163] add gem --- datura-0.1.4.gem | Bin 0 -> 91136 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 datura-0.1.4.gem diff --git a/datura-0.1.4.gem b/datura-0.1.4.gem new file mode 100644 index 0000000000000000000000000000000000000000..d1216675f068ebe213245adbfc7813a7c50f5b6e GIT binary patch literal 91136 zcmeFXMQkNN?2nW@ct)W@b(YI?PNj%*@P5hnbm~p_A@#hqK?z9%l5{N_&~n z&S8CVm0V@nE|vT%mmMuVOf5`3Oqr~G!Txs<%YTT2g9Gfp@&C|&Rucy0*soQ@{`e9>f>F_@b{>%9PP5yuT_CK8aFWdi5 zosz`DfsOD5nLh3#lbD;&S8xNkAF}FET@3O$@H<7}pINflSijr_4K``bya)Bz7 zHLQMYd_C`~wkp0xp0p1)mr6?P`IjdABT5voWWj4>_7K!XSv?W7;OdGXm)+A!imsaI z#IBB$GZGncbF!Z;fn9(JpUhYNi?2DdfzJ^$eivJrvZIv7gdt^4ZnWLC1dN8chLA(d zE=n;6E&o${>M~xei1fE6loX=89NfD!$Mg zR=A|3*?AsTSMFWdx!j@qF{q~L4a)W=f8Z1Zs9t`B>3}|a&(MBUUxQ)R)(;kMIZ(rm9X!Wmaa zvtc-iH|N&J-U7bQt%Ns@LQBi|aYPns9PI-2Ey~At>JM$~uQJ-oS8N%D-<;h2AJ3@{ zW4AAK#E#{!XUkdvIFSStk+4bC_rP!1uuoK+K+2o&0d~|Mn(V6xu(4v{-r9DV(`Zb~T|aa+SU&ZdFavby(-qi>TU7UMoNj<0WL3Y#?lPUn?D*dXef59fEAVY5ONj`vOz)%Rc5htCA1GXWVY;?b2Xo=7v#%$jlhBxf`0~2l23X?=V60=eldgX ztnADq$wmupSZ-|V+pb5h-L6G;+Lfaf(yX$|)_$=3+c~#-z;$UbtGC~qja=);NwCxH zxnjq4%%?3*(CyAkAo9-_!r)&Bgh_?vezWqi zN7W;jKet28C8*09%zr*^MB6hJtAu7RxRMa}IN5ASf>hsZ6l{5}McW{2AqiVMOohPm zf<@oKYR3oAuv>{7>Jff*&NWiz>+5G(Y`8R6L*6L`QzbQcA~fdC-GqIIsu!W<2O)g)pXR#H=VdYV!9oWxKZIJH?X?vA z{dvCK?_F@S?XNrD*8aL7)aKdNc03-xU7jvH8fb6-8|a|UUGHSz=qPS3&IqcpuPk6O zB`ERAsk@!@>$@CMiFjwEZ;5ux@iP|anxCK+YAfj(j=V`cfqBD7c9atS?htb2-dC@@ zY=IG+{vMeNbew#3cBmy0{MJ8C<@)?(pOF9ly?q`y;U>5}8DMrE`19_yfW`N#W3EC- zh|GxJ_v3p$pj^Q@i?hyC>b>>F=aQ9{Lp`TL-drYGkEROcms(pTn z$3Iy{9%ybsw0}LRBNapJ7x8x@WSHyMpQgCh)G1Uw-F-O%ueIBE$N7skb}a#ntZipN zZvwx~8;rm;t02vhMf{vN1HHA4pr_{3d%&E%93{*_d{)Z*F|9R=sU@c8-IljwF&_?V zi|rrMb3SQ3+I^Pg&-Ce%tLsLX*;?lc@`qQ9mKMF?d``F@`a7=g7)En)R1sHAv)=u} znN<=$srB1;b;(e9fIJDvrOHgqUpbHqO3&qQ-knt?({mDo9fu z{Fw8ogTi)>8b3B@*s5f`vj6>DQ?=Vr?ga6X>0ijoOs(I5CZCe9g6Cc^ zSd%{UM2Uh2nBmyGq^z$#)c1=2yK;clTEzKbFJ6-&ZVwmH?yTrf^kHLf&p(lAYjtBZ z{UAteEYGJ{Qu0G-Z~oN3Yq*me151HqjU>jl5i8$t4dC+Tu+9~9$YKKNzpg|EHmEk- zYI+_g%8%%d^#})+ey$G`QEo^chCX~A!58%#Xu%is;ZJ>TgyTS%H}URlw1ou5g|e}E zY3+}^1v-{poye`i(y!{MufqxB=m`b4#CHx+b%RPRTQU)i5i!^JI^xiR9m%{9XbRyXuv=yR6Z&VJW)p_Er-L;f11&*-n-Ze0p{vJK@if`phW0DR6N`2 z6>E(kJnKZAn(khQ?Y`SlUY{@@ClbF5R&Aa6S_`GYTL-dV^!Taixrz6yp|2_PyN1M=pYD5?j%0qahjuzjP9v%kqbyj4O_BnAAa2*7v^u)U zqOHjr-Ynfo0!%(61|OwPrxt-B9$&r)djNMaKV&sEl8ZhLrF$Gf3LIa`KOK)wt zti@#snGNNpR)|dqkCiadUvmCr&i8#6mx})qHW#@20#Ef}eB8P9-!E&6{+UuDo(Me> zU*D?5$RU3|M+1BxUW!hY88CtvAj8g6-#fupose&_GWHL8OZk`TiC`aicB59Gh+01| zttpvs{Ku0HHCvr$`I;v$5(o}8QZ-6dQ(HP$o>Bnck;c{<&4zqP7QcA4Q42DmxwP0jk@1Hy{4P9Z;(F%QSVA7>wG}{}dOWnFhsNFQ$hg0T_F?W?dKJ6Au+-uH*`07w zVB6@U@o`v$>PJ84X<+gM1U=+PPPhX(Yd{|-9M1sMA6CjeGBU{ZXm~BOOap#P1m1+B zDODS?*T4@Y?>Jl8`3uiHQAi(0+9k;HWzs$zg4Sf1$7E8F0X=Kuu!vcNkJjLa*@L_O zw%OPRFC(kD-1tvsDMM7g`cbvMb_NG`AZD<9Bz%6k3}u!Ao8{CvD?WQrgd?5&qFfc2 zN@+VPw4X^z)fZ4RZ7XI5!T_Bj3){oe`wT8Fsnen1V9;*pC0M*es#YW4qQBvz&F};*VcYwiidal31fvcSA(oG_pnrjXW(xWxk5b^BUD+VorEE*J+y?E z_f^P!1H+HMT2hpg@owCiHkJ_Iz@OB~gHB~(P?sNa`5y);^cG$EKNI%jKv~cdS=feb zULg9Br!8OjVgV?@bv8j*%Si^_IleyD>rTkVUe?uYEO@aR%BzcT0N*eiczV{AwJSk7 zkpzPs!mH=02f0gye4~J^+iya=-El^+t=x}_B8hKf;i4wj$FF^Sl65{hXUGX@ z=+kU4hLxL~y&&B9${%n;@KB~d(Zk(g`g5|;PR@ryc?g%QfjlOU?sp7QW3feC8EN@pLxH z23M*>nZxN_&gkQGZ`}2Bkj&+XFY;>(Xz@P1nSF8gQbdX^4o;3%Y2e^6j`^IjnJt(t z#r$}uCkJOQ9-rBI$8j%7TjL+arY5bIN=>HTQRtq6bV7`RL}kv^Ei4B>1-ppzx8*-n z-Lt0OqtwsZiMen)@6x*-&RQk1FCnk_Y9^qnWTx0yxy-2S4(NUa%lz(O<0Tinao8em zUiKl~Y-HzFUKifG-4S@g&WlG-%zeWur7XS^T5oPXGIYPd^rarRL6gQ}erH_{X3L9u zN#Ar_(JV-7Ggt{%D&TjA;V|{trM>PYh;WF%P+TyFygG)_17K!0fM64YoQj%xEX!n2 z-m0@>yhPkpz7BW+9Y22;wKr{~a{}~w+iTS}ojfj2VvoW@b#nmQkKO2NVu;cMk)?K7 z0DWTZo0bnJX6Re1h-!>9g8DUkeE`ScM9~?M{kg$`bPqJNfnLrAxDZZE3{s|L6GysA z__@M5;p0&3QVNXMrp>$+!Xl5D=UvS0^3f?vlNNUf#AQa??hz~)@9CO&!=ysEmbwdo zpsuPNOr#CVOTJoFw->*oL33LtQctnrV~ngNiKS7WhbCuJ#0+DKm{)|V^5kD!Yy35# z-Sq3pd3Y1Jb0v$|z$cPB)z!c)aVBpMf4CW}BZ&-DPzAIydjVdLc57@Z7*lj97*k_P zGp~rcRe>hQjmg-pGmPfrio>Dl?j4aSN36}es@?}?$@law zjQ-unx+0DHm-LOW(D8XeuEqWo!}Rydj^ACM>fpEa(T88b$!kN7irn{l{MNUB`uun0++TK&uSJH_<7@Ml z^Zvf3k*^;^HXjYj%dgh%Slav_GWYW*6&`1j(VaQ-3BjRWQdL`z zBCRLcP37KeL$}JI$5w1T>}F2=*zo9)4U-K0;0Q2*9jdC0FvB;=;2kt*%t%%nu6X{> z78l~n6eJ4tbu&wgs?r)N%0x@fH|I_s&t zLU1zlqT#+7k*-utEgnJz7vh%b6>8n z?K^uvr4+&IdOrhb(w?*9#$@E}s+Jps z397@r315|OMrO)jEHBq58}dL#3Jb%42w~kqD6_1&eIT z5N1i}$$BUzI%Aa_VB)(Nm5p5%RW&zBA5|LfW|DES9 zu6Q&999ixZK5~Y+)W9Rm2x=L{1HcCPckU}aJ)q-MAaL!#SP11Jf7r@Z$AC5KYh)>P zW&BM>b%;f=8wc?^ww!xOgyj768dE^8oh_qtXE~`b(APtNH@q_SkpJVViimEfqPlZs zu=?Y^>Wi@!PWTA#H=h20ZtHvwds&+WhIfbC3N?GIYbJcd=mKLi3C;jB3)n~;h#PB| z0&5sjP0{`2mTu9Fach%|-Mpgfw9tY4CcBqqPG-bieF&ak0G$rEZtuiSbQjOpcl)qy zQ%7{sx{Gdf8qiwHymvMg$xoyAZ0qR0T|R49u-rYF!%@#h;W+pSC|yK9m+OHfhZmra zZrbqd>Lh}Z)IHVduoYN4WNG>d@ixh;1-12`$0Dam5iXt?0qGOQ>DDRnXcmQU{5;Y4 zM6(pl5sukkuzj?I1jqn>A z`+h`(>U)85>PKQ;I_KTK!}1dYcfF`$hl@kHYe-1*&5$$xahOZd6_3;yxEZ6zArzJX z1Z9fmVtpohKullT3&+%bFb$bUr!i|j!)UzNNmgW$>opk$mA4P}73_X9EYnYV4VZZ% zQVF+o5UC3iX}wI{QG+SFJhUN#aH7+#OuR}ijn5%IcI>GOecOV*p*+3dXWE$80Dm50 z7K46uwGPsBORL@&6Qxx$8VnQe4>YYh^FJ%?x-`*uM^ym_Qshe|czb_(J5gmrb^peiGdtf47AGojJHPDVJA=bM&8Rky=B*b-pBOiZBCVYCM~YRam)uM}Wc^ToJ5)g>)C@i6m_H|pZ79gC3}z_l~UjCevrvFLb@r6eTtdpOM~jH6mc z8Vb7YnFqd2zn|)&zX{}Szr8)+2}ZvQ0zQmUJieY{!KnDtWnYCm(J_|sKg)B*YGLrO zfR@D@8A-@kg9m;zS&~94SM2K0*ivdyG_n=|2_3l9dCHB^xuECKjTBuZwUVaK2 zQAdeI#$V0FC#pnLu7q*!&9dtv4A~<>uC=t_c5N<%%OObWM^-ZT49ss)B>_$F4yW9wOszPQ`}l>!ZXB zNu&D+^5(}yo~U@-1$bc3#a$wR=HRHZwxO;LsZm%|I=daIT+&PkM2u7147YaUx)jcG zmOFhUickpOvz=N5mmSa_$Ks=5G=hz}D{`&~6On%&%)q!T5W{-FF87o%_+1o4p`Z3e zvd&Y#@zwA8^V8-dQ5pS98#UpPP^*jei=5IadCfAFTLP~0>*>9S9PLG_viBSNe4qqO zA}Y0L=yV$a?&VMGjoNBZ#Ch6AFoASF}UVW*T8>J2Po zN(>QgooHuKDAs~n#~@xi%@#LB5+V1Bzlf`B1wDsD23Zt*n++$a&j|aY!OUNjcYM@; zVDG54MH9nT>xa}UC;AFsF$ttL9I5XSe)XeQ(C0$H7`s+GsukNLn(Kzo2@%SGqA=4k zi;?5-!1p63K($UnH_Rw*DGLtw-nNr+d8~D<>)ws@5JzQ~Fi_74TJ0l$!S@W|1Cy=- ztz3m>A9TRe`#CVnanrNkha5(NDFXwBe54=5TvZ08ohNY3eLVJ&m<=aYn0;8Vo73l6 zG9z;MuT1hds<@9H+s-nzO+GS+)bSs_DCU(}#b*NEJvI)Cc)V6*P-GXe-nUcJ?X0ik zU@f@IDC}%*Qo~Rv75MQwz)~=O!!AsBZ$f3<II`6yPYBm=d!ks<` z1`d};>cy{3u)b7J>E6+nSVYV1lj?;HODFJ3l{4>{42(b9kT+Q&hE~Bh44A6ev0YDc z4!zCtgmFu*Qn&aEy>1+*RLNRQKk_86SCxes0#pTyP{g99U%}@I8)33f-6# z+O3jmHUl2&q>zt10tAs)1f1e^rXC~D$M~dqP2CI5y(zfI7^@E>UKV{PP zlM-Di#BwFNdZd7uN!rl0nZQEzf2=d^3bcu;Vfc_*C2-&VFd)K134M}+&wb!gU?TnO zHetX$(q0E-C=(Q&z%7qDJ|;# z(nmqM0zU5fI5{HcLdacbPigK#8 z$jB8ikoS_(7w;@tathqS=M&L+1EV6J_I(``o%twU%Ta4{d07&Y$e>fdFv$87euHq~ z8{8vd)ehZiXSJ!MavBnDEt3d+;LK(LAV~?3z(pw>FP%9}8pUOXk_t2dWRQ=1F)Kqj)&c08nV;PB=h|F6|QI+a( z2~SF}W#fy2KK;qoXb=igh%nnK_9|tQQ;hNJ3ULeDBtC%ZQp%i~KDg*s^O4pGjmpaX z{Q~;egQ~v!Cbhz2&}t?lF0J>&Xt(nsb~BuGpOS#E=?RnsM}H)j*=|a~ChnAE!96P< z;WL{Wgb){}JzTfKAcqeaS8!i53;-sDX5vn}yw8#$zJ+*oaq}a7ipsO?8s*l&L}Jd? zs(;#U*U=5mq}_R!MWij#h0g^{P8=CAnZIi;4olmWtTxVrc>P&#IpyQ_c#EJ zW)T$+(K@uB+yX6g%ovEMBs9XYZVcKYvvE}X@H@Nh zQF~J&qaap)>}pF1Q92_%l}vC@dc5hR_%22Yj#2e$VB{wL$Sp1?AA z_ueWTxk4c5KV^gAH_>DUe5)Tdk9E+q)}&IJH1x+-^phH4#TaJ^I}SvQWrdYm#w)4i z%kZmMT}qAVQ_GRUH0t{}ZKMy+(Fp_{jrc&dWc8h;dWk&))fyCg|1byWGC<95V4=JX zhB8X#*J3I34$|koim~?fH+NA}z{;2P?}=~vi=(;oI9(BMitvi?_6o}9Oum`Q;SrK< z^eC&!dca!@hu@GdtCb3Q8>wSPdUdSZ=Vg08aZo>^M7z~R>?<@Ry+XGUZ$p7@t|d5; zX;LYDb5Wu_du1fR#8-f>I7Q<@AbII0g&BqPUQ%q+|B?V zFIZ`*rA0)-6izup|K)1^tz8F82^+W9vW$byof0tij)FTg`R|E7FkGYmScIM?RGe<& z@Y+!UO*U0A`4}RCT$*Z`+&5)y(p(67?jlO6+(-ccNdiOJ(g-XMpT*Hp3Gv7-%y4MW zFXw9Pz`r1<_b_=eY49&9ZTENd?Zo_?AOL8A3-UB)L1eSFFsM%FMnoe?#^8{q1gGg) zpeIk2jB%27N85TXozrKQpkGQIzDU%!2DRghvlomRF9qQDRs14*;m!*QG*Lg?CtKI> zL9!Hw@5p0@4)q%IV26Ues15Pg7`g{*K3DJ-X{>O;Jsk;Fq6)nps0e!j zZYKhIgDej+B#VXH=(wn=hB3v~z!jWO$axB5<#ablN(KWU^NpHSFv`NL2{4OVAC_r0 zsg#959BhExfzo>(OrJNI89X*EI2jWTdF2YTh|^(Vn!4+zC{)ws(@X_585h|wZt6~1 zb^`5H^@Z7^Y9AtWo`rMpNSI3U%Hdrz=Eg%;QN&~HywZ^XzN4WPP=0(4fFhZLG0F=n zDcpoB7E^tgZ!Q%3PZ)G|{@=YI{T3h140Fu%-gGax=BVerJ~Ms(G?Hy3K44f z=gmsi7EZNiJqibHyPp*i_A4jJqcy>vRp@|NyM+=8RiZ0_2mu388=^a#MXo4|2nI(~ zWjq|KqKRVKC9|U`aaq3EOu(Wm3=R7-%cjwz4l%Ft?TV%IMJN^8Z-|Y1xyJ;;J8Eh2 zJoduwaUne}p$dcCvS{q#_3=!mcFpVIneviyfWF}uLHYNHl!M}{v!UT<=h4S}{IQ+% zX}^B#S5EP=+RsMR+3hOuPJS?ZN%EV4ShV)>9M$d)nsYTDT8n8SCmSh1e8B ze7>9Bv1>5^;mkzp$Hs~wGo<#wz>RVl%QL)bc0JXHR5&}0y2{ChQ2-Q=;T(Ex{AeY{ zk1iKBq9gj0a9bDb6e(?sR`KSFl~Wo3?xn}(9jv{)J%QK0WBU4!A- z0O*!j&k-jOlwN$1nadW zY;>z+1`Tl(n~9ec9Xrf0p6~=)C~OltDfon_BdQ{&O2C^+LV93W4ynH;9@qQel@_B# zh%KwPCND6E!-E`r<8(YqF%rjP^WyGQH-6(55Kdl^P6g8!EKceNIdDElW-%(ddYRF5 z0_#F%l|o!#XpKvQ~>QjHu4^`e(WmfA5{dY9K1*Bic62vgYD@>`e{ltX(u&b^msg zJ8*4kykTdt3wb35VipHsrD5I~Pwj#tSY9~TSCc0;n%uF?8J%SSF%YiRwn#El#gRKR zOF=d=Ux%#O!Rt|vwxZa);*DZeE{Abq{Z7n(SjiTf%*9#clU6cZeS=8RQh&p)gyaF! zZ0#}M=vHJkE!4|mB@f5yWTnEGmocBsN4Y94f& z^T1~xM`g%zEy~0EB1H>I;WU{+Dd)`;562LyPLrh^v(uWYlO@L=&~wKI$;(o0QnCl1 zM3M5!jLMCaphgHhwi#c7o+(a|tBlbF$)y;uUPA1kg&Wg9Ml0dX8K8uckUZ5m3jZil zHj2HulD)V`gRY^~$}WmW5j@UokTo#m^Ls!{F`QTz!5)7$*woy!4}zHpR<%ro=jc?? znItABu@F=U#nHXcB^1>~dNU$mM3cl6=O}C0jKoD?Bg`o0N@MXv_j|f&ZXm+g+csodJT= zFFAE~%ZNC{C`p|lRk$s%vAS(EQs8Skz(G)sg&(99E^zkhjnH8ki^~oebHVnbN00E5 zrv)@%AO;hm*~3&;Qdnm}9?>+~2E|qt5D{!LhFFR4{5(;(Oj|%(Fx) z@SDeg$uiJLbgn0Lv-A$YK)EioI^XoQS@@%vY(o6TN%Eh{<1L`d54L5~5s*x~Z58`P zlS(5%ObPJM%YYg%fq=@MN$GE}ftVUd3o3SNhh>f91*%puh%nu?;jjxNs7|Wqi=2oi{f!@*uvJIO^JME2lThfQ;!FYv0sEP^Ml%R`3 zs-2lYvyWOcu3&&`vPB8Ngh^7Es4|d@4CV+@S#hknEg%CrYkAsh$5YWBlY~oD@1GFw--J_OnIE2}OU! z1*fINocwMYl4!*)?}qGkxd>;?hlx=<)PU9mT}QZQk8G%j(iA&ibks30vNw$HLeUI|SBP zs6uzS3W~wO6NC*WFni0Y3*Zu+6~OXziVZpa@R>)nA+)1QDn+d_<=9w$!RAx zlcNvTrnON^&?XS$1lhcMkui!4784OikSn5W}zP>mu8988h~i^{(OQ9cKJH6dBc~F z>f^$ij_Y*SpPDcMxNEgXb*5*#HTPlXNA39bM~_FM=7ylzG_{5D|Cy#>q)Eq*0$Q}S z_SrhC)<&F|?;U^sZ4o25E;fTo@6gBml_S~bl&?B^oJ{EVPh{pI!hMf%Tz|Ha)!k^^ z0K)r77s_w22m7zpBg3v?NW5?r8jQ4qX&yu(Ay7F&a zzr>eHXq7R89iJgewX(X=3!t0Co+>ySX*LxD9uy&8kAlMGfsptDB7~3q5NlCJ=I^jc zY|36fRle!12f8IpY8k5>QscrfR);4razm4+LKVDvbsS+W?H`))NDwmoyAXdx#MSxE zerOh(m@0UJ3Y z>4MLc-pIA6Q!xBq9D=44jmC?*+lhTkx6pXEz@*P!= z%p>`qM=bTa8R!bLMxbn84jGRPuDmdASU+MKKPD@0Ma`K~B27Y;^akou63q614W28+ z>^U`c9m^SxqCmel;y03fs~WU7wbdh}CcHe>F!%K!YT_gSxaw@Zq`4SP=obW$rnz-F zC5s(fWJ4WrqsNy_QJ@f$GMnMsx9RnBdtRrS_Di#!^UkgX83I?KzO!2KsnDTia9goe zp@EN4nCe*P&4)YIvMq@_m;zevZyrYoW6GXK`A@fD?E2z`1`Wpp;?xb|%kLzk)V6=+ z+ZniAjNN@?RgT2d-KGpu*zIuIHExwEABxJvR^rKGVN|$A-DdrZ05F8E0nr%IJKglZ zO*9FeQw;0@U@ccckZu0WVB<}x#}dk-9AZEoRTP&7?YLXT13O zoMWRTNnYw2mWUYzVn$nzo^B}srjF<#~Eh*p#}K2KE4V2KVgGU z$vcv(-*(c+flh7j-=tmJP@(xMKdSU&6d=l+F#a;PUnGCYz+W1Pp77{qSc$FG4#0&m z7=W^h98&AKPm?9N8*fh8tb*OEXrwC+1330l;b2}MF040P;dh_RUsTqnn(}lCv!sap zcSPnn*lw<|C17F5<(?iSHgvT*VvlRGY8oj7X+J^L>SYITIcOb?2393!2n*U(Ax0M+ zsl^VtJ1ZNr@>@O0QkbJn`HSapk~F8_6BOjGL?9LHIzBQH^)`*Y6WoXyF=0beI|oWl z8ERTWbVRRIP#3>OZLC@poH)qzFi`9)y&Nk-kH=a>10()U!H2envMCgWQ&;22&%7Fd z>zjbKBv$7=c8_i;(VR{+UTCGXb_uD}(2oZAqd1!A(c`(${p=3|L1uWCRCvh~!P#T7mXMfS zr6~AN$!Thd;m-gR>FJ^~7)ivS2!U}t{&tnx3W^UA22drRMGpSoU{KDl5VYCqhH`5^ z^Kpb(60^&IRt}Xljj5kB6A4&jrq?tTPNz2!hFZb|KsUhfK}D_PHQ8h) zMtE*c|AN2SviXe@H88P0g;5QHo+bJW&g5Oj*{?j^kU2|AQ8|?Vf`2ldcLo4%F_Mk& zkJ?I}i3;RC8W81$y*~CAc#7KVHNJ%Lzh6_6zTuD0$-CY+np(Eb+qpz#aT!I}q06Np z=grC!zhKTB^Jfg`_m28>1a1|3MvP_lTPvZ*bQ=GL8bBU|Jh#-;sH z&$sH9Me~$kD<*aLfsFX`R=z)4wEH0Te3`uI7J&q+9`A~LuxqP3R8C*(<qidqP;&g^$ksOA`G7l5SfZv80SlN zVfE*e1rd~@F22;Bb%`La8mR;EA4Kny1!@bnAPjJ}&`kObrIqLD2TZN^qOUqUODuG? z;87H4nf2Rv*%NCGo^Io3q?g~ zNX@aNOVLc%K+A0faR);KDy!Hb4S~St1Q86b3Rx5ApOCUDm*K1ogra`rUUy|P($)-5 z3{<5Bn+Hi?=hK78_hu<<6Gj?Ug3PTag3L;PuP%_9syI;d06Bkg6$@gk6*T^GB|R9{ zP}w;e;+%*qom$T{O}*-O=($61DNGd+0J+crF z7YZQTN|)~Q7FY?mjH`;hXFtPtfly7>(i{IXiU-`hZ?jtT*VUtypHJf=E)?0FVM}6& zZDDu$<#*uqjCd`5uo=pGi7O%#E9E)vBYLw&vGKGHi_fke{N39t7E1SV(Hh zKuK=-24(_I6!xgv=oiyyE3a=QwHSn_C>K*0jw01rA66Q7EnIe{1RxOmEf3GX0pe{? zh*JdsKOO+Fk=G8#Q@`bT-E4gcADizu5}7goYK8lwD>jnHeq@?l!rVCMjh!qx{h)RI z+tg~%0zHlUC?mrqK1UGp2$w8^htAi@gGuhy9pVcJ7BL{i!_SbO{D)@;wTX;VqAv6A zXNjVM0_|buo64PsqarG^4wQEWrTRfiBZ5o;Op@g1l;o+U8KRD0N@hh@I~jWs?7YqUQ8a7Wjl#HNG#$Z!Ml|ifq z?stBgcqMb1RtUafjHmmQ^Nsxz7RrE2Q^u0eWY;kjOJ1dN zf5xX9`WF&@O4BE)6eqYe>H+2nN1@cgRqSC{XBYCBM z(@DmKrck@EcN`yKTc0XrvND?yWMU__gsG=Gi9JBxGZTG#S2l-L9&O)>W~$uwpFZw{ zfmE6Uw0ZjwR=OU7;hjUXMz-D|<)ueSD$0zc>EQ{_mGl$q(`3g7NMcqSv`*C>@e_zp zMf6bWpIWyINTMfjjpCuwv=Fsu4-y_!{3(uYmwOobI4-ByJf&UNoFk*YOL5TVi@Yw) zz2HDsntLu^NR^r@tmya+&l0@lNNt(g?r2l2+Jhz9Zbyh#=FS>4svzx_jR5S#R5L%8 zJ1V#(GzYs7rXw>KqpN%kIxN>$C@UKB%wNQ$lC4UjtlR;??zG+gL^uz}LP(b&YzS07 z#SusgI;(8})jJLI)+%5$iBc6=A7EOq;5HuOWDbj-hg<)s>kT&niKFPAMlOB8 zgz8S2h=Uj!<-USKUTjxHt$N=40&^5!NBJjDg^Lv~4uPIinNrr;<&A&&pCLgdLaf5Z zgg2Or`kHJDOc*V(z%qvU%;^Q4o`VFEmSp=XhdS_l(v(g_Nl_6jMNzR1(7lVN30{s* zOq_5WQ22?rrw!rh2<1p+VGnP^K(++|j9sv14#C!kvEzkGn$9*Lm!{mq5a}J=S6fKQ ztXnT%=)FG!LCK*{L|kI^2FWy)S;%^q;25_2?68CcZ%+EKn^W=#TS{P&I}PgQ;gjWn z`GE2ABxR5rD<&1U5*00x=W>=i!}3m;UH~T!H$@2dgS*z?{h8CKh3zl-q`OtU%}c?TTC;IWKm?C#ygJ9iow>N zVvW9RrvbEgZx=!y->oJmepH@U9I}B{nCeO~{c^=hrVGf$B?-i`ZQ}nB?Y9X) zw-luvE2QB+Jl24cfS86ECf^%OPM*ueYztu|l8Vlu{&jPa8=EO&Q6?|z(DmJTAc~yw z2Vmm(K-`6mKx2tX&Dfou3CSG<3$7w>G;B7EV_Y?LlVUSS$sn4d$VrPN*#=YcbF0%3 z=@NRKyoh@F!aWHamRJ`Rdi(>-=W~0fV`nEGj0ItsJo%Ouzg;f$9vYlxr0#~6wYo5V z9B<5{6bDWuvX1m4^hOcofKg5Xr@Ut5At;N23xU01`R_!5saWca(G&N{#HWol3#A3N8U3d>nuBju8)2X-dLlaLOK0 zp(-n!>~HyboGcIn9?u&BX3jl*Ma(-r%3C9IT*N_<9h2Oh&HSv~)LFxd=h8*ZPSXAiz6DFI!$;=YB zSdEVoD5d|Qoh?fEOT)2KX`U8Iq?<&e-$czz2f52B4uy9EeX4A#%7RWTT^s-;@CK~} zGN>TKw+_mq)0SB@VHk6Sv;3kvTBAvxkS2HF4(%XQ6D6MoV2XjuQeNQxqI!@11P!d@ z%*PDC&M-}Mo-MKKV2DeqiH`=_v8HZN>0Eh(%m57t%Ai{Y$2J7h9`Za3^i3qFki~F8 z7Oe8Y&&DMB5xg4G99wC+5kWPivI$g4IACbY$Qqb8>8UJj!z_)K35EUcJiJp{m}0R| zcR#yVgNA)-gyKEek$r>{O`f?ll57(k6i14B56^8b7oK{Hi-g=ZvPA93R*qPWqUZ(o zgQ(-wHvjc|VH=od`Ls8@?r9O^XjqM>Xv`)WDc*vUz|x*l3n8~70lUIM;9G6s$OsfCCPvOAyvCjAd2yx*-rqHw1=QkYm!3>z87AAV zExpsv_~L3aq`?H5HANRUu$qfVXrzxuAZ0xLlWm%|BS%#p>6#W3kFb~~aB8~8zh!>8 z79|_&QW*vftS^X9;)08-ohjyp^ccp>SA-{%b-*C2)rCtuiy@tidkOytkK;YUMO&=0 z@&qih=q3(Hpl0YyHEcyhrw*R3b>Sm3CFr;y0_wQ5OL76xqm|BuyWW{ZQgjhRtuhY7 z;h|(qi)agfr2QJ%!tz@5aSUELJizSVf<2F7()}m7q-Ze*sBAw!h0z ziYOFy+?&b(r^FwfYD>G|2dsBjbpV4KF9d(o;GqTqpKkWLaX=JAI-u)x-s*l)uuvgzii7*=%B%z{x@TWc)DsKV_qfthemPfYaNlO`spl~U|M zYaa-V;8zqD9FLN~1LZol1~<|ZyoeUr!{QASwnqsdk(3bH_F37Klh2X}T0x-PClP%^ zKa_mOMfb;i=p}Os;k1~<713BJev6ghw=ye}vm6)=u8nNT*t@hTWAE~~au(eWLQ+N? zpvVG(mQdQw)n`dhQ|Y^U(Lg}bPJ!tiKUOZzu#o=llmctE1kye!!OR%O{O|wID%ZiCYpK`Ip6r@K?&^Pm0;z$&6YUS+vcu;UV4kF>o zs8Wg?HJ*;7_++JoM-Bs+icDT{N(UuKHIE_D~6HDH; zn*_EYhfaa7&=ZH9MVpAKC{CZ}#lAfh%}vBp_L_>gXU#g(HHW2wh{^sK??X&; z5l6+A_$Js$J$$`z1(J#V*sR(MJExbbMs@Wzt7Zc=c~`ft(bY7qK>wSk4pz8V)AfAyj}NxrmDSL5DiZ)`YPr8MK5nibOub`Bdq~ zheAP!L?lHem=0?U`NAjjuc)1lHW^WLPTjY@ZogHsu(LE)+GdBtaXX#j_FB_uVNkxk`nr9oy@(Kf zvM`Ie5+VPLiK7dr)p2ElJfe9G%3%;#Gw?baGPZ><$P2_Y#48P(nE_gRX#(6&j@1%F zd5SuO21%zI6GXU%#binwB_9=6xu;!ZNM+RtCp~=vvg&nx=sjaSS0UDy7Cwmt;V^~l@FPS4iId+#mh=z>~B3vGes{2+i z8|@N^$qNahlxj|lsyMC&IZpG9ihI$69zTrcHquL+YNQ=w3B6wup~8I4tI3Sp(#dRT zC!CzTg0Lx>16!_&_90XVGDag}H6{O(^)Ml(PnL={pA(F34nV4!oDtByg zVT)rNkkwLIr((3Js#O*mD>gR+`sIXM)#4RDr}6W2S@=?24gwlK9thav5cqXa6Mo5h zIHY3G*KBrhk>zZa(p0Mo2WME?LiWkX=s`plX%q)XXN(+4HeIwV%ybQy6fkO7k+!}0 zEV1Um7A;`((rdGAl)__y?JG);d6g#<%V0g58aCOh3s7e;N} zzV%Tua%qmX5^v{5CoQnd0aRGeFo9>Es5GaO6x~kNz6wf3ViILEBcVYl*qy^@loK1K)XNL|c!vXZkX>A$m{gi~ zw(7zrDAf<&z%ja7`FiB5m-!P;XTU3so4IkPB7~mtpT4ue!JMTro3M~l`5GMQpd|(v zTwdC!Cr&gLi)BKUq1-`UGVz2rE+*DaJ)~byvKFDf0jOug;+m{&05A2&uP z%6V-1iEBZVXEHbh5Yde>ByJS+XBmoBbW_INt2m=x+9B$?-^n4PBU>|%q^#6Ra>2+~ zPi08weGv2PkP)Ru{fvox6#m~0-5dcYVP5Q$w>bXCgwp)4W& z_Z<9{cAukoWYacfvXKwTgMpknY)Z?AabO8CirB8Ec5298GR=Y4N|R1Y?vt|(7zA_V zbA-tOm}J({edlu0uFEs32z5f8mf3BJv8`s_LbxllbOc)zxC!;(A~3x4V-mhuGNFu< z<}8sR#_!y(KGsl z(+BbV?t`K7AEWx|H* zg&0RPCsb;9%aV(eO=j_EQcH!CN~DKA8$2+8?Otj09g2h>E3#6RDo}wPkpq?i>P0vK z^5Q^`?mSy^lHFSXBxGb8HmKCXbmj@B(o)o7g)*Ur7J_&cA$>OH@YYi&s$3c_PT%6< z^z2{5|CH?UqfY;_+59gP8cO0nHP(&Bf7)93p8yGAaw-Mn=J-J}H;5J;iIprHBlR`#XUTKCD5l5DDRRv&a)N~5jYZs=&#f%*r zeaWi)z?Mu6KWx04_b~V$qC!o(33aP1ZwYo^y@@W6rmO)DyQYTRX3Br2sN`ufu%SRV zDkU>DV_`FT-m^|S3-!8WGp%}ta71CDyyx6TFSfx1WKnJY%nfjRN)eiUFf9@B*vWD` zF1EvO%C<8P_Z*WxVq`1VHfjG{2}l%pvuVzX zhyqh=mYqQM zDZ^-!y2#2B;g1D98IO^I%}$*&og?w4tTAIK#EcfbB=CLVlcRYT>hUWDp~S6f@t8>* zfE)-##6bhM%tpknflFD2qoC+ld@AG&n=O;cyfj-**^?pcXJyjx@;N0p)j&9D^2W=U zIUCa-moWqpj)X-i?$Nr)b@DlK^-l|`74ObXjs+HJ#6KLPNrpQT6qkjZH-~`XW3&$+ zq)oQjtK{R@{vfpEni#>rEE1Rqp^pZwXd^aarSiVYL6Z?G@l9pF+E`Y4j#5HFa#U+f zw@lH`K`Q{G<;t5tm@Z))%|IFNDQT&KAyjAr)f6QU0V)3(wsTe}C63hxR5Bms-r^Rt z1FMoA(deckbwiv=mIkMimQ~Hl^BLmoOw}c3#w=y08wq1j`6-T~l+80g5y~7@N{OoM zP=dVD;Qc^LBAb|W>CxzQnzkd7db5B)-mj85D&vMmi0;&cZ(k=HA{L|I8{ z)J)>&pJr-+!ZwI(&&uzg0zb{gXa&H7OASciC9i@FIXVwUr%*b_qoA#(KQO%~`!b`I zk4q#nO$Pdi6)|zjeI=BXf_*n$0MOB4;Kxjosejkt^QH~g**KvAf2txAkLH0q0zM#}{BFv9|87f1D zbayC1XbA1Og?Q9n5DcSjI`$uW)S8q8)NagCj5)B z7}f2Vc1ShIF3&ilm^!AS7h$I<gfQ@O0t{qiN;%?E_DZ0S06Qea(I=u^gfK_Cg^`*f zZlMw601)H37^Hx3?$maQd?WxOi1nmLZ)%lZ0!hZQt|3CJC&d_j9HUyN8XU*wR#y!_ z6t{!-Q@|Q^CU=tHp&04up-d>OlvrR6tl48{&A`)}%-U+4q|uu(In@a7nOx^Q&Vl7? zM(0BK)Xbb&Gj;0R>C>BL&J=5o*I(c@rsW}EM^q^{l~Ov!nx@fbV-_BfMCG7B_$&@G zT>>!!(2>h}PM{!3=PA7-(U~ zq9tdh0O`y|qhY#yP+^GVXUNi$173*WSGuf;)ZG{t+IKChj5veTT`3YiqnDK93o-@c z=y`UD#b(f#Eo`yG0L0y+S=wvam$Y!CMvv~n3D*ri>w7cEtg zSr~TQv+%Ymg&@w3$S5mbWlgVyhrSJ|x>ohLg5{I;4az!VPY!g^VBxGcuWwdW}sslXFC{ zPNXQ)s5MR}StMT(p`7y=9#H04#{h5oGAYjqA&>+ZK@j2oIA#ShmB>aya}z0V(M*!+ zMB`&cn}=hg(McrFE*(o`Vj`%;221TyrW{&$1RO0hpbCb^w;C_JSpxF9TGaNXuua^A zVm69~I%Zh4SCJ`r`Z_}f+_)eH zjSddmcyP-cOk>RtYdDr;&5CWKx+&RKsh6k_uK?oktMu-7z7{CnkrEiCk=BQ1hQ}dQLA_ijRu`y z^FROrX7^xeS%xGitKK}uspkGNS)<0=Y=0Xmz%qj=KsRfviknK3rz$Jn7dMhra=(5n z4j`1Qmbx%D0^xP$y-#|n9L1zex?Ulhf`&1-!Bl5Bn{j}mqQ+~cw{ioW^wUn6Mtcs( z^$rrL2loz(*n81kKAyGwguF+1*c7wKOUM=zu$&QAH0Yqs)^9yw0A!tkM{%};thtJv zV3#a6FH%&i1p#27)gH13^`-2wvpk|?QvfYvr~tmQvP0B=Byw=e{Fd$m+h#RK_Yy%1 zjx>kN>S&~+<*@cy-OxjGwEPw&^yQBt+yHYx%}sBg*~%fNFiC7W#ttjkE-!KGtofJ# z08OJY?x5XkbaomwStGjFVpV@>)adHkpUeq?=GI236``qMv(ZSejRTl%iqw^~w!tb^ z?R7x4zLen=#63`bLnqKRA|-sJ76G0`_d0-P**ane9#)p?IvgxI?{NfN7Hb2&Z($0} z5!_BxX&KVBjS*1<-x$IeK<4?>fkE-Wa=ZjXT)Amtk2^{1d#ZoXNgMl&l;3>!vWzFJ34rnZrLEH)!?r%_RjSyCA#!p}~|RTRqOe~z$fj45nb9hxYH zhb@|{7DhW!F-C;7NO&aNkg4DHi`Uoe7jJ+@+kmOHHZ|v>aoJRUlmh>()Bk8$_|e*c zY>xgPKVf_^{a@EOYX7md`2Pj2?MqBMN0?sC#oVo$;W|4vPdva>Vm|c8;{9G0Xu3`rky4u$k0u9szLT|O=OIK89ee?zsaaWpGQ(_!%f4M$0E05 zFNSN>+UV|aO<18}6N7&Y3y(kwn$qpTCEsKm7`%@e7-PXUfwlC_Y{r7w)a^TA-0ayKRhOQIp;!>7Ah%)(biFzYk}OF6aXRyr>bo9q-AyrqPK*M3>Bbo2ny8u z`{XB|;RyZ0M-pKR+Ga?=MrTQ zI6f3+^N<;~1c)RJB|e@svyzt*s<_HIJRL6g1p9P2LO5*?ZlG{sYL-R(43i#hwZE_kcraexTpp) z&~m+<6h}b|%%9Ng$e*g)$)r^xfNm<6l%7*;n2t^N+9N1$2sa_Uw1OsNv7|7hk#;N^ z2D0o}NU_Fd4pHC9!-A%xVIIeJIq8HC`zbcf@CS1m`elc2VdjLgWlO|AuyY)IS6STf z@{5ckBB7;Bi_2yt>P8^XX^Kr7#P}tZA-MquphtGcyt2~b5S_y@{2M)P;)|8`Vk2vv z7)xb3Biw)&LK8_lvYJ&g-^p?VAY4y>tMdJ=bg0+Y5-r_`9Z znJBWE@SR53AkRBZ^M?6<%0?={HVy({1&*EIZ+yIN0%`!9EF8HiY0l*g7`5@n3H#R8 zt9O*!%uH8D#!ZlC8}_5t=r!#Of?Lek&%l6dXbp>XJNkIMhI%BS-{A?ga_h8!C7($&pw7A%|Ts*1-G@SlR0Y8m=G z(oCO7q?3OeQvy1Z{L48rTL;T+%BI`G8xg~Qn)3eBNL7zTIV?#P`3nX0B)q{ zm;ne#XxB`sTzeEvc}mCpn6VJQDIt|0lh7@L{Y&t?&*p4TD9R38ttcfp8TeviihNIy zyvs)4#<_?E$o@^3chD&8B4-k5Q`;2eMmJ_R#xMgi4+@4@;=slBjLICBJa>38th8*X zt*z}Kw5agZ_Q8_dy@*pW9c7H|ENKQ3Ri>T_#JBacd@eRmb`!e;Q#!ltn zvdFo%DZp|; zfpO-L!2s~29~JSQlE)OjsWC1nJ&u?l6C+GZk0a^F%6>?pD=qL$@K7_4X$Vab$kAq?;p_U;Q(qc|QY0PQ{ z%cVTMNrM{qT!!zis`~CwlBEPNc_R33FxHOw(#*HCe99Zb@@VoD)F4n+&^tx1 zP}+iR+r6qPfmi4~JtojEecY+xCnBCpTNiRLTpGq;CF??#!<@>{8tJMk_M(!DK#pm} zWx}5JqUeHtAi&Xu0wAt*K*QQwRYybtNT6?u?>YtU2nFFvq9;~Xv-0bz+LXno7z_*C z30{iIOA7%}P_rK?DyW+hap5VHJl%2oV2kzb5PMKg5J?;0P);Z;2y`u`PR5YLv`T*X zK`^YtI%dqp-J#*Rff9Pe%lHU*G&uE~FEeom$iqxgVr6Kw9+wlq4TmICIo1=XFev}a zBOaMAeWe0FDd7U?NoPbf7gT9Ua{>lKzVp)1i~|Ad&cvE4LySz=i>@flE?m(3n}k-E zL&-`%Mm?)+A>^$YMQ$fboJAQIu2E_Gr&RuL51Ob6;ya{U#Dzr@)Oar2_i@wzjg7U% z_J8&06go=(Z;j81vu92FN@RCDn_p?0K6TDE+x+_z+idf1W4HSPJlXpPC%n4NHVYqa zo7ysW&6f7zvrfV&fjp{ZoB>dd;hP(_;+`wI(XkQ`+!5I|E21NQ>UN$ z^yhcDxwiG&$Nk*+-Z3M${QS>P?^$!zySJ{ZeBt8C%VU>Mu@1WRFPGdg`_QjV`_6~& zAN};I7tX%*<+Fd=*Sl_2!yAR0k681_%d1|x{*fa#EL`_i*P6F3i+3&ij{~QdnuN{xBS~Y2I z*OS+-yW~%kU;o}KOWW5UQn22s*m+*_i(ilIabee`JDhdqf*Tjkng9F7n^ykh+xwmS z)}FZo*WJEo`0}G~y!w|fzyGTr|Dfr(TfVSiYN6%M{aXv`Z$4?=K0E9+aMHg&aLs}% zj#%^kZ&h?(`paMLd-%eI3ooASuYK&T>4n1jmABq{>xLgae*2`?PHeyI2kT-&=jt(hWDfaQ@=^&S@NY_=3Y?Z!cPM?TP!HJnPVRCjE8t zi;E6i|Cc@HAO6zP=N9j<=#&rrliqp%#>c-N+3A$#n^$dk?AM7!J7hbbJ9+r@^i)yUvh2F`#an?v;D(cAAj$j{Ufc>Z+`H>!0IV$J~(5- z`m5WQp4vTo$$xp{hQIZLgMM+=^V3o%yzrxycXS@}*R`u3T)yh>&ED<192foCn(p_k zb&sF)@W>v|U-9a!b>3Z9c&_`w-<*^Fy}o(!!FUw zan~;0Fy!2QX3gp;J2XFg-ch&h_Thi7x&4Fwx3B7SZp<%v;MCJ&<^M!+p1W-CF7V z^VwJa`}nWyvDZ(3uy^XwE6tl5Haz<06L-<$Eymzsa|cIB_{etVBMpSgS2qvuC2*>2BMUOxK4mlnVN59jvb z34i!(m+CJy1wKv>w!@9$lzV+8>d!7CRqvfKN z|L_0(?ZS9#YwPNLuDBt3($|l@;D%@ayi@zcC6}$ee4pT0;rh-gf2{qt z*qPmjRUSKc#qN8Y|L!k#oxE_v>6fhe>a?#OV(fhHCI5*3VCR1}@BGwPPx#Y1nExdY z`DdMZ=1ost=&ZS^e)?(kKgz8;{eZDQ+v(T22RAI+cFteU{r>aMJ@U! zM~CNp?UnC-_q(6lv8{FN*LHnn%8uDFPtBix-Of8^k6YGx%jKu$w*S(#2i|h|Z21WO zM(D|@xqZIAYOg0Iru%OD;cnlYkiG34^Nrz$FI#y2#KDhdk%cq^OevkC|FC4z_eVrqZ z|MlJ@Dlf9WdFS!h#ItX{xaP^r-e^4JiZ^#(`H$aRvi_nS-#q<;KRkTKc?Vv%Zq0iQ z>n?lag^|ydXKB{LJ&0{-TgO_`eJ1?swFg!+$yV*5}{4J$wC{{jI;BefjT> zKID~EXO7%k|FZkQ*Df2LmtQ{Bf9Wr4_Rqh&YW0PWJn_wUpE_`#3IDuf#)ji=Kjy)| zU$qMC%5jV!E1&eT(QrMN6%}%_voGO`buHS;`3g<{G4ki9+!Q7{>Wc1 zJY)4U%NPCkKkt8OUc;`x{lPb~H@Rf;oOcZ9y!^&zxk~5C)~4c#flYw9{KMHH@$Z9pZDE<*Z7-X*x{Q`{H;K8Fk>l>kJ-&9&$4|Yv<@x>nHx1{N zpD(=kh=ka`K61`sxw|Jix6Rx2XZh!!9ka?GzP!74#SM+Wd-6DM@51Tv z{Vx8~_7mDK`QdZBE}C-tjQXC%w_ktp4KFRoygl|;i$w6{mG9 zyE^~PM_*sH=#B5>UOH>WibJL^e0aCM_BVD}{lU&VeZRKyTSI$%{_HP4e(KVnUOaL9 zxO3mx?Y@z>_qwI|lKvfTx$&ikuh{+f@tUf)vlE>ko%V9$f>`5sPaIh@y#B`rykR!| z?zo*(N35AN*mCLj&2f9S-7)#~X6x@G>rVW3_l%R49r4C#yY+Tn`O2R!-?eLC^&U?= zv)47v;}h>@9OCm^ONHB8}>eM z;P&T^`SYsD@nhH2-goo+kKbH4u?F+a7yV;rff0CeHrlA=jPKa`T$24!h_3moC5l z`lj>mO4UEPYU~r|?)k3y#h%I;C*&?&@$516zj^KqYxTR^-?rDibKcqQ(R~Jv+v6{9 z-+atppT7IvD+ljCx^eB}JFI%;@3*}F+pc38dYTvA^20A&v-*PBk8c0of1T%lX?Wni zeSVp{X2ypbrcc^%#HklAzG&+VdvHh_x{}KGjF-~h6iev zzw)-xvG$k`|8ak2Ty)aXKTm6TVA{~$le}rWzLI|Wq{C0Y@T<|A#=mm?OYw&D&$)N+ zzn}HQZ~pD}ZLj<7vd7mfZ+s@Yer)xZ?l^CFwSD!YC+zU8Y;ND@c3L^K_Q=_jPCMf2 zr=Q9_dc$P***g|Jx66|A&v9;Ab?GIS)cxt1p2OPzFfcH1<9^=rKYgxY&)241x%T>T z3$8n@f93fsXVXJ}>l-@c;BTDpkC)%u;TyGo zIDE|CzkJJ$@4j-}^9_~z?6m%_cfL7px2KL@cFxhuPn&u5GdHZcaX8;|_&!gh|Le|k z&p9Oh+^uib-;MYBZbuyU&-)MQPhT{x{TrwK$8*m|Cafx~JpXq~mernp!obCQul~i$ zD~_4D=9%aJm^-**BUQzzxK?(zr6JM z4_eoL=ek`lU$WxR6K`01LFG;lOuuOH0~c;T^P;ot|H#+>p=H@QC%pOP-PRx4@SlGi zc;}DJ-H*NYt2@V5y!+*K3mR_OZt^PYuER&xoZ0{GAqVer#iECPvb45quiYD7N?!B& zPcpNAebm~k^G6(IKa$$;`ki}Uy8KtU&J&+c-2LYAZC^dV<`-8zGwYm{;|{$jHMwKT zEo&3izkmI<#rM2z?fLNHx4*mc>RH!adhpmAF8R}aZDWs~zI^BG)lc5I^8EfuJN$Uo zE=z8II)Ci4#ZUeInv<`5ebU{BkALpPMR!M69$_AG!PC#w-Bj~n?FM_vqvu}L@uPjJ z&GmiD>uRK|X{B7N(d$MrTpvD41K-V4Tgjd|Y7FMRu_%bWjUZg}+m87FSPu6fDYoA;XZ zxmC`jYo2_2$5$R&cFxN~<946fw&YH~^7JFF{!90V?>+MEC*aE3Y3~`e5A3<^f=Q?K zYrPAldHI!ZB>(et>%Pm+tyuNNukKm%Z$qfBxlZH!a-dis~1Vi;l00 z-+krXzqjuF>Ko^E-S}+FHM6!Gf8gLDlmFv851&>zCi})U2aSC3%ikQlt!>_Ql{3zB zmYsk9pHBS3-T4#VIRE{7=3lq)t9L)^y}Ew=o=bXf&aJ#VJt=m=+Phxp8d~%AqR02^ zTz1Y|7iWHc$!iaH=U0BMdD%}+xM+52#$dyakNzZ=_-XBfcRVx2bLw_nbpOle{BXsu zZ+Lc5!=CF8T6Wr@`=4vQ_DJvI-Oj4tad^KU?tf*cQ*r5KJG|65{6uT-cRslGmiw+g zHF;6XkI!Cx#_Pr{GpoP!(%;YDF4yyi>pQD|x6pq0^rIFYvw!myg;Qqi_u!SYh8EX4 z={-;U-Q$lP-8*Tgo|7N-e*T9K&&?PgWRCgLu=~{WFRrZl-fxfn+w(8Ha8B&neQx^6 zfbp9TuH0?#i62~V|7hojj_qE&_P5Xf<+*Ilb<6fYY1ui}dl&BV+HdC^`MKTl&z-)z z^XxbNGSo7E@a>-#HXJhVzWvuteB<+<>v{i{?Gq2}G3!g)KfU|h>vsFr1;6{_z2{HA z>D~js__gdY$1gwcjKUkMe*C2gJ@>z`1}^oZ7JRwB>zOBax#i?>8!p^s;cqV7ar_S`oc8{I12iM*4gNmPJtG8cw=sv4wumAknFYmVC z$s_lAYSMg@ZL6>^d`L*>C%E!L3$SuMFd1b69}P~ zP`1*02WcWm5osDQgeDS@UIe5j0Rqy5fblz5s@o=XoawfP_8A}FhUcw8my3aj zylcirZr>&g+O*EA?yJ395%NY$lf!+3~UG4$8eBB!D8sb=i(`0Z<@V$>pkg#m?j$kQV^ttad zfoLu&WCj9d`rFA1>Wx0-JJ@L1MYet^dK|ykfWIsixV{r(RRI2s3ks2(kiC!Zy+bn~ zzoRTbLmVuLb7;EbI+uLDXL!X`TnPUVnKI4Ev;mgs(dMU_dJW}C0=rthz!D$|gv>eR1fc^Sk z-`N8z&~N#Kd_wSGO!%#J=Ce$r~O0^I3S-s`;}XT*=DxYNVm}Zz_bzh z0-082B#A5K&(*)nCzxI|c~d6a?ZrnX&1|wh?ops;0ZF#~L;HEp!(2mP^pIZq!>0%Y zvM`laT_PLG6B+GghcaNu+onRVMFC7>ln(<=WucknL}9-xo_Xer;T&L%pSJ)&MQ;BO zRxa(1=oaIYs{xK5Hba9mG;hZjM1|&K_jqfKL}eE^mo^lS3WoG= zYiBmp4a9&D)u2^MWW3ahth-!lO0%;mpLVv8Oru@RbwZGB_R^`r~+#gKwF7G|F61lNZuDh?zH;0FIP|IiRa{Zd^p z#ZXL!Gb6}@xb&h_XFSW(NdnA%b(2wACPWFcqU*{R}sN3 z^_kF}smB5xZ|psn#rpPUrF-B*GZa8l-RddZf^&s%ND}y9A>;?91<>I?!U3;$n125!&@G?{X zEY-f2Jab>L>)w@y)gk56&Kc<#b?$yd9&finnzqvOc(bOK{Es5AQQps!H|s$U;Uba4 zc0~5#yRT7oqWahUPy`mdTl|N0LjftqKjJVXwfxij(ej3r8u6R zF~!g_8%j`ljAso!lbZ2u=&oy^0oj%wDH9_o1vt~osD4D(8ZZH@^eJ+SHZ|jib?BPq z>ZBQDyg=@_ ztm~v{f5zYWhTjU`tGBuWSL(JlU2wZTy2>it82&ZL9Q!0X_YLV>!Ctrp$goa;4gTC>>!RH{@ij$ zW_4_cN^>d2)0TAJ&=D}a><+Nl;%z*>(Hkf`k7VpeafD3%_y^6xUlnKEhaW6>V*r)x zeYEa>c5+~(s9shs@U)R~k!!^>`GBQ2M-OTGS2jI93PA?}Q8^1w#qQIu(m^ilx}?q| z7&J~`zYXqh<7ZgkuONq0A{4MM&Cq3fh)#bt8MrPoBHWS-XD$fWAgE*nc}$cSLy*b- zqoomE{&_+W&h-AZ*ZyP8osTjwVpA)=qEsS>3g#3u@}bez5K^VQT@77kWZ+0q6fz+X z%pr-|v9{yrCDWP(9^{O&fge=PC76XP(8M&0#+Y3rKNd+MJ3w0jfO7w3ATf3MJ-@`@ zll|b$$LO8;FtqvE6M|5=QuI4~$HB?|U*_YV(HBd9-%bU(a|bfADE`f_K%AL`G@=zYdgOH> z7GqXSQ>#^Q;UGwN?3wk$e?6_>6kG*1q2_fM$nimb{0Q~R8W|ap>h10Q^}@x)KD9{4 zU&uj6rzgcfY;@;wyJ>n_+Ro11i2;8D?sDSexuR3l=|O%lapdAR(%FTVcW87pk`FOU zju))%u=AyLt#y?S&#ue3UBVEgf{9BKGftDG*i;#~cD;Q9Gvb3m=%*uzIHUiye6 zbXR0XR@OzqA;qq$s%l%))&vnfDK|qLTVg?Si}jNy$NoVotGShxqU;>vg8iY&p!5?} zYmrTCQEE{b>2>OXIMzY&=oa?Aq~NBM;FSW4-sV>vKbInvqdQ4wL`B3D-WyL6YZYy3 zatDOQZwlDBgH@O_fuU}}5fb>K3|5_E$IYD2fi>Q@eDid z<{x@&zg45oT0hv{-rn!J2&tvtzw`g)(nwO>))r?pANrz>@6SW+t2ijKy1Dr@x!AIf zVYqokZ$Ti5sYaA5yqh=Tn|)H9xM~PxR4K)TyW9PBVLWu*GmzQ^S$zO~wT` zNAh8dWFm6A>Qp}ZloVSMr#!&T9QP}vLb%j@$?5?|N}-mkVnVl1KRUL);h4yzRs*X6 zWz?c886kiV1~|lsm9>Srz$$|pSoeRu;QzASLPU5FPg+l?cn>AI%20J1W8PT4T3uQF zxUh;hygMAK*BC)%A&yCf4CkQSN=`avu!-8hqih-o!QHh8OP~j)$QZ2ySLqQe54CT) zZlYZ3{Ncdk@a{ldrgt2GqaQWK>BX^+GtmBVj(d&i&hw5FCD?wYsdw`&nHOgHjL8n# z2VPr~Wh+k~n3)__l!l{q55A5X2G%f*RGsM-8nZH9y^#blL89+9GIlOR9f#17cg*-> zsFQj-W=87WZH%JLN}AqET->ADGVz}AKP|J2_hw{Zgd}cC;yATxMyki6!dT7dNh6BL zOLF(u%i+~ytxr!I+`lxhuC_M6!&gJ?&Jt(Gg2BC*tH5Lu#l{NV&_im}mRO>yv^5%c z$hO}Qb7N^B5o*H}#W3fgkg(7ye5I2RLB_3lU$+L^3NF=0iRSPDHX1@BO26``-HB2N z!MCv6ca=h`sP!DPba7CK_{6QP|0ejI-}L(!n-plAgHC<8Ws0uiR-aaacNgjQo7hcd zjmu`NOBCo5--oMH%YCU^cvw?cgl3_E42!I0LWOgLu_;bJyLyTN=Zpp6R)Bl46i!V- zCNZh--uwTTRhpww5Kp$Z9+I9^1EB}BG%OYYoBa|DpY|0bII4%z_TxQGJFGSn1&#zd zyiOw@Ce~K~1DXx0t@m8iWpYzU-#{=xjswmour`^^2(ML&2(=pI2zDIR@(bta4K)_z z#nNmzbd;7|%m+huYNr{hxG+SN?7P03Gx)LbaoN}94wEmh5?vWs`(Ai|rqZG|-djrL z)lt)L_Ju9wokf|x-${Kzk)VLTp$w&r*i0Qz;eE|y?qJD7Im%0ap=vS?)zz}2ch0sK`&yW!9C#xq#>TP(T7IL*975V7$#-tty!DJXe-+_6Wny#< z-*~4rU}q_NRpYo=p_Oz0z^vm}n_ES5)3~@)w)4Rmo61Ea^>PM_yAtFvd~$#NQA_VH zQum{*EgC%5TrAW%u zlK4p93HZ&zi4FJK>3p1cplO*QIj{T9@{m#Y(M$`h<*`U+ce2XS&$O9juKWEiowi3O z2b)GB?TLM`W(TXv_erdGM=V=ITGr(Zeh23T_4a6mbZbi{XFe6Ry>I zze}}*$nfZ29sKB$RrQXHf0M^_Df=L%v^Xp5{=tj@?7Bt?e_w)N*EZRj0VoJVw`m8$ zx<`Ime{*&tz%gH?>vZv+-M5f$#~e9}1s?m-QU)3ilfDv(MyREnjbLXYauKnJTFa#K z(){8Uc(!xo@Qsmb$#?{0|5ei!rMZYBLR=TKIka$}-@Wb-2vVhq>D0IEgKI-}ndE-d z6^ewd;*nC-jbr_Bwsn1#F3BA_QB)Ag@7f|3_AGW?i82<>*RleBU;G-YXvwk(x%&~> zfFkd=Tj)$!lF43|aT({m=02Ercdt~Vvj+j71(}pBglehc``x%7a~+ZR;I^M+GvwSR zQZ@O^j7g)k7d5bTcAh?PevdKFyzeb|%Uou_%WGAZNgvd8NJbKZ(6)?&+2G7RmvGo& zb{Su!=#vJ{I~$H0HehhB&nF)o@zL>Sb>_8t)&Aqdx~p57`%cOIvgDw7m)ZON;yJaG z&6@vEd%}Ld9j4%JmOg~&kPz|o>Q^S$>Sqpk>PRNHIovYYNbGYdo>lU{$tI=q>YXvz zwjUXIl!8iybNKP&N~_-G*D*OwgJ^Do(@1D2&7nQsMvdOKZ;!Yq%`w@722YYJIsa~R z=`75z>l4svlVqAObgYqpvsbR|3CsFX;56fJFVEGml__ru}qzO|vJt(M?mr z2gXo`_+ekW8?Qd9vsMI7?<~eXSVdjYv9>8WV-}NhvU8nVvH^eE0F;74dPe$O`{M=N z!5GJ1(H?|vLw(Ui_IyE(a-V>`km;22#c;m4qU!CtHM)Ttmg{i!S|y^ltFSnB8GU{F_+kuHJ9^~YG$te@CICLqw$XD z9xh)EnKH36s8BNl>lWJ^)Ba@nrR^d;6zXO70X2}2lpgi_O>W^*sNWHI%%an4NXb#x zDEVH!L+&wDt*+YDv?fnuN`fXIqLi37mcf$+!5SO^XFA6QgAC1>$~-} zV^B(_qX(noM`%IVSn_srMV6wIj@ecf!39tKj{^&c@3xRc)d-{n{AsO`u%=4!VUdv%o5ZIx9`(0ozAd>_=9KC#{QjJQ(BaBz}yHU|+I>?`i=oQ*P(1cfnpl&RDH!NC#_05ejo;8st~ z_d>kDO+3V)DSOKqob@ozotLaUy&GqutXW~0<`qVWWo#F{O1(&gM>o7_Aq6uhj+a>E zubwk4a@0GN*AVB^W|Y)#Af9prg&CCQ=D0*|W%YYeCHwuVVibLn?mt9YYnl2AYI=Xr zUnHy*GVxCy)lb$vG(L0{F?zo^&G1<3RN>+xXQx<4{k110@9PQTJNEpTkI~Oe z&tG~=#f!)+@~JNAG32tas*dUr;D9EZ(B+P zpFBdCt};-yX(T#_H-KZPEODh=TIudEfQ8fB-cp{E5DKrLUPfeYYb%0m9^zif;T!|>Je$!E~4Y1 z1{_xxw=uBPT?flZ2^fil-|Q?At>@$66?KwI@yj%Vts6aGX#-VV!%{Y0fkd!mGh}Ds zO%3lZ%1s&WwSl7POSm!3(LWm1f`%qGq(A=vqgJ`Xnb+Q8Z*s3-{8wk*#(ZyD;KJ-a zlXTm-&YwGwkh<}^CTLStDt_xQE?h4Xt%)$DhA@Y2gqPjX`e;>H+J*oLdHTVqrVyaz z$q_0T6emt}8I2I~KY}3He=TQ^d$XY!_?bh%HxPLV`JonBS3O^1NhCjM|hWTmd-nmOw ze@(N{$p&ua^St2Ti4Z#Qx4{e5kv4j$Fb0zK68J{;$*4r1;k@jjiJBT?XnOXc>ep46 zCHTg#?XGyHog}G2xrG=oxuJYS?g%A(q6ej-yeO3ZigV)JmS$t02ffMwVLk+_GIXja zXUR0w{jz4yfC|&)TOt%Hprc{hc5Z@GZlN*kC4zPD&fgUAV$@jE@U?^T6%VEi>!Ii} zE&jO3H;OvshzsX5gu5Gh2*KTCuYg4j*bjr-`1CIhf3O}?awGbASNO~U=#6-}zNhH? z8=ca8x@YwyMw@pNKZs;wak+mcGZSy*iE3%UT>O|Bti#9ZHGL7qG2HI+mCZ~SoNje> zw9|=ED|DJI8g-mob^l!IdS6;RwDk>p1u^qeKBYpfi&<@C#LJNWGNXd?q%=eM)hT}% z4S>}pMbQM%pMr;zH#Fc*Gz+|XSf&SKIUjf(BO4(~C*qqX4<2k8K72UFrO_34+q9JR zrVv`naqy-HP)KCo6H`)U%Yz|4Q9DEhnlt?Q>qbah0yk3 zV_j3Y68J|0(TE_L1Rygz?9q#iE$msFe!Fb9G12OFQDm{Y7mrQM10LlZ+#j|MKj~y~ zSA1e+)dEruOhLSMiUlaHtQA}|oT_JTc-Dd!twv1!6v-%VEi%PWC5B0L<5IUIKA4QD zb-7>oyH>uGgbS{TQpmO8F zWIgpdsra{EyCkK3+mMj+wnMbcv}Doe@P8}FbOngGAueZ`O9fXDpX+7AIc$fhd zJTM}@KG4&1RQvy^nGAyc$MvHA@(mYr0((ziAI#E(7vRc#Xtihr6~Sq4;k)A!I|WBbHh@I*KqUhv4w?B;kTRaJKg+wX9yY)ukgF7{Z!)fMr=j#mncHA+Mk-L|<4YlN6=d*$ z!1cM*VWpNyLYw=aL7{>KWuWSk=Bw5D;Y#&P2)w?6!0UnHEoA@I2God$XCnTG*Av>^ zynGC9`Jd94nuRq%W+3L8Zu+lwz!gq~w<1UxEx(^C?%$TT-d!DaY-RilXi6=5Uj(p7 z+1tgVCuq4J0tEB^P?TEX87Z*#jEfas7;;P&L+f99i^q3SX3UvNQ^6KIv=oK-Fhv?8 zR>rSGIkgwcj0NdPHTZe1j<6+mnF{Q_KgPGZBJF1vRIo+(>i3LE8&fL=Z3 zVY>=AZ<2Z^#3d~TcDqgq%=yCfDreJhRp^Qq8y7b&xFySY`glim60_CgIfK z%+EaZy;~(Q4S?0ZvmB-1a(OXD79*GRnN-6ueFG}Dn~135&e%@f?uCQ?_e%)d1zowO zvqlfsWz7|B5B~eyLI5NF<**3kIG86({cZUTOl&L1=b4rInod14D`6zBS}wk;6_ ztCKO3l0$hN;PN?>wT?+gt>ue%#5*08Ynr!vJr;hy3C(}KnIq%U{K>8>5a56{0vK?^ z5$yMzDw8)^i2{~<5TjAstdo_Ow#9M14?NX;PDnD|LOdk84Y-r+?I#bcxu(K*MD{@q zjQ#TU?QBt3$qoo9%x{!nfl?0r64Y5@n-PrG+S=|01_?caLpd^i!otE*&zBe>)|5<= z9j8OP+J1JfQ}AyrvaYYQ9nMdWtbKew`yF8Y1R}+4zF+`K)h*Lb?OL3(XRG9Tmu%q< z$GFGo;g-WBz+~;eizaUr&7)e0u)_pJY=APW*8bw^mBTxPtS5(Cj-Sa%Knqj{L3l&= z#ZJo>fsiAow7!Mvxv?aSeP&a4KVx{u?j zE6PgiasriCAxWr#t#~{3t~=E~k72_fyX35aCDJ_PvDM~Q(Z4m#4q{QDsMQBXm*xEI zWnlZ=M6_KU?9}n@zFhNpQw438ZDyB0eov0|F_PMb(=v4pn)kBfdqR|EGpHxk}RphU9lImh=+=-VR;0&@JO1WI#{_LwI7`qMlb52|H1MZdso=c5LH zdx)=N~J675-OK}s#-H+f>dU4S$0`fg8WK5Rq{ zxFkkojL|RE*i5IfMG+M|uuJ^RI?Gt~G+8@vOL@oPR+ssBQaEcLq6_DeX`L-cv7_?l zt8}%Nw;Yb2g)Qz|{8JvZcFlDXUw7kcxo!JopiQFd&ir;8YW%s!t_P+4ZfFEkHh(+z zHMB_v1OAqxnIa!lgKhLV>3To>?#kZ%-lT;{`U(E?*7;+-fow4qk&wyLNxVde|0yvG z4FueDd)PO0h4`~&lqOG`3;d_F2n3PKQ9JMoWM=Z*rRUa!TRa^hfPAsrNT^NBG)r&( zo)VF}z})rdd(Qg**o&0X2lQ+T_aHBU1+iC>6nKiTmuSmr8!_ha@hqr`W-bCSj()ug1(#8&jujW~6JNDMb zmFkB>VSj%bX-t#4ixsq?N=D zz+^Wlm2-$9xaNLgazpbs$~cY!1@?q{k~qV~8@io)7q_a+(6z3;mQMcN{+GeK$m_aH zXSc63b4#Vv59m%n&6A*pl!uA!`{(?L9vfqJ9pmh@mO_~Q$1lQ6s0=+5-C(BsJ#yfh zo;nae=3IE0%^X=Z=h@CWuj|~?9I!*<*(cFC)!&3=-(!SW?rYhDD?4`cA~_sNDvAp6jn1&! z6(uMGE9BX1uTlc(BJYO8i5-9lfrDW=1wXPqe~`n2#y>#nh&7ZXgWue4`uR)zQI3vc z!`@6c^gbV7IVs4V2cniBBAe|7Gs;4fzg1~>t6j)9L{@o^-tPgJS<6_(qlvZg%^raQ z>Amsj!|Bg1wAqg8XfRv=z?+rQ)T6inD#5-LmjIh)Rt23ghP!AYk3XWY>BCZ@7jnaCSXPgt7YJ zTOP8~_K0}}A-bF_Q>RRD{N<5Xp^A%|dcwn%L^={P%a8j#USCwPK>Hopup&WaG#mO0 z;$+lrJZOP}pluI$zHZok^K6dD0r@7!k^LyG@$CNkDm>i6`DDS%hUlzthi6%Rw1xk@mwoD?+8C+>ad4dWtnr+Bs5pKRcPCDh=_`Z zSfWHtGTjK*6vl$nVJo{#lWX@}dS%sjibCPR7Q%D}*FUcc>?`6XXdt$49kZQBOUL$B zhVA^Cl}P>oBd#gsH;0Ow3N9n>TVE&$zv`HidZGMYuJE}t;j*;8x37;8 z2C*W$Fz>Sa2rgENvxfJ@D{l$T(ydY)a0wao>^r}^fj4^RDo~uO4_f5=;hh$4BIp{< zgY#PmJ~0GE&lWDvpvTmO2DL z&2S0m*TPSB3ND(#1dJU;wXyohAmFwbx^?`H0db1VC7B@}eh?1Z7Z5vd0E|)|qRy(^ zgV&(a%)vb8z6pr=pA5=ng9% z5Q-NmCI7ujvHbJrI;$LX?n}Oc-}wJT3o}$3RfR`$+4d2g z5g&$7GTEDmmnNq9ov83DwE&%oYOC2)C>fy}BW%fs%@j@*v1ac-e30k;w;fAD0>Yy&$bJJ{;AxL0k`DUw=n$@x;fLLfM@Xl zM&3F|#c)j$!25GC1orVF7Q%0ZK8gj+7*+Kv$FcL4;Fq1GCz)-7CCwu@v+`e<WvAF?9HvWZ#Ca$qVjiK=V=d0^bl6u1O>i)o;t>&nZk>)@9<~DW>(PDm zl~)~&=~ncRgrL^rt9L1%n3nI2)7+2L6#hw!+(Ry5q+@6$MH9OvA2aNH`i#4m zn!be(_N~649=`y<@?YNmIR3>=UunY|*|`iyS0*_(vZFI>Wr;(l9tSxasLGP1XYlvJm!rXd`{fryYxUF?nHXLS0&P`DsSVN1w;j-J-ue4;?vs#x00pWmZ`|`TTEy#m`|Bf{%Bf zxY<*O*AB|Boe^@JQZB=2QTx|myY?u`i{Pj69C2E5sK3M%0+ zWD61!axRR69!fdSP+oTXNfhU!1t~%FpS&M$W!#s!S7ZQ8@G(-1t^4*T^~XsA_vLUt zE|h=jGN4m&lm9u(2_FqmXIn|3tiv-bNiW>^{+yLh9v~S-pX`S6{#$FYDnaZ#olS&Poy4_1lz0ojD0~pII78T}_y3T&V5+p=PorgXg?pq(UE|oj^DCnr^S0GOCUSBrqXOpZ z0<^?*R0-Uj46Dlsr^=>+Txx(oR5ouUqF{ae-$W(Aj|}L2H@B*k4O1hfJ>@@nMQB|= zY4hKp%R&_5m}>8XUt??3Zg&Xpa*@ts9qHyHV99`P++@cVL-iUp%tr}XM8MSL-jAYSTHTmwIWd1KuMIPs z{(KBg4*Nh@TQ9xE`vI&_Z?q2*0l+R5mlGF*sV0|?UUn#I5fJ$Ixc zWZ=-n`f_4%#)79|?7=V2VMogsXWgY`7e#K>?-?mWFJ#=u+K2q#DSK{h-Y@J{=D`xm zHA zp|{q~w#^Y{$?@%Wx3AK3XH-@sS_&&T$CW56BGDWLUEndrp zn_pk@7X;w&v#qmw^&LamQeEgT!LZJsoTGiTMYY;~*48r>)!sz(8z=SW{Xf24XF5nP zbPN;*oeH{L!p7u{Df6(&l z|3d&-tHz7&Ht}d^8g@ep`X1!AF%qur_1407 z*4KUiRa3L&`1*U0@{}9o3CW1y%>MVZnME>{HLKR=J07Foh1)VfGYg`~J-wMM-{$GJ zf7L2q8>Ov$1h*gd6DQ*Fy9>tS9iKCa#Mw781p;Jl7%`Gxg)%_DthmmUwQo<@*qAAP zOkbv`l+AplMQw8n>>#kFc{}!zl@)#<7c|ra;57phl2p=Yp@kc^0D_wNQEqCW8FYHM zwG?`QAhVqi z5e6v%W?-F4@?ddLq^uZb6Ep_pDlDPL>X&z?!Vm#A=snO*&OOZ|g+Nq%I!my6lVhu@ z13MwrMx?!0!eV=Vq*`-31^cRKL|rFN_-nI6Kz~$cYq>oIo7C-3)p-&UHWWFmH-$e0 z`lpFmtqdT4nb(R{769wl3AQ?Dj^l4i2<6I`z5V;h{R?=SVmg6-Ep$=@H}kuL!+@n* z(J9o~3ol{4^M-9n`C7Ld%z|NU#Mq>Rqk)#bg3TVJe`4Z#yv(k-6 zo1KP>IwY>Ks`8oU#X_5Qz)ya%9brG&MNtO|Pc}#0O2b36k<>rmCd-Hm32g&><7Y-j z24CjTvc<<|s?QbSN3twpI0o)Bj4#^F{oA)~rw`dTnFFp(IHVov-FIi%dyB%mSN4c_ z{qyz{Wdv&PMQ3>-LJiLd8Rv82=~typS-#g}W60@W(29i>3gR1eko2+UqQmk} zjGrDCu&E-QS$+$hd(IDL+%cTjp?li$Q?3qpuh>x#Mwa%HgOkBe4dvXfLUSv{k zDpI#gf`;g(!os?9DHcA0N3_UNt4dxCaM%5g)yg(|`=5_K0 z#cho2JH^oRQ}yt6Tb}pj#Gv`a)6@a}wWpSrE;9lZ2K$)nQwSz#E3{0gy<28&Wrnz? z(QD)$f4=?^aH5!*aH~i9FJ(I__YV!JJbv;ZnD-%0Q~7T4?}-~zw#*A=;kh?hKe0>r z*JWpl#3@M(AD;A?aSuNZL*;+C$um)>1+K-0SX#Je_MLW9OjV{5W;=gQUDn+A)bR8G z^o1ug9r}65nyJ>_qt3F9Pc`C%Ea{@NKdsd~(k)+Z+c2QOQhuVwAza5?nisRtC{53+ zFK6-b5qVX|=swp4;~(In=g87 zZ@QD?mfmV)Mh^XCN6HDpgDAwJ{q8gQM5qln&siti!If&?jYEMxCO{-#;p%M4G15Bb z)BHSw3m>-!PWgwve40Tb)~mU2OYGTZ%9L&#Yil#@C>? zNf|c=^Htjb346yY^8=h;55c}Do?+{hq~6LT1*r`*H>lFI)%#3iPIZ`KB60Wshx&b5M3xe`7hx~TkK z@^oGfCp))HbZGnCM`u)?kzO?43&UCNYJ6Cw{~5t(EpfqvC0++eWP;*-A*K!6_DH4_ zO^2S>ru7D*pq40OxaUm3a<(lvFU12*>F*Vh>%&v)=2DyDs;u&PzFW3$xdq@@fif~O zOP~LAObBIQz>)9?ZxEiNJoD~T$#_yHcIFvvUE%UqTbHYMKYmX}=f=xL8zbh&$)}mO zmPkQFbi8PG{q;rbbM6Dge(I2VX79LxF0j=QY509k%)Ty@r#{Pkh%fvHBC41ZmEd$z z^JNN=!t8QjO^Ys_lHbZC29XerXDp7+OpAbYk~M@mbmF62N0ogAaizOqG-Q(PUlUR* z9@WV1eFqo2$ub9!3zc=td`9G69lnPEs)G9GO!A?q!o_!|)_SiAJ zN~=jY9RNFZE>q@_0LrQJHI9U;C56;<gTO$e^RXaE+kiTD)KRa z3QI;c4NmhVv9Vy)vy<}n&I_eXo88bRfGP-aVpj>&LtFE^@;fO`G|4LeXg&a2UkI`W zNabYNR2WYZN1&omDJwM#C*31A*W=Wp2L=C=meE~X*24T>I^_NMG9zMI4cTRww};Nl z%ezRvlup0@Ct2f zy8l!ax?W0jE_H71rud@H{(m)I^W~_`ap7Vk>;Ks{3isQVAbgIh7kz3xA4F8=6)G#Xo~NoT!sT5r9vw{j6V&3-9E-stWbP%<&G_VHd1*IRnw61|k1 zo90#A{P8D8t!92`7$WkL5GFZUhv4TQCIzNKhM#VuhNmA&F$VlmN~3vE2Aed(;}^yw zqB)|NAq23ptuDmT=a39H}E+?(e*hoDs3XNX{fy)W{FHpM?K$5b2 zs?G^D@BqZ9i6J9cuI}LWbkrpkJVhB`TK61*rd@IT2y4HZKN>1`Hq>&pw6y8btm=W2 zaEw_5z?A}QdxpO}RH~ySA+(Ol*KcgvlPfp;w_aGfU&Unx3E>ql;b#UsSO8jh79cXU zA0lJe;wAthg#FyK&qXlNMzW-BR6kT=A7-Eh+!n?@^n<{*Xecgt({r0_hCSX@e*9T)Kmv4FRmo7En?M}7m<#J%_Hdk@r7N#L zt**A-S{18N^blUQQQgn3)+mVz>waD2)iGKyL9&}zc2dCR*r7kpeIfZm!vJX9ruTn$v2X5@nT{t(>J6#ZM@~82?-rN#jw*J z9*Q!Z#`YAb|HMC#mhRiIG|rlj?d zo!8!3_!sNX_;BXDaoBn{#7Ij;Mug*Pvh~&TN&-n^%<~;@bCL_HN&Kxw40u;jqL68G zm@q5Y@BMZ3smu815#Fma~Xro(T}JFyS=j*GdD)5t%ee*Ynb3?xj0e9I&vtw@QE;)O=kOXV=jHKjuVTitM8PtG<S(UY_ zke>lku02XHjnsK{!JRVmq zvv9H|^+?~wsno~4n5St#BKPFa9yWTH`XJ!ED(aPQlJ)t3%S{|#670eU^Y=h+#M zT1p+{C`Vq9XGGU|=xX;)^Qlum%Vbsk-M<$PxPVh2T7+h!27jD8(Q?qE8R+=br(&W? zE7W}&Q{pg*LbuLmn0rgLJBlOw(qVZX`*j_4kHiDMr-w->k4jY!JWZ=wx~Ls2u#04* z%V0ioc>K_ZV>8jmD4T{RbHZ`Xcyl|VKxXq!e>Gztk66o+uhdM(Dj9FLbNYs{;8XPH zkGp4_pn<)LU#s~l;JmxYot#Rk_T^P$)C){a0Q@2zgx zRSIBGONFpB8$awOs%)zKih4G~XbnyjeywB!YnrMEOHyA|s0lqP(61F;yNBc%e%<43 zNupWQBkS%zpX4z4A*w*9-(`E9CE?jlPHQ!%xb@z9(CSFUUN|~4o%2VtZv+d|=9IfJ z_q)Vyp4s!8O+|apr<+UMAJS#Alc!qo*zO(A>FW3Q%Zi`TTGmz1BV)) zuDG8)tQT1F89R(?sWoCs=^o+Q^1U^x&*p2Wgj9$YDY-bmXhg*KNpF2l=vQhc;Gj4Aw zPFu<-gk!(HJ%jCp<=K;vNBTo>;t+)m`jV-4L!ef^YIYrD1XWRjBqmyPB!M?A0+ChL_^cI%r%?vuQovpF(X#! z!$E!wvCJb#vU3RKdX1jg;cKSOFTEr}kK6>k=IiZhh_@`Cw=Ngc>S%pvcx9(oC71Gf z$3A2~d`$E9w(@wU|-@BPG&G{5+E z>qFN+u=oDo`qfYT(0j{&eDTZo-@9|}@>{N2U;o-*mQHBZ(5X=D5AKmX3Jdh^#k{zd-}b3gI#e)VJL z|MGwR4z`gc;*}a^8J7P;otr4){k9#^wVGbz`yuLK)jo^ z&%g527oBqp?|SR%```WIr+(^*_rGcE{Fi*k*IS>d zfBKal%>BYge(vLI;V0kWeBsOf_UX6JJohV?f94PV^VU@JML+xQSN!xRzWy(M;%A#Gh|7r2Pzw$@_;M&i<HJ z+ZU#OyZ+qY`S6_=z3*i&y72b)7n}e67uNp$U(bHmbDw+D%EE5|LVd?~dOi-}|9wKKo~XR{h!!egEJ1RPc4b_Q`MmkG7lpN8k3^GaGOCrI+QK3y=KS z-iO*BSeg0QjmA5_@xOlI@#~-ZTmNk3&%eJD{tx5-@2B7OeXSq8_MXON9`%)19f$ zKY8H?F6{pNKl$@dPksG+zxQ1){`D_@?j7&^$VD8?$Nhis|9$c^|MpeR`@ZDgeZ{~0 z%>Vew);pg5(_eh{!s~wV{I>)@_>n*Q@Yuip#c%(AcE9j}uUY=!UwqRCpZdvH{(Sd6 zUxH`ae$$73y!fv6Yi_>!Ee&AAKl-7u54?H#y`Opeo1gnvr@#2<+OIwLx$pSL>tFh# zFZ<@7{|~=<>cipgS32ML;s5LRU;M};SKt4V_M6}FCvR(f_zNHV!H<9Emw!#`Q~&3q z|8eTYzvuqqi~kX-Mc%yfn_u?c3ApsX_5H72od1faR`Q>-a^L?QpZfE6|BZkBnU}8r z(;xl)&%N`je)6-w^X+rr^R2Bf{qdboy<+~qf7=KD;POv=wI{yi{S8#? zz3*q+uc-V;;{)H_e#Of__2aib_(O9WcmDJz@!tLX4{x6Ql8?Rr%`0#7KKX}V^`Yx; zeWbOI(F5N615o2!N(%^&h_zw8y?`DKqx{q*0Q^Z)CM%J}+sewttZ!MBfX z|EIV9{wr4B_OYeuh5WnkJoChz6IVX*)gS%!nIEX${HgzP={MiA{%fD@eDvIZ`mWrU zfAH7d^Ityr_u=OFrBB+Q`nMn4?)>$M_gsDSC9}W#TYotIO&|OC$A9T{Q2EVY?d5<+ zc@7wbAAi&T)_&i)@2kIT?NzUO)i1yFH@E+fZ{PpIfBVbN{>bvlZyft?pZ(_l5MF8) zfA9k0`)h&q_`V+|<^`$k ze;9lHwkscetFi=7zTQN?#7j$ zdF}_)n7-Gu7R-E+&0zxuN8cr5rz=q<{*1fVgS-HO##Sjm(UV&BtGPn0?!!)NPb_qiE?Y zb&{>3nM!IS>5?)Fa;haiu(mO25vH}^ZV|RvPM)!`>pBmu%$e~P;NpZz^c|C21(<@6 zvkf)PikxkC%Z2gLC@`tf4(GzdoQs%ZN-{Q-Pd+O-5>+0KXJHApoQ_KgWNG{iXneF~ z&WvBXwyG?rAEK;SQ0FQZ z##)pelfZRD%z;vsT$sVyvucy-rw{hJv4h)Ah-V)_gkeWX9-$ECVw7zhxJ6D8rqVwfu-DxV zE-1`~w{AJGAT>;HgM;ue7h`Cx1^#WQs@tZs+j4@y--gj_w_?abuuf_0g?az?Oo?$deQeBIrxaP39Q^{Ez>3Go zdZ%qnR_h@`wJ)y7#aDMJne*y}GMBu_iMe>m;43Z*G-&CnW0sYyJLb0P02bmwZ#|Y( zwtP~CJ1yV3#@b01f>l+9@^2QmMq_!q9$Ms?z2?uSvA2#g|vK0Ky1t)Q)7L0 zH|8I0HsR)_^mUjJ&jCULbHMFcH-Plv1ohS)zuz0%n~46&cX-HMNJ(pY+QO374ZkCq z|(NmoFwx!OO9ZHEoR}ciI_;vw)RoPbQCnT1KgH37{x# z06j@f$9l61xYk^5+Onx>8_!DAcC@UxMiB-X8+!tdNsBWALJumJX+@8_?fUHoZ+55K z*@Sa;DexOGdZ5OePOxL+D|m_74RH-GUO;l3zFl^316(NU2N%k^6t-991~b}r>zIbN zPt!{6}zX?pxZ^u-m?qwImj$n z3fm121_Gr^2o_fXB;Zyo>sfmAEb|E4esJ5`z_Tt042XP!gA?0_m^9YWnSeXnzSZ`C z@D59sFfY`T<+qY~+rzd~a~acNG2%slfmGZV^ji-jR;20G>kXH4!4q7}%=rx8rW4dT z*CwnH@;z{bOUda}m-X1#)3^^RR&rvc0RJjl<10@=M~=nkaCN+pbxUwzna+5HLTTL# z)YQ&4V9{E@5^cGSouXz9yObw%!@bj^#WKCLHjtgeOSc?r6vxM?QpUt1f&(HLGz(&Q zTf(=!({DDsH6=xG+MaOWoRs(Va1R{;{Xcd3^qI3M`v2@m{&yt*b^&L>?d&${pW8%U_1ZG!kuWlFBBNR`-Bi^{tV@$v9*6m6s=(_qL=gdw%4S@9U(uJFt{`*Np z{P1SoVJgeA_KTSyg^ljU{Q-))?gLfSx*v3|dnCrV1}Kyp><4o@@LvzN*>MBA8+i8! zavXr+VIBY#aHkJuE(gGclH!LCFciKnBP_>k`(Z}nhF>EA<3qR)DRpx7o-yS80IlP7 z8pm$A3FsxLb=&~2A=wOk#_$2t3O;(GtGm|l zLSUzk-NE!*0N`5pIO3ED^kcWvozAhlG7;wEcw-{mSG=&@aCVLzm2w;jX;RZ4H>jI# z-Rm}w8`SQdj@vqRbKP)S8(m;$jvL&8+i;E}ZBj$)u46dgq2CQ^$8VQ-8=IZ*s^6#| zKVpz!Kb}zETAV+Al3K4-^V&{hzP06s9dF|}a`u^nIBxVf5QoPP(-6?4DL!@=DQp<; z7*Z}8@M9;(a$CpnI%24g+g#Ux-`wznont68ynIYS@Az%6cFeZPZFh(4QI6Y9NtRp3 z@p?aQGmTbA#|>sQ0!o35536qL!NUxHSFs#$#O(ck&yKgu^VgKod z)yM=iEKinBq5J6l_y0P7`h42|Yiep_|9K?-Ul#dHbv*RdAjjR=w%2W1Td@3NO{d+4 z>pHBUVS9j1C>_tGUzX+7f!+4tvJNV80FkPwSc1NWE)zNR4pXLzNTYYbZs7HOSw$)csj55a=Vv<^Z zM?CXga*jX;=sJ^R7WqP>M?Fs-x)?aMyPJ9APV8M@)1v)OjHWp6aq91GX7L~T#PlzP zbZs0wd?F8+YOPrI4*Z+Aetn`|-`jKBer>bb^cpY$ww_FHqXmphwdpjll3TA>nzz6P ztFlmikF{HcC9bkt$Wg$Ay^(j%eeSG3vXVY}zGwA&@As_e?)_e=ln0mSa|jhL(Ypi_ zh5cF(`S3%07+70Y-6RENpP^8(LGP=5Z$hPbo(S>gcNzzr72*}DhlHbRKK%0FvN3y5 zg*RNkN&!d?G}kq+vok~~zuOwRTrfnzz}+x5Y437CaCY2biUqFQ8C(uS(KwAN`mXjb zjE;=SL-erR#KRpCrv+G(P(et}nenlZqZ>bPT25obfr^3ebAc0^-se3rYn(i+ht`MBC0zNG-@?DU9(r!Zuuc>pSR$TkTYdWUV!v0I;JJs-pN819cE@ zMWH(=Ko@#KJ0ZL_$0)#L4icrvhX7VUslPN%=?!kCR`1@zd+AM@9!Z7z%YUYhNd7Z< zc5>=`O8zrBivROq zo3QK)Zu6owebKTLkIOxtce6`pKL%3DZ*64P=>2XY<5s_@d!&yFtwf-Fp^J#s--&yrHF&2W)772y>{``f_)}wlF{#I&?Gr zC8k3)GlW=msCI^xy&j;k{sP(|nz?%^?oiDSC+fX7IERu44|fcP5*i<*sRzT)4mqR4 zF}_1JHjJ$LAgg@10_}U-UqeZ~hw5Yq(fE+)`u@t#kAD9QCt*1V#(p?N_d`Sbf|TzM zH9Lcp07Er0NK-IWBZJflLp3s3=Ww8A?iJ$>)?N&U{|BfzhJ@vQ>Oqb@QX5hQbAYxE zrIG5rllu>o#%Ji2y$`CP;rkv+hjf6U45w^5ScAb~wm7W%YUl=s)N>u6z2OvMLpM0I z=4^QK^8u>Zjy?9h@7ld#`K&`~X^@_9s73}W6NlWG{WXrmH8MagIaDJ9beF@;>`1|R zR0__a*JMHvU7x;((GeQha->@w=~hSoj!yqeva{MD>3k1j|9>i#|K;rI(+yv}YxdeTczW&H++}>U#6;}afm(*9B_REpYhjuw zAm;*iz1!e)YZyJBlR{yJD6c^%)a!6`cxQbb6MSJ_u1L!k60n?Lj%q=pnEXqHoQGSx z8#us4Hc9L4Yn1NQRMIsU+MrhEDOp|MVs*!Sy$Y3r`rIidacaZSe=Vh#>VjW zTDrVAe{=hN z+n-cT+QJi0%-?td6EpZ5RlfnmMz^_!_sgVZ@=5#@mlVqJ zv{@|Sg^vxd)snn*Zs1U;Tx=}_ilt(mo;rW-m2gbW*6@wl7LIFer$d@*tl4n4+(yN^ z=5GMORjJJ1czi)>I$Ms{pll~p1_pBL#%pdYJawaJJwCfUd#z~AEiNuB7Okgd7vUM! zELvCQ9=r7fk9Y3c?9%f5<)yjV#miS=?yt^WpN*e0D(0EY2-0+*-Um2Wa3j zZT_V(u@J!3s92kwPCKlW%e8v2iCPDz)NM6NZoR7x@||V_wn?~Ew9qs`<>?72anTrS zDu!z-$2I^^)9Kbdzi45OQlFQe+~OJcnn8K1XV47&fjpf|r_bBq=W_ar$7ytzt zE;R5ER$et_;-+*nv_ta|QdVH#j=|&|)f%d~QqbYkSgTmpqNf59Rqg@QD_`}>(`Bk* zRPGH4E#|_l<(s#b^-|I^xoU7Lt+=f%FYsGXwX<3V4AAa&$}5O{$sA;rFB7$G*W68K z%Y$u-DY2r=sGO4ybCr2~c~V_C=%mQ*t^s2fhM3?K8=Jpz`P!{3bJo($xy$p9&tJA4 zpT9P@Waa0c=I1wnsF8PbR|*Ps%1!Oo1ydr)!#Zm@a&-U0y;gHWKJR=R@fmgRIJFME zCqBz^IE$x|d6X*veF;d%C2UE_h}3J$CCVsLCvAg+*YGiSZpg{@N*!Xza!3UMGC?jJ zX$VCq$Wtms#VF^WT=M2I%Mpr$b;VoR^(YZc6_YNh(Uwb?EOLUcnlcu_tp%mb?^p&J z*b1CM4PX(~fbm@#AZYLm!*~23Q=>ku(Q9Y|KVv_nJp^$;6EX^F!T4^BXg~up3TnXk zP7Op;e{*(m_WIoN+~ShPwI@^(C6+w^dc_r=z446NDJrE}$oWu#agi&KB*l0hXj^h= zN`Oo>=tkgo+fnx3Uc_g#5u<)EODofNPPgM@lGhpwGJI>z@94W{-N)@Ef`S~$8KQpl zBaOlJlL!&(M+Q2;=~j>!HfEvMajV@xeu=@C6Knvs1A@+Y*x3Z?5Gcm8r^Yn0iiU=h zj6$1+yW!M!OvZtZa;$T#DxEjF8x=gx&340$gQKq!jdRs2b*1zs(61oX=(lmy{Rnhj zsM7mFgi@2L!!k{9R=3pa&A_jBc`syl3Z>e*Ol2g~dA`bj7<0jWd7t_iApbcvd3Gux z|2ci`+(`a&B>yCilVz*xiz#t?Ect@)P19)sY7NqFW7MfVHr52R0ffsHdOJo*4@YP1 z;XeQ68YP^qV)D$xMFj)S|G6{glIQ>2DJVWV|3`TK$6Nkwf5QvBaea6>0rAsKp<4qN zb_tM2H>4nC2lvS+8OafsvbC<4fSLpx(y4ZwjWB-)1(U)_cbtY-%ZL0MH(4zV^Y*5< z5&(91-SjvY7+jH6a+#KM~lci0LzbYZQ2*xfzu}Wno7&sbZ#TF((vk1G{@>u zpjD9}c44=v?7GCDdrc1rc;IV8CAC6o9n)qbi;GgGma*43)jV7_z;u%U5m`ja4+|TP z`6g;3b3<0WR+R%a<~d7yG=8e6*E7KT3OPRJ230HZ%dF0DjS{ArXU@oeya2kcF+!={Gcm$8b*OK3M zTe?cY+O_u9_Yz}UFItaQTK-zYY26MB6eUd?9g)Zv{y5f#k7Coh>&(RZDJdXelW|tx z6DS|ipqVqE>X^F)SxRCk$!>xKfa0vRgL{M0pnI^sn|>G0=9;VI`LI#QnwkYN#%^23 zjQ|g@xzNg^Om6DbAvr(^kuCm}{81cq&u# z#NKPmWwivT&9EMQ)Y>UE+}1{iGRvPCi)5^r_jzKHG+(W{hsI?rn4yEFZ~)_@Lfw{|;n?S%TYap^XoiJr;t&>n|rmCrhT77TL|S&(SU53B(v6X%`R zh<;#^uoS!=$h-Vh5vIY8_pqZ%!`J?0*%k84IwTD~x_WpEk zcWd9a_Diq#y;eRoe7P97sf_)Zqz#>?BMKkhSr_o-#nQKsFE>G<{ROsGT(Wgx8Ca=DLd0INq?GtEJh^h7ly(shA`P29JPQnC-GR; zwQ!5#bs*<8T~dTSh6`J|p$+zS1G zmgzBJ5DN&!%#c4G%5F%?#&!_`l(gWqcC-bi@EAVYGUbs_5WOml-}RRLxloZlMB0;d zgNPQjtShz|RK*MIRiNhtB)b?{JJFAmh?uy+D_*dIjjl!#wDmpB7&EF{PxHh$J(!uG zQ;m=SEqkYf%*3}WE!-&auj+;iG$2sP8dMoc7v{>AEm*N~yIfq@21R<&vv{{bmt_f; z5I5qHS%y!bhqqVt!fZsuzt>`vJ9fMx>Ar#p1YHT3`%v87meMqq? z{Nlc$j}kVWhbP8(@V%M|vO1~qNpM8@Z{QPkM0msLN^58pQx)48b(~@ktH#;KGoD!k z=BvsRMKecHfygLT&0@?nCJ(>(TO>y_kQtz>Xq&|<+N;_tA8#x+vk*HSGOLr)mRDDN zFxjFS?suWu&e#BPK>41Xk#s2kXoXmpNaz#RfSmxe9*GHn!2&Mao@KQBN`?tF&G1h&^SY4|T%2goO$XD9WZN=%C6V6GFED8s` zbkZ&qQhc}}`%3i}V}R5Xjgd*@A>v3vas)rjlbmCVP~?o-V}LCKg(33rp_$VYYKcp? z-JO}}RU9qAoWq-a5{Bb6uES#1PXLpGM>lp8vh8HOu4))7XReuS3rW0_Ff}+J$ z>j;Xqc63o!Tn`MeWyt%eQ2E5`1Yy11`_Byv*K4V3JW=}mVHaFp5sQP1XE z?g`HxTVIkqRW(s$i_2&aEmIpv06-bju_>j3=Id4hv7-4S?SGn;nY;~nNli<9zuRKa+UGT`hJK$@ zTaYwAM8vFh96?Yjz+2C@66VEydNquWdzCWGknO@)cA1FYFvKR=bA-fgudcL#Zr=h1 zu&H&A?mCi?lZ>8Fy&yB9ulXJqfl>76!UdEBX#!q^*8A4nQgjBaGF`M>D<2kK+v?j2 z891tTt}b7{X0$Y96UZUxCg$hoAGd&G^wv?(f8e&rf>gD&gj~kJfe{YY%_RsP{k)*~ z1Qz63M2_L%hqY*_Ka>Rh!pgV1{9y4CdH*gUd?%4>BV`6gK+5*T1aPfRx7%aaxH{4pXlX z)u}c7Fd@O_c_KudflX1MO9lM3*B6ZCL;V)amTRnozQ3p^UMsf{?+jJQ(VfH;Z!ZE) z5}>Wd0xwbFbeYu@y`W2H$TO1sJznk^1ch-|a%;vtM-TRNXK>#eU0rn>7*nEzqt$LG zLctEp5lId==~@#<4A?8CrB4b0*UeN`B#9!N{v>-K$Yu$d5Zb>;KB^P2ua22Is*8MH zgn{H6F_NK6+m&Q3BN(UY#IUP}KOU7BUrlv77ugn9va1``#~Jl+!Y<|uGts?^G$~wb zXzUBMpkALlb6{(P{QIORMo5ymBl<=V2pHi47q zIJeyvhwo5=j2>+C;lnJ%CP{?Q=w3sqt#o?I0CHB%VQm|b(IQ1~g236iTR;Iws$&2e z`Sv0Agc-z(T$a?2%4opw!()s^EIpC=Bn05r{dIC0@VlWx^V;}Hi?*P?({@#7ER%CA z6w`WQ4g)-$nottf5&AA;iI@|nnz;;-OH5#8id@Y5D1LwDBnq>&X8J-9&rTL`a)6@> z>dACMtYU~%%r*%t$PwAVbg~(D9KDI8GuUV`jabrcYikXDL~TniN|eB6P{|Gjukz`i zLA$6PC;^p@krkB4L=Aa%L_^KeNn}7mbc+KpiWWCU^{z_TUCZ34;rjNZ&y2 zTO92xYA0%7W1adMS`7Y#0byj$28&H2?9q;tP;E54HI$rVg|fDZLx${bgz=`2oZ_!L zld7pEF*iKO0?rjZb?C5~LK+OojDD$6V601dTOqQ(G_kezSnvQsbX*;F6VOcSkr;>4lO887=mIg zKtOj)BE8d1C)-d;#yXnv=_4GcnECGWl#LU9DE*k+5?MfCC15)u-Gl1S0IH`1c`U<& zKe2emNX&f|V?#2gBxaYH7^la0qeItfJqkXy?d(`&gpa=CEu)d9&*rBcr76!Z)!SV>pZxh*x}imUUkl=g_P{ClMdy z^*vNbsn=vFsGckZ$@cGMDX@X`QsC2U=HYp2_nBe;8M~hj(*Sgk|M%I`XOi}xXW{k8 z{`2VUKOk+jAWp56(nmL+uPFCx*rv?Kzm-O7#T963JKa#`kne6^A%gA zh#@ixPkZyl%ESfv->TprCD0-qH?8fvotjvBu4wvXR}A1P@Qo^0@I#E0D%4`bw)U<4 zqEQ9rs7HAm#=dqJUDd|jmbD1$cbS%tNUgjY6h(=2(D(`$ya>&TDuUDRSrvS2?UUzl zz^++uz&{~y(t9zo3?7sy=-TwOWlx0Utas5`1;&IHiZ~=Iw+o;9 zCF`<;NaUxed#EHDZ7A7SlDcFox7@A)*S;7(5`HJAI9@DNOhYj(SXSiTeuldfd(A&= z8+4ex3H9{`1@-_wWse5L4YRc}jv#OB6bTZ5w76kVl4;o-D1KI!Bsp_>rTS0F?!GY{ z44MHC3hjnIf(%Y3u*PT+?!xn*tiSjBpU}kNFaZPZ|Fct5N&m0&XC_DV|MT$vCsIBZ z?gDuO<;2`&yH7%sE-6qun}OfmU~5^4KWoTqd!b}%lr2pUh?V7b%TS_Va%8yY@OqYe zBZ40AOcuZ+d5g?23lsm-+%L>E&K$H^UC z8RHyFRdXF;EyUI*B7{tGi2p@z)VKZQLS;|EJ;AZxOZ(S*=P;>svw@M-W2W3X^egHi ziKCu-a&GY%YwpG}{MC##{G4B$v*vGHnR}WH$c$ED13x}LckK%Bw2O03%sq{vZ!rWZ z1@fMKa(4dO>|@vFD5U@L)%hiB;UGy3w3*&4++&h4U6viW$Y ze8ZN^eIW*<>4Azd2%nEJgXq_Z6M0qq+pO}+&NHiJ&5w;3vWJKNV^_K(kpE6job+I64oWmHVbHJ4e9<)KISsYyOROH9m?q`MVmKmJu`d#T8ZLa zpg}@>0=Q6QY)<5jv{1Bw%3L!&R8?*t6l;f}U-QV|O5A)sQEy61#0YI8sY9N_m;$Sz z{5Lk;s28mb4Jqn|SUjWAetf%;O-J&4m>$Ap{xxPWBB{ApvDI|itBMXZZu!fBgl3yt z63PV~8Zd8>N@Zl)eVT5xtNTbb7+VFX2X!n6m`5Mw&l9kPTHQ#rVv?!dPTpvN`JjiB zO0DUMN@x5z5V_iK_<(Va^B|M-wW(mC-FAia?$CbBm{?<o+HL1@jPd(nH%69qi;d$Z*-fpZBDhODIAPs!H3C(68}vGTa9qsDZBp#VVAx z>BLRm*Ie|eaDml_+D5iBLHWv;OW(NA4_xo{nf#@APX%rH97mzkrtGi21R z9_}U6AVV|ioeJ4}ySTMyZLP2{4@JMFbuS1zhEcV2W#*Me)hNZox$RmftG8(vZ>wbeAz_sR@l2~hV35naebSalIfi{E z)I(D4b-7(tYyahT{Jok_SZ%YPn8D@=vt$Y8tFtqcXaz~X$V*QZ<$$Dn2282VA(M%_ z$tokdiCWt)_i+8X)Q>s}<7l@xg%lkkO(B^P7X&Xxn9#)7f)bOrx>4d@sG0*BV+@V! z!+XQ^$WPTO96S#r(OM*~4cmT;Vo(ur9B&%R6E{Yel&p`$jg~hvdH7c7LkJJR{3^U2 zwJK(On6{5R{Z#%?3T%(&eX*49rjs8xT_mju=Tu|I;Z&srInY62B6nt_2S$t#x;!h| zlQpNKPQb~ovs)xK%rTKAW^)bMznPxs+Eddbsz0FY()0=&p6CQeq(h?9{7K@M5z~Lhn2{ z#IbwvaojF4xP_Ry&x}o$ic?0Wq6j4pgiun^@-|xNm+rv~L_8Vno;v2CrD)Qv9itT) zt)TIB2Uy0sWaXpHfV*3rYjSWX(^*eO)epnKS!%NQI>7MWd;c)Mb{X^+z3 z#sMsfWh)?Y^qGfU)Rx#zywd8-VBD(s6;%{hck$Mzcu}wiH`sB0y&K?M6?sxw6XV3X zE^^(bBgZNV;qR)I(6+f)k7H&tG-@g-OG`6`0?a*Jl$|(@ywF%*x+L<~k}b2hXX?jG zG(-8`_l!}2;V@~n1ULzAu%XIZ^4%asds9SQbTy+|YM2}z+$ho0;>nQznwe7%RmM0# z{(E*R7619{xwDfa`R{|2|6(4Q(^_Cl;fB%epN=!z)iq*)Mp=s`IYeXfS=v^pa&&!mn^$EsqAYWX%qYXwA|xlG2BM#E=D02OcxX3sE^o)g z$EG(p4?!TMX^iXJsXDZy{a`PtNzE>2xH(=@lbl`B@bkN*COx~P;S-mP8iIcMI93eW zO3)W00E-a>#i$FNAR)28Ij;>ca+@C9$(eB}aGfc32PP+q+`mcc-7#ViL=ivYTO(zT z)~~S16D9}{_f8vE8Lg;W+N~W4-T|r{9c#!9^Ge{v{6&;-$8_H`q5~njo%9<{dq>)H z9GhH~6hSqLsB^mt*&Ou5FA-^wwiNKl-ToQ>FE}Xq`$!8OaTRAUA}tlt%|; z4Ti0q>9H+gk(`L_q4?RXVAyuG8j7MqL?!^5SPwO|l);;rQW)T@Gr6dwJlIwt0&=@v zeLp2Gwr|cZU!|!tgPoPJ)iT$J86ze`W|{1lq`6ijgt^1Sdw}pHP7biEPGl}7SwLut zV820su_nnT!psGSA#o+7k0ldEY3YSvy|sAFsGoFOPjTsXp8*$%2a616Dkk%edsNtW zE1pd#%j-Ql6j}Qjez$eBxM6m8i8<}ijjdyhMT$%Ob?;ZyZloc?Rlu=%GWVLBAsOhK z0m)!3}xhV15rRk%G`xbh}*su^Rhc((r2%aN{V_3(CbzqGo z{@vS)MKBrH0Wii_F+fw|1;)U@7ZL(vtRM0+{jW{b0YJ>e6SBodXfJ8=Nen#Jh1lW;_q^eQVBe0ZnBqrU?s;qj!!AnP84|piluWbfY4U{7mEUnI zEFyy0Z~GixA5&2xASfdVJ-h~~c@MKQ5G#%m$n3h?1}e%i#N9bcb%zn{;Oa({q$0<6 zJBZ#%Jc1OV6YNB>3{Z}b9`Ce!?=&0v*j$q|KOVQBhZK++;SriH`sNa3ogwKWTW-;3 z^$c{?ci|A=HNU>2SFc|nghS}h1yqjWhfZR7k|6j>v1o~OT!nz49HyL*^<>yxgY8}AHc04L zND9?MBd6o3q}&|vo66Rx0O44=>vF3lw_q^>9%o+3Tk(Rgb;FL|1eVVlkAXEV!FcSH zt|bn=CPVLVu2|wGu+xYpj<}G2Tr{o2K*%$R7(|B8=#2ELfqWlcB4g*f+e~GfAlui$qlAbb4Ljm$q${ zV|Tr@9e5o#Z{h?VNuiKJZW`#3Gc{>VjBrFrT}6`HMz~~SndZm8_&ff2yvM^ zZVe>_DYo#EAzV=mcY^EHgufGbm`Q|#1M%&Y*{hD;BZrB&c5-8hY@n19B%j+&gyIry zApld@FQ`ygkr|eU!B&xuB-luHIVc6y5;!cN4bulW% z(@WP9LAlIOf6UYwX`3`=2EyR-(vyJSTYhT-5k*o!(0`vZ@*TLC2q%VjjMkE$WUVDp zm#Z0Rgb&^m*;EYdBObStk4UzcTwkdcGJ_!|!2sq`_e6?yAFzPfV)C7X2K}uDMjB={ z#f~;Vq3q%gE7z!95k>Xh^kW9m*?XwWg2(LhVU{QhBMNBn2asT1^eZMsi_kN4pl~w< zEu9lDxacW><*vT`2qRDZpuQ2A*iR6Q_pltGq z*8(LyMJl6!+FFziu85&h(ccGkcChCG1y9p<;)RTuI!e=8)uy&ahR@7At8(qvn#i`< zoa1#OtN;s)z3y8R&@KFezx%eCEPZ>^gC)RtdfODmcH32wi@HyUVMc|Lg_bc;xi#TK zI|90j`Ww4)fJD915m^cs)1XlUfS8Q22{`)gp4D-Y>bQ#@#L+{N1d)w2Gz2a0+x>!} zFQA$LxD))yL+Jt&44pa}T*HTts|KHd;?y4XNtVp-C<9iE*W|M1bB_{Bq=j%ggzz#& zGeil3PI#3-=4ocJFB609W^z0kaV$)p-{{EBX0X_6wA)D$1Q~2dnziT=-5`VWh|z|! z2^V64(wVBn%s))3s;HQhcU{TRxEXCDpA2R$k9OtLvg|VOB{W9K;8!$WmC^3$CSU-+ zRc_OVS(0C%17CY{M^)P&1!_1#{~xXY4f$&=+&Yr@Pv<97`rq^Crp}G@zek(@H{8l} z=28DPZcQ*Y)Chr%o|5nSKx0aQpQo1HWUc>+oerXTUX)hOhQVC*C``d?Et^tZPgf7 zuQQG|czCY@Y^U$-Jwwoxrh=SH>oe|?(r1YRql{podgP2T_rI3+54&4mUjQC-| z1&sXAj>We-nXE~qC16-3o*l>6*3EKJ5>fzdhAA$fZ<0Ncuo>I)YF=jt#uWlHqY_=> zY?CzLhJV}N@B*(=fi^0Y$M}V0-Lv+96^O(MD2S+gTY1`874(evT2;}dAbV|y2C!(i zkqALSYso8P2rc-*ULvjpHqh`;t;6qG3h8gJ@Xyu#Sft;$&Lk^}MD(7xf8J>wpIwS% zI%Gc;dS3J)Y2`4PCYL#ZUr9SaL9ywVHl)o+eBuLx=(ILa(Z$C=rH$=zbX2ivyW3fx zc%=+k2o=~|M13VI?dlP8<%<5MVt2?AaG3I9OwLh_y<=O^LM#X)atpxc;O1AnyBL-T zZw*{hx7m_TaG#S3Dinh@B)WK3+C&tEtRrdp7$qMX^;?GOlnOqxaLc-WYiZe9cyexW zasCR&y@1g#Sx;S^zkJmK0v|0rug*R>XFWEzbaUY~bJo?l*(>umo>-!?vp46}MEl#g z=x&&oaTf7WVJtGn-{lRH+NC_IJ;OMipCBiO+fw~v{G)y*b=RneWTi|8kd;lb)Qx9e zrxAstAf+@FTU9ItKcZ5r3B@?8Tkmz5h!wuOg)*j>ly@XmOwaP*iN#2RRT8A+?F$hs z(-d9;%kGa~WKZ%x2LYw*8>` zWJ+?vVhw2%^Bu!~?Y1X2{zle%7`#J9S$8iGwzY^)&T5R{0Sp7!=Hi7i{!I5Ye7}QMH+PziyFpgiPH%LDGU|h2t1P&Z z=qfD(%Bq!gl^l|y%oroq!7);){SpbKjh&4R%SmOC6$uOb%K3(z41q5;4LG8a{N@CrSV?lt?JE2%FEbwUtpQ8Cf)vCS66k zCMHgbc?yW8eC)8rOe%SjsHxD1gP0MZGXiVd4{np87m*>FIJ^@!6d@_zS0b6ASZsmY z_R-$q4krCZJWDC}!fn@WD{m#OP0x-rmo}=`ujo3UKhSDQ+;xOK#1S^l4dx=|gg5~<>rM2} zL^jzSvKd3paIVebJ~CS6&tco`&qk>>?8Oz1{B&)tb;}}sIhkE~aT<>V|e@va5 zOvis4#s7YR-T%w37r*|Ml?B>#BH9)wSHiWhH(97f<|G_lm_v`NEZGgta>)77Q7h8) z?r;=16;ZqeO^+L%Bjowtc1V@tHp%P44Lp*f90?_IPjmrJNSrLNV)j>e)gZ0muyh6o z8c{~3%2C8LT2pMWa^1?d7QnXd!dhTNeH?)L7(_=5MGv42`k|8pxomB$uTfBxLm=>9)0{QqFy{wdf>L&)|Im7M5%b!zD~LY9_Z_q;$TPlp3n zCT92ubxe*Ox!GFdSX7*8cSOgkZWg-g#8qDf)rs7ddf1Z)atAOme2jw$ar()21@WiE zq62S_tPYM+(3{o&Na%Z{_&+-Jhup^i{D0>B*(Clyd;0V!|HqNq|BoXg?RLCIsB^ws z<1M#S-t2VR<|DC1G9F<@hIAr$374fJ#&xPHtk_O9ia212^}OWXa9ff#p)7CN&8?zC zlI>V|cHt{1br&%(P(r>)m`@ZnifPg;v`4zIFZ2@moC`xTzAH7oftW# zFejM^&8$!vW6JB_MN8~ws_PABW7KY-wq)%n$L+L=gV8zbu>eBn= zllCfyb9L5e)NDVCrG+7;QRoVB2XwU-8Tqnqwa^ulZ3STh9b;-73=kB7K&C_Hfa86r zKwpvVyKoSh`$BIR^1f6ng*l}ptrk9(goiF#=Q#}yq3z@C+hK16sspQuYHOz@396E- zM=j<3z}vt^s^VkX3^$_Wnz46gz#g8W+%A;fuUN0uHm)sqn+x~KhQ$Vdx|lZekDg?B zI;x3OWwM+TbJlIwZ38>Jv7y{NXujCWl0BL^@fM{l2-}VqBs?1>?IiNi0ed`qXgA2Q zq#~EbXzs}SR!>-Fs3!qEK9gAv#Mmlk@!`coTU6BAh`T*mCRiIlf3_?x6kADnl-KFl zIHc39#d*nepkLf@JAjv#mv2%XH7@l&GICb|{sa{66@?Y-B!k$6D9NH^R|(CKiB_4& z%~%Y@99dJkOi5vyHx)A)EidSlQ}pLp9X-A%-rz|m*bw3uk;g5>)}`iccvo%b)jK^5 zj*f2P+J^7nj&kZ6=XpFdV#RP&VbbQPeWZfDn%f!4F?TJ z(90#1FfK8?&GilBEK!RoC<=vP!InJ2&;@f^jNuUz5K6C+7dSV~(N01>XEW9cMy868 z9-c+xJ-XXaE9OxL=qW8Z=CUQbTPmTpe7qL`4QXGF$=xu5uMnR?H_3D&1|z;nM&nmO z`>{ukQLO7ng-*~L2AtNL!waE+4AWcnu(Kt2&3(e0i-gRZQM=V;-0FrLdzvr_=H*~d zqdJA7h6R@ZSnRk4k0laza-*ngo6b$4$knP8Q7*siHC>7Hfom?y7uzIJ{y@F_aj0by zqh_OBfccU1V{=&zu_W<0iBU+B1&JhxF(h^<3QHWVX?G?AQ^>P6N}Rz3+d~dOI|dFnMALB_#=b<7hd$p>Hl-*rcR~k|8pbz z&!eUPYkmW+o;SMV8QI>)O_9)eGK=3rI~J{333V_xU0R}J2Hd!01T=x2&aV=|f|a6! zltCR|5-w<84hkmXHd8T+<4P7LoAK7)7PmG6U0y{x8by*5UanRjpT9O&trieDmu&u9 zFyK+(K>y<*XXf27t`?D&RQW|!O!TA9A{d})IEL5*myMF$;aqaOwGMj61kQH4AM)~q z$(D}7d`8Dbqu=<{7@Zc(=tPbcHoGC-!z#;LOzjqm-UKe2ijD_gy)x-liz5(jBcs>{KXB?AYxBq&>JaCmtlu0?zg8H_~I z_T;S`pC$4DJb~53r_!jaAvVmD5<8TaI=N95^k&GSBxI~2#PDjdM@HFb8&SFFC+{IL zjgqPE1}X@ZA(GSC4y!sAjED3;((0)_%U#B+P|WY#F29PW|10h2$lT~#lBlfqlJ?d~ z_LJ6>#vsd%ZEe&yrzX2@oCD!BjJQQr0CjRKqx&1U+73|cDEVZfUq|c{{ooDM8&pW^Umr&t4Ppbb@UQcxqGfL1x9+we&(IW|THSYCTf{DJ2?fUIkk!S`l4I zNn`kNjM4qw@A)yS1ph>slG|lZMnN5=jldPhv6K|yX|9KngwdUvLo_#MBhNUeRxi;B!zVl_t=HYXu7EEPj1McSdlUWTDRV zOLo3VwHzjjbltjJ&mM|NV5UZ6zPt|uO89<84a83f8IF*wuEisXjUES$cS6K|S~>|p z5Oz4HQ)H5&=SNXwu{%cGd<{ovA*U2yZBk|GLgI66(0z<45lwdmH7VW) zt=~?PV~rT*ZBhm3{Fa<+B_zQIPV@=86zHp$y1LXwVn@r(y^fKOA_3y$n7WM<6N%+k zPOYY;s-(gq-4t1QVs4z48Q23xm?!fZ6)ziWQwSo&&1;Pvj-6eI&yCaWI25eZZH0ai zlH|MA;bs!g`AM*o@xcv8Hs6 zqQQj5B(S2C;?a?ey<$xPew{cq3I8omzOpiP#@zJKF6@XrHeZc2?IbXBsn2Mmqx3u~ zMkyyce%;xL@?B82ZhSNzauQPh^m_(44!gj;!l%$jb6B!Yp>oMOaMYByPwe~cm%A49u= zgP<5Vz-YN{QBS@~Evx6y9E@>NPsbwS8B2kh8&Zhz;EOWqq6{|{(_c`4+3s0Vw_(}S zs}kQ%KRL6SOdWl&zWMXCe4a_d63@MqGFt$xccB@7?da{7AGxpQIdHQ>&q8*08 z%mg9Ty(BO*dQI{iYMzELk4w>i`UbAS#tS$y<~`DcgZzKsWt{$-nuPz2=)a@*H;@c6 zwgJnw=xW1YL$~lt#mc+k9%VVK8bz&%71>9trBaEngeWWJ?mi+Y4&CK9>e^~am1v{S zC?!ZzdQsH??NMlfbxsVejw%Z8!##cbEWz#LSZfaO)k?351xow`Bi-{Qtt)VdGQgKx zj@O`rXiDa#w%^KI;}e_XMff{`N-_L)8Lq2t2jeo;J-A(*9hK@)sZ_@;42&K>?buv| zClrmKsQ_cf5-KsA`#LiIew!Yo!E+frQ2AHtJn1+`a;I8* z=0(TGO@OC+8Y}gI+caKE3Qdw@rr?WIE4pgc7{@2pd59dy(HPva$(Qh3Z9)=sk($dM zSw^=pOREe3sokCc1`Hr~ZUA@-cQzVQV8E=vbBNv{$pq1Q7>_T%hK4$&lcX~k0B!_C z65JTrsMXu>YO!O)=20ic%)`ky21_IX#^^vx#(|Nk_4C*MJ8}QJVHJo`=Md~c2HpQ> zlKwwar_N4}@_#=Z_rKZ(JuZLA6;iM!9E-|X6P_g`KunUKrU4FqsrS3Td*9Ai{cX0s zBrX`;h-b#{?K-A`_Q3qmv%~%KbQq|AmLBA5^)d*gPo=kW}vR6m38tfS@i%N;$V&u*X znW*V|{~rePKaBtH)cMr?fBMYm{(o5R|9;GWbpH>y?MEE{h~w{jZH!pdhYSCshVMw? zKb<|7(*K`7Kg$30kl=sPsLRR9yaKSmNZdO@{&&Yp3Y?;av2S`$$|Kt=n4FGdSGgWmp&F|t zaRQj(gtdjNNiqg71dsMOL#cDoiYEjnliO%vB#u&2{mAuZQ8{d+&_*S2=19qP8pE^X z;qQbB!GWa;cPR)!lF;J;YWFx(GT!19YWWjf5`W%AB*HGGD$zE^^^&*%QZ9C=vbrNT z`#zAV>>!Dv5okVx#^vz)Qz)fpct{r;GsmYur;@cvp`~;TAXIFSR30}3#Wy=kFOkPHZ116EZEXgXQ=TvqX zI&th=5u=r2$}EP}NeTou#ig9qIOTW;(Tw-R5hoaP?(JEO3#=2`K!&sW;lq{kYwNpb z_9x))WN}J==4UF)^0fk=Q~QOPa?FGw=B}>^|ysx8B2AL^Uza_Blni zF33l7P+$ufZMB1D5BcO26?(QFM;A%c6pHdFu>iN+fNju>L{bi%_#hxhY7GF|+<55m zl{6R{%d`&>L#7lO#%y^~Drt_BfI7oaq~|^}3@-Xxfn;r3ooT1Xfr8z-#d}st;*aZS zpC--)p|}V-wcBuH0BAs$ziK4uMh^ujZmqend)*T~PMFzs_R)&n&q0@?P0|sF<3bLe zA}QB}pp!@gz3~VIo1@Ori%I)|MQ8PnU=mDHVJdlzq}vimi;Cni3IJbl7~hIhW1+jb zi-0Mcs$zd)C+p760+|USHYT%+hPOnvFTij-u8Nj%Wa7!*{2v7(WrM>iy7j>}x=VI& zko@n=WP<;jnmRuW;zfM0MoF?!o3WVfa^YCY&4v;U% zd5#q;geYPfG&lr94W38yXp9Ih~3Al@%VBfN|a6?5V(2|he5b7QW zffj{_7$~k$Zm%mCD2nAG>XDAF#u>$DtbE5R(|#Nz^F)4~y0x|Vlb_kcibG=~Fxabu znNQr`d&|K8;$e6A9-xEp|5V!l^X$|J{~x9QrxD(JP;RgO!NH>-Vp(ssi!nwA?*;*& zDKiMu5RRf5(Wz`o1XfmwlHD*Clsr{Vrw;1@W1>{_t=cxTT=ee*CF;WfgBd>t1EWht z-?D(oso=22)b1N28Q@bJB?Q&(yRn9h`~ZDh$|XcY9I7g-#aOxijgB<+_nrQaXY4yV z{C^g(egglWn>sV{|36CnZ^rtT@O~J5|0eAQ71__JVJ0Eqy(<`kHyzs#o)$${H%%0l zk`TZPZS9HQW3HWO zAr8DjR3Pg$Q<1|iam--+Yj~R*&2m(77W*qn?rwVf)f0@u5(n#2#wLAR1A6Orm|&SI&f ziNTGc8M~xUU)^n>Gb?f_7vQ_waW7tYLl?Tgc(QWxLYco_D9bBbC{pCm$U5Xsvyqo3 z^peBtQq2bf9R0$lPMxx2tG(2vt+d|7Kz;;`tIT=*^~kXveP1zBo;@f)IW@rzoUpTe z?E^QA2NW~y#?)-)iARt)!V(ILI)VpQZkHZQASaFqY?n(X)$`KHvdxhp>ru>PToT$0 zw_6zEY7?`U`LJnQb%0yg$KI2l+a-YZE4f|hZ@C)qILP|?2RYpmhkb?!nMVcCCXbt9Zjdz#i*8c|0q%s z2#LUc0C`^GbQFn%%MOKpNykD*R1dGBkm!h05jSr_Q$@nkm}4a>T`(qJTJnvRM_t4G=C5+;s<|h0GWW#lh!diHmrg)XlUtzfi(ha9Jf} zEPA7=h|uvtpom2U;igL{3Zu_KB`gx@;B+Qpb*r%S_AZvFh=zUNPDO3kSpihk4$|MF zxPFb&4oDskoOE;K*s9v61tVG={?-(N`kg@~T=&9I!tD{9Xhed>z>+Meltw?XgJC=E zcEjTdEt4QBGY+Vl-oA6#qMgxu4l97pp~S_dPIrC%LK(jv+BD+QAMPwNMpx6Qk<+G2 zbcCbKi=4oXUk{P-B-AA$hhmGkdP$U-9S74;*|Nw!r4woKOLmWpAw#X?9UU=KFQmj& z)GIl%2_MY=Wr$IZg#Vj5e=5oUou3@}|2!!9-<_BkEkpjNH%OUf>VwOv3w&Tf&d?{) zth@di8yJHQ&s_mB4718rmlvu)grP8T$(p}m@y{#si*uKk7Z#uC6)kSK zUNQchmG8CyQF5#_w7GSx%U2e!0_xt1qF>>=rn7_E2jCBg%>&W`P+2!@>=dXO4%mhC zKs)faD6pj1VR4d{*=yHS6h0#Wnuq=2iKpZ_X}XU78_SpOGoS z5U?=3re8;+K;90NBnO3p(MZQbD~6|Ns>V3SO^xzw=*aMTa>^H?9~a9^sf9*btPBVr zpP##S1<3dt%X2rDix59Pu0N9W;;YxDv$` z1C+qZdM$=I>u^P@$3ygtxVI%s3{-Hh)m?`o4HC@|MSugsq~p~v5f8F(ezV;GwjUeX zNsI+<*AOlJghEnbUOdbWNZ8KiYx!_x?XOb$Tl4|9j@tDF4fm z{Oh}=F*ze4Il#)iYvptlDXFcEX1+}z{>?C{D)^Gk;+^OqbBcu)$wI?!Kn=Sd?RJ$R zw^}b+RS*7uo2<{P9W#BdG6`p#N$fo~8Qs6wzTlx;{EZS-OA-<DDzY{NkJ;*vIKJ|!Tn!y>)mqQua$GV(ARz<2QG!SC&neLcHBE1 zv*T?<6(TZDr5;|R>|CY{6K=W&7!OmF8Usmgk=DwI#n%#A`;h5m2mTHoo^^hdbEWJ z0Ljy^cOjQ4nkbhDXI9mB9j0bB)6y7`^NnT4WMqi%5XEQVO1dPS)%FIy>3%pUr8Pw3r?sQ_|5am;!Xz3A6Xe z+D@-6Vv$FowMdn%MK^SVtyFe#Zn}y-zI}j?l}ZnQ-*y8}8UUCSCN!x&Qv9{xb_hHa z)%rD$FcFcisQA($`rA9qOz}dq|4>E_1j{rFk;>@t!ty9xN@2Qy>3y6dVQ3QvDFoWF z=JuQfew`Y9*d;Nbxye)az)Y%2;vO)bWL^cvdwaEzu4GuXSu?TLDwd5YI+ID=qf3l5 zTTFzD$4x{{tHhJ!1>-s`%DD*GAlRX_7Hg;_i0ZeU8h*l6vW@ANBBX-7p~DcHk&8+o zJxKh8S)Yu|%T)8i1tqhU2d|i>&nnP39_|RIVM^LB`^~1`0-C7QLhOKQ?kd9O=CKkd zo3bs=otBuAWQ*#wX+D9rWdbd;=9;;MRDEV;NZKmXiioYnc#DF_`(|Tm7m*H}lDZMu zh3(7e99b2WR0KvK_J;9URx{V}It@25Qh8)$tpUDE<2QO7Q2`zD)>ExyO*ol=b&{?4 zZD8(tqPYlx^tb`loHDW1hK1~?X=tEW3N5(M_jk*4w^`Mxk952n%(&_BW^~9zIiewm z3c#o0+6HGyQmWA_qz^Oc4Kmtdk+_}P#Rm4Ftkm8Yva)n1MN=R&NNOB5H0)y+D-bg@ z-Y+Wxr!T-6R1jKPXDs4KT>lWr{|zC`edYg8oj#L{|BW(%5&!={`G3}gvK}yS*IhC} zB14=1FDbfaDy-PkN5Ny00#jpS1;xJbOva*Nc9n6x1AQn={kQ-i-vydthphK6jQ z@rHR7c}KsDm&j`*+al66_HbWP`>EYrVZY51;YJvg;k9?FV3`1DSqAb97kLMDP3m(9Ayu-2anPr_Y>A@_$n!`Oi`E zf7!cwWd3n{%s&ob{1JH*Wh(r7tGrbCLWUxcxlN{9D3A(<%R(IwFJ7i42m>y!1n{_q zJZ=pV0WObAO{Z<`T6^+)55C*Ts}fh%XB=F-VC`G`_sh0L=X`%27`VvqFj5FcqcNo( z;=7MQRp`A6s^T`tz3vil7A1O-r@sZ`byO36MZs8f&lfIe2@D-OuTExC0v(6tYY9dhN!s0<&1!=`#rq&6!?G@yf^eU5E6agK>Hwk%Fd$cokh~5_(6%r zK}JNBP}T|R=&UU*6(vGwVSvq1c6X{sYBSSIKmk?Cfv!Q%Yju29S%xGktcHDG&73G= zvRki;pOTr#9TPNC(p`>Ys^iwHnA+V|$CD>sK#_r6X)cVP5+&0!4hSxN(TW`t(YfoH zjvbN|m=y5TV^aeRNJRDsiFib6AI2;9HRlxK8R@%}6nkyQ znxCJ4oc++66pl6vm8*W#4&|aNr&cNAQd3O34%?J0y=}FGvsb-d9qRX~nnz=KW^hhO zgjnt!2bnoyTgb7TB9~zqluSxRidx<6C>~1~YG)13OOe=8j-{yN!6Bs_jhjXw5_6zF zV$~uLOB^yDQQ%lMa*xc^STZaN&bD$;l<+;?+;|S!YWGLzT_N{K);b(cZznPOF!;Qk z#OC{1e%IR0FWdI-68e%x<5gryd#``TG^W*pNrWV<1`R6WE=&jS-f*zbqDGSXhhSX| zXb!r3M`{N-vVuHP8_2t`fP5j^Kl0?_ZEtP6T1wRpcvo8;XCt&4-fh>q;9X?t)1A%? zUe}>tv|{WHg{M&g4s*Jv=uC@Wcoo~MdaWw&pge+77};VTx-BMs%ib4aleuCy;ZoPc zm{g8O3mB#qr=c%`1k&vV;9**RrY}ZT)h2PuF}DWgz9OSgnc-t+8#);7AxuQ!ylvpw zA$+OmUQKPw3Zi0RWJ`K)^8dp!{~W^pGwuI*dX)d~fy)1n#QYQ0UnA?!hxPpT4t#sa z_MfLy{@ZWe$2k+ttT0`yHG<3Lshgc0e^>ai~$uMTs0*R z(B5`;Uj) z{9nQR0ImPaK)~XGx#~J~Hz)(iihW#h!&({f-DRBHvQkv7Txv~^tNP=sE0e2GC}{Ta zAXtfR#6(YlyXA#24b&AI2U1cU^Ab%|i6BGVyVPlR%5L3*zSP~in(eNza*!6CE=+q6 zgQ6V2@!|Fv=Th-2zgOEV`k=3C-daQTkb*gKr5Ls)tWmy%NmC7%3D`YwLuvt3ofD9D zoZD_oY(*P8Y!bwAzOhMEuL4`-G_1UR#jRPdvWu4el-EMb(8*W6@>M#9a9m}6U|_R- zL<}mc;dVO8NkSUF(;+{Ha^2gSR3sM{i%&l+lIPbkqV2)pqNQKC)WFrO4~={a-YBOAHa#Lw#?_}Api(dv{h;kyE$g$VRW zg)d%73=T+6_p$Le~tXVhWGz6@znFi|10BcyU9xCxBHKVyg51M=JMOSmNL9;p#ESJ6bH%z_n{M>KV#v63;cjgAjYg=n zG}6x!+#E??hb*`T55*yil$=0?G(0G#Cj5T8Wl^Wc-!<#rJ+RW#Bi=X3Ny!Lo4UIK+ z1Vkd%E~BmWEYOo4I?o`bkDfKkCMxKv^bAbT-JhV?S^htKNlGE2sAE1oaO3Q3BTxw3 z_NI@yy3mFg?_#{!@q!oA^t+F@ePj#uymN~Z#B_X|qK?=F>K=Tb64jn(zE8s}UvcCW zbr@bzrmX3(W5eHFjQ1gH9n_KPLAo+2$CjadnMSTlFR(E3M-@eW&b2G{AoI~a^4|yQ|8wT-NdEgE<-bSb|AXp}k^j%bdj9pd`|tmACguM#HM0LW`tv{X z|9Lq4e-5Oxc(6V|N8kc9@&I};`2VQRjR9<_#NnLh0Wc z7tJ)IRPzWtw_N3Aa~zYHjQ$?id9YK-Az>e+zT?U z{>UKo2n<5Qqp}EnUfF+Q?=0s(nCWE9=2H;J{CJaHN{N_9%5__kLlF{nT00h-bXr%J zuV1r5uT^tNc+mhF500062X3uUG5UGTE#Rxr`9XqtAaZJxB zo_(!jvV&*#34oa$xf0p6%*d6>NXE{J7MU6E@kk>!-e>%8s{aqEj{*4q{JC?<_>bpL zof^e|I!gNw6u{i}H@v{hvd}1%%BcT^gWki00}9lL!A9fr9%b~#s+#5?WA_@@@#;L= zbDU8B4r0VHGTG>Fvhmc_*<}kRSkE$R@t5^bLl=X4wA0Ee95G@@ZI2*6BRLj`TTLkK zVeEvbf$FEGP+OvO_?iu1rTN?v)q5Q3NqCPU<&MHRdUA0VAzf4~N{n7u!A4i(W+J>A zO`igyXjTK7U{N8>Fu}0tdVwBLQLF9o9S?@1f{0as!8hUcS`8@nuT{ag7)7gchh!%? zpfLguC2dS0Vb~COQ?w%2jiM!@8{xq83%tKIsRfTi1`_^?tGnr?{Yh84<4K@4LhOv=k8iV1`-re+IAG!^f%ISbFY}Dc74j5y(?>a1i zC$<=Ezuj#RZ3d;}5TK+g3Z+j_!S;ux7L`!T>v$xm%bT+@y}NJD)--yXSjuAse-$M4 zI!xHKbw?ytN*o4{)a=q@V|J&FR{5oB6<(>YN`EY(VWm(Oig-FkHAZelZuRx7?9p1( z4GO_eD@lPEv^685VrlW6DySMg(Py~ zOUAq!1+)CjcFnMG38nwlxobD)7L!{498Y!Vx{>e~4a>R>mob3IBVbewkB!De)sKZs zcfy8VD~?Z!+yQQ0!zM=-9@iTM>s%11#j!{%?-?aE*P#y_HA*VP9X>*6904~c(8!4S zpl{aKiO5xC9osI;-93!w$E*$AK4jd~X4yU*R~-wAkR6#60qmm2M@z36JwnCkkZ~21 z!(&gfTBVs{_%SFF3HE)(A;pXWwA^2*%Pf=`%glLVBWLUeQd=rqdfRo|6l7ys{MqDq z^X8*cRc>9Kvl3(8>1k`0p3kWDs8}ofZ8hmc$B*+yYevV^Wg&4+m5s*T&5V^?6Sy#u z=X4uh6Jur-4P(Jq$LG>oT3&=pc}&hZQXtz7Die73(4^L7A&VK3I=!Z9wn>RHXts+H zY}59uZKcKoas#FOh`v>@Ec~g~TTsoK3)D(WVTXHJGKp(A{E}G|+0sSKjCD&%4@!P3 zinZLX9^}4d$5xyV?V^iVz$NO>F%`sC$90;BL6HDu%cMvLtP;e?%FB{qDYY87O?2QH zuLW4cVG)HM5XGWlR3L$8%wRT~-bGy83vTnGbfRIvE!2RiacFk}aT>a$1tbcGG%sP; z6{;ZWjTRr~s0lbXBodV$BO-+Mh^9;(TXRJQ9$X+P_#!6r8g;qG<5tSv(4i;kw;6nr zNR4T1OG0wd-$J59)POwM(+*FfceHK NO5*AFc<>9mouV*ur)@q$rHn%9*}a17ia z(wK2vF$vNa7tbKg)Fl#2+NFqJaFl2XV-+aA9j%Yt187XD;e~yXFBD-)i7X|&ZyF87 z0F6clq_LD@{E!-)3L3^B@J>M)8t@&cuV$mKh6RLvJ&edXt-H_cG)w1QFjt0@;E}n2 zF#r{UG}b``qxzNzbw((m+w$I!u!u=fG^S;TvH;E_6H#C$!bAnkwfK=eZ=&%eDRelk zAF>pwxBRDiRPkTV{r~KJ>vr2V*68^gKLtnWSJ~f^L|+njn#OKiTWNg8myG4~GMOwd zTB2=EBvK`*IPRG_YyO`XI6uzQ&66DL4UhytfCMQ@c3XROH8u$X8^FfKzVG$zT>Zc2 zkNW@R{5(x9cqnWD=1s{}3ICFl{S&qG^#_t+ZiRDyNb=a*D3TNUF?2%|A7ts9d_?7M z^e$04OQqI2g;b5cvpIo-W3wf9{VeBoW`9TrN*rEk?ot$oFM)qSv5d@EFi2qIHdkam zb>Lz>_?X+2a6@T82z++wj)Pm~u8d@hNgV7W#`ba zL&C!bGZ{Pn$Cbp@4FQ3bNZoUX&X~^P3Ra8hI0{2%G=dl6TU?L}Fh--Y%DGjh&&Uk~ zdIs?!veB+Tlsisuog&Zo6+y(6M1N8JNUwwm5&eZ+=XcxS%9ea)R z(?^?u?0BywI_%TzG4`H|^U%;B_Vq0E@Uz6jFMEuhc*(oWEstw}9&<^9lMlpu)D2qDCzjQk+f zQyxc1o#s`O6ujG69&ONuqVK6Hn=~kAGl%(<#-s*!Mqk9RjHDU0z0)RKXiGZhG^I0X zAiKGP@@5n|;HY^8{7@0Fe0U2#TJ8wZ>9bkoQ!wjpvE=K_M;|&6eEO3)q0O95DD%n} zUS1^FF8RfajS^q_tR3frpU`gRaSoJQ;mE4HQrXv8;0F0#2=Yj#%VHT|u36Lrdr3|3 z@Gfx`RgKKw@JSZ^#BG(@e1Od%)Sn6ApMZ)m=d*A5uWp=(0&zi;z7GQet>-(KARbIP zLodFA$1^|uH@xetB%<@R2c{q*Z^SPA6S^)a?ueM^jLB`2GIc3_6JC!T(jJ3fOAS+| zB%z>EfEg~3GU_Zj-$^6R<38!@lECQ<<81IpC%WIO{~tj&s43o@--oIH@A|WC$^YZ| z*7J==`F|Nd%pi7vC_v8m5B;;j;pqwJO4oZEy=Se*1OK}}|DU6AY-eWA#>XhB|F^mQ zJa_&#*0&zd|1y4Feha1W!}_MZx!ENxy3rkB+&_2y=qdm0XqbLForPok za5S_N=N5aTw_(%v?q~GU>QZxScS^Wf@3Tv0@;Pc{tG7v3Z$`dH>-*J0*Y{o*ckA94cVoTD)vd8$sp(=DCq4>5*1n-3*VMQH zL&6$>4DR{ho;Ibfa1$qn>C^7BepbYBc{-D9Hl=3;G)|29EuwIAbzw19q@|;Nn?kdT zad7K~tzGrvrh|5wl^bqD0BRLK>kgfWpb_n9tz)(S^Y8!npMU?~FMpW@@i%SqH^C_a z2c2UwybP8J)ddD8j=G@yPKKj^lQA5tz1GrTTy8?q?2;iLs+TpLB>>Zp>z%bCkm2k! z2z1ijqQQ3jMwi;34`{{lDxnk-#+MG<>;We99Mn*d$+(O^X<0mLLWmH0tEc@jhZ&6g z!bB3_vcjm(&*Z0jeB3=a0OROnVjXnwnYDU++_CUQCw1mqOSvi?k$_KI`7Nb1yR*@G zwkE===;HWT-@$fgv%G_Y1APZy=Q<#(C2ZyD&o5dUe4vxFcU#u6whcfJEe^CT06fwn zbN=D)KRtybaLQ+oTqmncJs+*MiB!4u?D_UrfBu?K()`MAtP$F}YyAjXW16#ZG@D%J zE?;>JR>R9Z6DFk->?x|?x&7RTq&@kvLQDIY-u*%o!MNO7!Vl^}A>g+uzF zUb_2Sszps#izmOw$#Q#Zd~??rv*0p5=^gYIez^+Up}$f02Anq!dD)+pXRI z>G(4zyffVnv4c(0!E5(2tlz^H>ERd>AJzN(8Tou~8hYb;pFhXZYl6NFt2guAdS8Eq zUyo~!{b!i>?0Oc(^}hd_d_Qof1O*AXuGPiHI_ctc7zedG0PH&nKFt9Fpx=QzEboDn zhi%p0MF*DtooQWnR~D%M2d@=y)r5bAKjt?Hl-B=Q-BIk83BGchGc{|EqFzh zMkog4^-ALzXq6a;pL@-DRoco1S;{x{dvLfaY`~Y&Wq=B-0LnHyExV<+bvz|ZSBvK( zBv~_w=FE7uR9gywb!VZs+5=6O-ClbS+1XduJ|q$guG;yHi`nXv@Jg~l7V1CnFQ^lj zMks^YbT*99hX->?Uj?C?MiK)Q3E#9&mlo0#W3h1X=_10`M-cJRAH8F_pIn~>YTjXB z8%>McvAmw!!(f+*;{&VUeR6H`>ny1rkQr70{K0hW#R3d0vZwv`UOScTidGILuaMWf zbb6~_B<;u=LSg3iTDp9Amju&0o)zseRFL_=W!x)|Kff{iKg&~#*FQW2xBrH`0gCVc z&F#&tujKqM+Z&Jde;)Gvzhc#K0id4{Of$UZORPZmaP?O;jQB!D7SDPRZAbUMCE zH8+thyi9GJwhh6t?_Z&}?N0kD2;%k!UN>@V45S+QSk4)79MM}q1{t%8&r z5QOhyFmiGgmko?xPG$YiOM=FJ0uOXRm7F?*W2WbZIO`YSk-P~ULhq#w9p!p2n@Z`X zRqM&osL--(bd(voA;(T>#m=xh zRdIO73so|KN-K!7LHUEA+$cP4t(;L!w(Ude@U@mic5+Lt)Q-xUrDO+%^>h@vB%_0( zGv9i#dqLzW0y37Ssf0|0h@|eRC-x!Wk4j;tSA-k%uThHUHEhh9meE6njvO%$QH=(i z0LIY_`7GH=RETfz4IrF#PZ;M&9hD-D0m9Jf+rMDY(+<#wFP+Ksn_u9!9rgE^Okyp+ zNeUz~>wv?cf;*{vXLp+a!I4uA?S=Ig)h-y12D%3tXTE@OGu~{4Bbxua`5&CXYwvm% zviPe7j8Vd9Ug7AiV!{g3Ka6O84L;4d3ke%!yqrK`6py*D1uAoL1Uf**uiwCD^-4Grm{{sLYU zF1>|(#FwG#e2iF@LC8-$NkdW>g}28m~U3n|9%iFnAcKqNQ#~ z1AHU!HB4G+8hYN<1tzu1x66}bS-;COZ#HygZ`v~Zrs=9^VEp*MHeVZdm0uO9tKL7R z*Y?$HJhuO23SH6q;n$tUda~DTrZfOJ#cp-?2Q6KdrRt!X>Q*pyv0x`nK7Q-@qu{nh z*$A3&9BT5+;5JAdda`6m5J}G^<%Swr*7PR8;)htlHFV?njy+N_A2~F>aHAnqRd_^a zKvAI5yUAaY1As-!ERjS z>(IGm_K<6IerG8*UtKxFsDqTBT3tf)2zM+PyDyhDC3$0ooTga<6l4P-V``ncXVaTYF8G5LI3jMgCEx zM-g{IF^G}(RDQ)*6iT@p!Z3XKbYpw{8&St5svR?!s5cppOR{FpNJJD*FK|zI#x)vA zz4Sh(nnC5;m*=JOUNy3_pG-X!n+Z3RW!PPg-pQnJ?1b0qfERUt(hZY!Cz$Q66_xr; z^i-^A&J7cXzulmq5@sED+FURpWjs>`~^X1d6XWx{l zluJ-Y;$QwKC{z`L_zh@TF~CInyuj>lE$hqc_#4WXPEfw`Io=eu%XXkeEff)3dXsCl z-9W;jC>&OC6D}qtb7F#Vf2F7p-^?bL>fppCk--cdidFu`my3v6EQwii(dmr@MMTAx zutZV`3%4g{C|JI70uo9B%l|qt0N#_z3&!TzP^~n}bZVZL3<+Udeo{>3vOy5hcf(XYq4L45{XJ+V04U1HyKX+bz*SK*hOB zs?lobU20}2L-JztxWfD?P2v7^v1ejpw#Cvpn8>Zm>QE>7c5j_K(9(}w^2fyUA-%Zk zTUo+5>XI2?nl*|niJzwDW3tF`TKLC?)_*sNn-pR)CZzw8`XfTj!$|E*JOo*CVu!89in5kp9*5PO8MYtECl;3 zm!3oKogva`Ga{FDN!F_xl~Dj_aKb8GSJch|2vhyiGQ6a?D@s33qf0>DG+?kI$D<-o zova(BSSS+iBAijsaXi<6$P*_-Z5;L1=vXl%!Gu4_aS%>n!7*UB0JIV~875w~{uf)o zf3+%+Vy6UtQ}EQMl(`yGMVh`zgl(2MD^`EVxrDk6jF4iHkxoLVkZ6rq;&7x|d7^5# zWzbZ+ZM~kI%`vly(!nm^v*u>5V-{dVvfS)gW(8|n)4H*V%O7F>1=1~8hMn5C=719L|K6#M@_-+K0ZU5fwP zSl{~U(f|L^|NlYy|7Vg+EM%5R>T_cCwe(J*WxuZIedn^W>8wITQhyT)DB&fYkX`B( z!zzTJJ3#DQqhS+@hCw;64zt%^-D2U?ofh0yC$kCO$1Uo*2=RB*)~A?3f`X)k z6>Gs+zq-ZJ4j0*X*)!-Pc|a3hlaAPf79U{%e0Ex~4)~s_lgMjOCl3~b)dz*jDB0|y z0Nc=NQlkQw24^(7$iCwxv(&iu>c2&=u$=Eb*FY^kkpV8~UbnauBQRawYe4OW+Ot!Q zLH23+cV!*ma1TlyD_*Bx)*5p~saqhtk2iGRV+Gw8FPzKEkcNOHEM=+5k6X|n zfC{2H2M`taX22+B9RpPLdlij@DuyxLZI=*juyXw8-~Y!d8%5Ivz?z_o10Ag_9&HeH z4h}SL=0qF4Er8I$>ZX8bHg!WF(N5R~>dZY49rj$X6wIcT_2CA7>xv@HV$koq)1!lhpdbBo;JQ|85Ql+(-C}Zb_Oju*9MF))h0Mhui(payz(r>c z%+7khzE_&eow;$w? z0Sa7GF_^r{NxcvKqyBlpKFl|+PAD6tqB)HW3c&e*3r{$t4I!k&x0rDp4Y6E%W*ZNI9;f-D%FQB zESSw<;Z)(^@P`fDMDok#vZ#XhX~X&_L7PvT^q(#I&$E`%Mb9u3EI)0DSyLwzGv#nZ z*Q_e|QB0|ns7OQ>Y@@S%S`~dXlKvFh1nk*L86XtzcttIf&C|ngYdE|VJizJ;mDEbIHdx7PC5%xh;_m&Y`} z+2`Ol&Ej4ObLB)?B!$-fP0Ot=WGmnF_YMXN)ZT*U8R0l@2)7tPQ0&cC8moS@k<^|- zmo64|PI#4@O!OpQ!c%lNEs7Aj4nLx0F;QaYA|TX z(V}6=WROFP95yt&yB4yA5YmoId^*_wc|fC~4>?C=bb*@&Q)@Q0+;QaILNIpEVi{k; z2wI~%0P`cnPN5LCgDBRs4zHk_<2Jf*ZSxX zUS@?a03)xj4a~Q-=s>@rxJ2(AhP&rM+Xxt6DWDxB)j613uFa z_-yV0w-f_D*AMu7?g5_x{oww7F8#eN{r#0?DL~xTgSg#95svDsu1*AGD>1fuDzu7t zySZ$<-C8!@K3g{4{$cUdAj6*eYLeEn_nwm-xVw=Dv*a>NnwDM5)6gBd;LIrA%Wx$u zV&wb^@h7UR=dA}NT)#QQ+xqZ)@91!Uuu$Q8;@qV?VXDBiF3A#D)6fgZUw0N&t8F4P zGYtI5iwL(kEKL4+>xWj_4VZN+CT7%17yc-%&#BJ*Sv5zYUe%n;WT)-L&e$8a zKUn{?Lt!JTh`K(oix?vyj_sm z;LYK|60K9Q>%Ad#)eXreffWRNs}J002zxfWBwK~q3s*6Q)$QGalLdeUE8^omWO{zx47%%qp8SF5;erk$ul z_0+U$=6i9u9!8}%XBfN${5?86>095O_4ocVSQ7q@FEa#Gon@MJemO^%%>FNf~P-t#{-!CQw#r7?6np`6BNzo+B+j z6!omvLR7|GFhg_ejRWD9kkh{i;YeQ@%B9M6PG7;Ng#`;Qd~reF7rI766Wb3)EXY8- zcG3*&vvKw7Qf>S-ioK9r*pum0Gt6mB0m?9n>Uqjd3Os02-7Af0!d!K+)JAMn3Nzu2 zCmHpIO)T*cW9Ww8AsURm(o+ph7JICZ4+r~Jch?%c-8&grr)O6Gc(4RgmvPMzsJn;G zG?KXKFqk;LikEK0=Z47*HEbYyLxq|f3n#!@5d!92`{XzB{)YU6O8xg}z#F@t-0@D^ zX|KToTzZ$|!7!M*iFE!c7|$k&@;YUAOGPCk6HwKe!|28HCe7H)FYhoe?VnIA4t%|V z+nnP6T<0Tlr?ezDtj?|V&lQ?mG-8Y?uFqRgVe=h0m+&fd2(Ja-tzi0IAKHws;eTxX z;2Q^H;Bi#4`S^AAA?y7bo5|1c=Ax*X{u zygs47jh#yuh9~$8TsHa58~FhML3W=d(lf;0Id`!Z1~<$v5SIRXYwzU1I_rPGC{r7= zUv-m2=AgJhyBYgCO)75?3`WSz^@q$p_Qo5HTt7A3LAH9$7@1eW0}3L8*(@c9EOf6~ z2$|e{;b#C}3;PEY#?UcNi&Xg4as+_4duMy!pY6SUGgtufCJrsZD6m)z^AID1FI|?A zsgtt8q={oe9$m|Yn+yebn)94cDHhC(iI~P?_u3hwCn07*pO&Ei)XX<+klq&U9Zp8PT+tlLPr{#j;RNrk0dGW8c` z>^1^Yov86fsqr;$lDZrR!;jdj@i~=0F{Crfk*b9PPs@O)AKsL#Z+57pdOF$4Kak^K zSpzlK*%asbhI5yC4SqEHKmJIra!95D*}DQO%Oi}XsK3?U!;yQPha+c+wDk5r-hA4ku#IBdU zkRn%GX%Ng1bg8H`8iy@2T<^brR1^O5mOrz&%}>FU0xYcSrpN84nXYq3O(+>%9j_q9MD+<(ukCd7I$fOe{HOCi|SCCLl>#YZ$<~3lFL0pd69P5UIS7f-t zA@OsgAs4W1BaJVRaxv+5YzfN~^@)oq1z;9T_H6%lW=90u*($^KD) z?`(Mt+RPu0T_>DNL}z`NyNrp!bY&Uh7ic41LYj&{P$Ft={%2t90zbG8o$1Zp1@N-N zaWI9lt(EgZ7w3Q8Sl@WIA?1I5w)xfOWB%vI{Lc?E|8qThUdw;mvWSd3I6XNrguZM899@k8KWSKiz;uWj&jQ@X|}jhWd#*HbA!Tifn!m! zJ*)y?vNKh{sN1(xNG#h4b@HM`?K)t*?rb=xUO4Lbi*GSqQbjZL(>0 z*_?Dk*CuBvDdS>1r^PlhEdB|gOdF57smY00{geHl_m58B_74vC9yTX}(h4#nY2t;ZR3gX{^#NgyKI5 znwIz@0&%ooaj(;oahzC8{vgxOs#RLUSqLcs#&?$IkKE7HeuQarz*H20b7eArXpRv> zKy&0^O$Yq;>X$3e9glWk&A)Ue z({HYRfnRsXScL7l!33af_+!l=J*L^wQ(a9(_tG>u5Sk0=v4Y$0K>x@6K6mlAQvKih z^Y!gbN&mP0Z1Yk7_o)AS)c?WH;qlwkGhN{GGPrJUG6g1ziqZMnkQfuAY*%bU(0?}^ zg*QnHI-_&yc@vUbwqrqBVyaL;*P~nr`qPP{r=ZiD#ixQXG?@PUgb~;owXJ1RoG0p;%WT!AMAY*P%=v1z?0_ z|59YNMlRvVLgX<37^?txG3(yQyTStLv9;>5N zAzX^gQaK^#0#ait-*Vcm*6uXbep40-8!A5Xpa8(d7-|J8JEiaAo7v=2`F;YYPY)B% zATOEDo&L~2J3Bn+-_Oos@=TvFwccpvj5$gYVLFF|_@o*AGtM|kWJwunO3O%t?6Qm! zB`**JxL_cIv`+-PtPgN8&87v~Bz^<{;L=TCfWT1iigiWWK`v?;C<+Tru4(;DQ&ONv z3M9wDz0O+G8BjV&gjE*8hl=X#!jGM6+9{~2lOz|ROYU3 zm;#+{8rD=uwf(4tFMwACNCG!jA%j=h%xOi}bX4IxM2^_Hbs~%I&WQCxHWl`X390F} zz#STCr~D|;{juaf&`(ho2Q(2;6XIW4{qOa!HgfXc25J5%|2OBZsD8L_maG{&gPiTKD1HbMCqi=i#6Iu-9HQd-lxir`a>> z+g>PozSlQd^(4&tIgSc2l}CGpcqJDn43e0zTJ^eEomtwWzygdU_X(5J_#Ul5+Ih#x z=!{7bp(G?%ZODeq=6y-S*-`cMhgwamqVB_8IjfUA3O(D zxznFqVtjCzgsZMKcIzX$XfCHK|ng*O)sE1dupL59x{`V_Yc`@TsX=+$1oAKJ&ch8Jm zM!J=oF4D%BlIknY8-4xf$ik3tiFf|%&G%_B#-nWlltw90_P@R)+O>!(`~5^lJgrb2 zi8mloAD7A&sgskm(3*37wU`z8jkTWPu$)(QG;!m0`C+igmY&JY`isexuFNUzp%4qj zNVG_NdMg^psgT-nOUD&KYj+KZ`5tWl(uhLr=a4V%J%`Dn7hk+@L$upV*XDf`V#3xq zH0^#VS7ZiH1nF}fvmy}yX>x9=ViR^rkLk{_821q5oz?Dit3cQWO#NIg$S5X6kctPs zY&HjNB32O8yqxnB>=gSs5!pU`@&SyKB|22;=JGExvu9hT^uE_@< z1-E*_aTA{K92u$m)_3p;VSqph{V>GfY7vt)@7Obxd0fBqkDTcKVn(mAOP>fteV4Va(KeZ@^A%)YUo^*5heA z2avtjY12qhH#-*{WVxCKQZRhR<_$C4EVM$#Pvj0y87|(kkpcyp{~a z=@hveMuRXn*B(_lCf@9!8I|wkWxQ}(tFnKU_aSiDM03O=eLGo5gy^;FT))_v&G)>h z9vK)_4qw`CGNturWr$1A`K$iiQra$5=bze}Qnumx9M@kW5#RoBc|efFBDbF!@W%8c zUVk08&+8}o$+<>IZD^HumJKD!a8%+c-t=P|?q)P;>e}`z>|6BZ&N!Q+^4}r04>p}$#(yJRBFwP1)7$RQ^7(>f{$_BDVq^jFd&)+iki1_4FD=$1j592q z#zT{TfPbFHc{*0kYEHpZ ztp^s%d(5L3twLD;kjAp=aY^bBa007KyYDy#4?Q9pEXuqH@u%i{WW#bknf}S53yMSB z9=y1UN~ii@QrvrA4;#GUR?QGH-y=_Q=8yjCMESN)aI~IzXQb|$S7pIA~5Dkch!l-GLDQ= z_9sXMX0vZu?kE??&D>3PLYwSdw+m$V{=%fC)w7Y4*Neb%K+OssF7L&Si*%!yh80mc z0P~ItA>PIxZdx0(Jwm%-!nT%`r5ZP$g_$)Fv);RL4dYwJeHY8}0H4$6!Pe?qHg$3`bCkP*K zV%u*j8;X{rE=DP|Vhq^=QwI2@wMVvebYD*{ie(OOd-};PEt_<;9nV1$h^(H`O)oFQ zK8J@xa)2zNV7>k*A9~BI+&Tp>R^W48SZi|eg-xP-%~i{ZpsMdJNhSG7U+srKV&xR! zH-D4QN-ikaiim;3QYDbD-XSMn!0JsTrPEk?&6F+#{RFr6HlRv5@fUjQ4 z#binqw?VwISu97LX8s-R_F%5YKK<1_b+tgX-@>H2=x_k3<6v&JO}lD!cNM5Px3w&< zW-(4*>w{l}CN0ezh`YygNI3X_X8qSGEeyC1DQg}G<*^~D#gBh-<9957IUiTt^fFO1F^DX~AY@%2qs0K5bN8!`jGaQ}V*7deDCIBO zlvoQX5{tLme1z13VL{>EYF8z`tYQs#LpEXUX_DtgY=Y|aoMir?2*W3_7Bu>kG`8JT z3$NE_us3e+0)(gg>-w%%eBaE%HzPx}sY>I3`7eXm!yBK&4X#|DVR_Smb6;C5sMT~C zsh2F?4$i+SvUx_hM%K0^T|4YG>t-?mV!r+6N#Jad7jUj}ehb(lgZ!eJWZ|>d^&W7o z(U3)u7+g?8c-~-=wVCBBq+?ByqJv6Zf))*eYR0V`O-unbNJm9w{in!W{esoYWJ&_$8A^AtBNhk?q zKm1iOQ5S)C(I?eL*j2w!1-yz(j{8-M53pS(oKcLuxP z6!w+WFM6gkPzUc$PN*-C->L}B4yAH`v$SW?>~^!C9uBymYUdcw6? zrK3Qf1DBgdHFlaKwV1tp1)o_KGsOpZ6Y88c3<*n;cTsSGCawHKmX&ug9Ac&NS61;K zbJ9tR{Dzzp-i(4X|B)w?@qfz|FdIjR&m z9>le>ED zy$3U6>5`WpNv0^?kK%m52e#K%?c_?9J1hxs%Gb?nm>W&$@feKjYejA&3mg54ac4 z2(Ck1DP?oR&kZV8uYgH8`1bbF*WtVq;T+Q)Kx`kGC%&GGz)FVaK~#2)@mev+t_f|B z%K(_-6s&Tv-7*_b&y_cDrb+TXoIOFqKAMWl`u){kc=s!4^~q~AQbAtjxr}$DCryoG z8frE3s=<=D19WkKZqL_Klj^k?lGWVDr=STR}gi zgJE%<*2bfmJFasit~~-&_iKgoUh9{rz&qe=9L*bs#Q~e9XEWb>%(IToN;TdRHQz=k zjRD$p;7{v*yI@~@Mr*Z@<#Uw57%F0rs6b>T0TT7IW&~?w`2+${fNWY6F7{+!G0$(0 z$%x>lQqJeH&MUJ}LRqNanW*v1^TN#T@$~cY>xQ^g)5F+Y_?|DEO}Ofj5&g*VaZ=Y# z!gzPWI4EIx{9&eYL1pMqXL5W$Jf$=rXX_`V_b;@K>^EWLeGhI6>wzra8&Mbpbu1?S_gF z6EZ_3Fklz&BBlM!HWSO|%z|JhJf>xI4)dr6#$1Nb8Ny{`TG=iFe_{?oFMw@w z=s9FSjfGr7iJ8XE_IHPkEY3e{&1dngoCJiCKbbz!uxx$SB z078&Ld*5uW!&2x?c5B*@LQ*uheLH##-80ge4A>@1dGHR3o|-SwN08qAzy3DtLYi*N^k zCE)M4aj|ra7tHra#TduZ4Af}{o5^)uU#S9{EnWnh+Q&hSjodx zbt~lGd3@rhM0{ow_0HrEMIIjT9G*76x)mO%g~^cYHBNf<&04B#&7VHsyrJoImIQJ9 z@yvmoVqP@xbBrOJX)7;ZVW;90c`@+FY$*tT_G5NEs6cA;scUc2BfO`rW{^qKopz{c zSC_+1ZMwuH(@l0Z60bGi(8WOgB1qG|yXN4d`fB)k07{6Ga5VJg_+q~>3s_Dp-4-n$ zyzS#lXm{SdHPEP!JsPJvgye7^sV>&heQG9L|2;z%@HwURS1`U_;@DKS7-p%NVvl9=>Al96t*$55!iLRT zR4g1@)d?5VutQTQPJ_YvTO}k>IAV3!u^qEFfv|3TiqDi69}}>bz0>k~a(-P`@ck<+jOUK|)7$0j%tc<*Ik)iLq&D{I~Q zpxPJ>tU~*!L&UqG<`qzs0;h70?~-zMNc7bJfT>E_n49$T4Xz;pQ5i2GIeupi!WqdS zaH_6|P^@=m6Mc5)oR?JctT>!DrTaI=sYergh?c%aE49IBn>W(Rk#YzsM;^qA zqh8y057exHP82NtUew5-Gc*7FA!6#MN=q?oGaWP=6%k$!eST=&xwTjHnc3G!J)*X7 zu(<21Q^8^U;2dHQb{Bvi*Ku*v?i>7RN`3YrwlVu~W_@yOrP#D0TYOZ%gz8>e_4jlV zmN^YGN)#*~2}b4X&UKA~l*a)qw|rR4S~cIKk1$xOY1n0Zl#4s5FvHFtGr@vtkIL2# zCPwkyT_s6)gubonNVv-9vuXatA=R>c&2W`E)<5S?uU4k{c`M2csz;~!g$)XM%?t|l z{(O-3s~$S;d2JBpSO7hl-FJId&VRztC3;&~sAk3Omhv4>=OzB`w+)$Z7!|GPrt}k` zLB{z#Cu8H|bK4Kvh{dVOiD>x(cC%DXc9xvBH+6||PIYY7y7r|-1qN@iC>env^2qA^ELg?SzKC1fY8O)t z3%_sBm;-0=cItj2)Ba$5i{p+^JA`BqGS<_YqGaCwgmUrY(1jEF$1=Yd<{iidBCzr99^{F!d4CQ*6@Fi}6S8+#LjeRg;AsB!qc zcQ&G2;i@I)bBSv0t01oR*v~?e%$wK#PQP>8ZyYm-rPe}5-nm(Clcz)C`Slo^5&&Uct7Psi q-KdZEgz`R414M-a{}76_0~r Date: Wed, 3 Nov 2021 14:22:23 -0500 Subject: [PATCH 028/163] add file_ead class --- lib/datura/file_types/file_ead.rb | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 lib/datura/file_types/file_ead.rb diff --git a/lib/datura/file_types/file_ead.rb b/lib/datura/file_types/file_ead.rb new file mode 100644 index 000000000..110b9d5c0 --- /dev/null +++ b/lib/datura/file_types/file_ead.rb @@ -0,0 +1,45 @@ +require_relative "../helpers.rb" +require_relative "../file_type.rb" +require_relative "../solr_poster.rb" +require "rest-client" + +class FileEad < FileType + # TODO we could include the tei_to_es and other modules directly here + # as a mixin, though then we'll need to namespace them or perish + attr_reader :es_req + + + def initialize(file_location, options) + super(file_location, options) + @script_html = File.join(options["collection_dir"], options["ead_html_xsl"]) # There needs to be an xsl file to transform into html + # I don't think we need solr at this point) + # @script_solr = File.join(options["collection_dir"], options["tei_solr_xsl"]) + end + + def subdoc_xpaths + # match subdocs against classes + return { + "/EAD" => EadToEs, + # "//dsc/c01" => EadToEsItems, + } + end + + # if there should not be any html transformation taking place + # then leave this method empty but uncommented to override default behavior + + # if you would like to use the default transformation behavior + # then comment or remove both of the following methods! + + # def transform_es + # end + + # def transform_html + # end + + def transform_iiif + raise "EAD to IIIF is not yet generalized, please override on a per project basis" + end + + # def transform_solr + # end +end From 1f4361a7434e9b0635a9f736bf4ebaeea0b28369 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 3 Nov 2021 14:23:31 -0500 Subject: [PATCH 029/163] add EadToES class --- lib/datura/to_es/ead_to_es.rb | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 lib/datura/to_es/ead_to_es.rb diff --git a/lib/datura/to_es/ead_to_es.rb b/lib/datura/to_es/ead_to_es.rb new file mode 100644 index 000000000..318053df7 --- /dev/null +++ b/lib/datura/to_es/ead_to_es.rb @@ -0,0 +1,33 @@ +require_relative "xml_to_es.rb" +require_relative "ead_to_es/fields.rb" +require_relative "ead_to_es/request.rb" +require_relative "ead_to_es/xpaths.rb" + +########################################################### +# NOTE: DO NOT EDIT EAD_TO_ES FILES IN SCRIPTS DIRECTORY # +########################################################### + +# (unless you are a CDRH dev and then you may do so very cautiously) +# this file provides defaults for ALL of the collections included +# in the API and changing it could alter dozens of sites unexpectedly! +# PLEASE RUN LOADS OF TESTS AFTER A CHANGE BEFORE PUSHING TO PRODUCTION + +# HOW DO I CHANGE XPATHS? +# You may add or modify xpaths in each collection's ead_to_es.rb file +# located in the collections//scripts directory + +# HOW DO I CHANGE FIELD CONTENT? +# You may need to alter an xpath, but otherwise you may also +# copy paste the field defined in ead_to_es/fields.rb and change +# it as needed. If you are dealing with something particularly complex +# you may need to consult with a CDRH dev for help + +# HOW DO I CUSTOMIZE THE FIELDS BEING SENT TO ELASTICSEARCH? +# You will need to look in the ead_to_es/request.rb file, which has +# collections of fields being sent to elasticsearch +# you can override individual chunks of fields in your collection + +class EadToEs < XmlToEs + # Override XmlToEs methods that need to be customized for EAD here + # rather than in one of the files in ead_to_es/ +end From 52b38a0794a5980921b52b7570491ef60ae6b2c8 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 3 Nov 2021 14:26:21 -0500 Subject: [PATCH 030/163] add files for EadToEs --- lib/datura/to_es/ead_to_es/fields.rb | 250 ++++++++++++++++++++++++++ lib/datura/to_es/ead_to_es/request.rb | 7 + lib/datura/to_es/ead_to_es/xpaths.rb | 38 ++++ 3 files changed, 295 insertions(+) create mode 100644 lib/datura/to_es/ead_to_es/fields.rb create mode 100644 lib/datura/to_es/ead_to_es/request.rb create mode 100644 lib/datura/to_es/ead_to_es/xpaths.rb diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb new file mode 100644 index 000000000..9c8f35e6c --- /dev/null +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -0,0 +1,250 @@ +class EadToEs < XmlToEs + # Note to add custom fields, use "assemble_collection_specific" from request.rb + # and be sure to either use the _d, _i, _k, or _t to use the correct field type + + ########## + # FIELDS # + ########## + + def id + get_text(@xpaths["identifer"]) + end + + # def id_dc + # # TODO use api path from config or something? + # "https://cdrhapi.unl.edu/doc/#{@id}" + # end + + # def annotations_text + # # TODO what should default behavior be? + # end + + # def category + # category = get_text(@xpaths["category"]) + # return category.length > 0 ? CommonXml.normalize_space(category) : "none" + # end + + # note this does not sort the creators + def creator + creators = get_list(@xpaths["creators"]) + return creators.map { |creator| { "name" => CommonXml.normalize_space(creator) } } + end + + # returns ; delineated string of alphabetized creators + def creator_sort + return get_text(@xpaths["creators"]) + end + + def collection + @options["collection"] + end + + def collection_desc + @options["collection_desc"] || @options["collection"] + end + + # def contributor + # contribs = [] + # @xpaths["contributors"].each do |xpath| + # eles = @xml.xpath(xpath) + # eles.each do |ele| + # contribs << { + # "id" => ele["id"], + # "name" => CommonXml.normalize_space(ele.text), + # "role" => CommonXml.normalize_space(ele["role"]) + # } + # end + # end + # contribs.uniq + # end + + def data_type + "ead" + end + + def date(before=true) + datestr = get_text(@xpaths["date"]) + return CommonXml.date_standardize(datestr, before) + end + + # def date_display + # get_text(@xpaths["date_display"]) + # end + + def date_not_after + date(false) + end + + def date_not_before + date(true) + end + + def description + get_text(@xpaths["description"]) + end + + def extent + get_text(@xpaths["extent"]) + end + + def format + matched_format = nil + # iterate through all the formats until the first one matches + @xpaths["formats"].each do |type, xpath| + text = get_text(xpath) + matched_format = type if text && text.length > 0 + end + matched_format + end + + # def image_id + # # Note: don't pull full path because will be pulled by IIIF + # images = get_list(@xpaths["image_id"]) + # images[0] if images + # end + + # def keywords + # get_list(@xpaths["keywords"]) + # end + + # def language + # get_text(@xpaths["language"]) + # end + + # def languages + # get_list(@xpaths["languages"]) + # end + + def medium + # Default behavior is the same as "format" method + format + end + + # def person + # # TODO will need some examples of how this will work + # # and put in the xpaths above, also for attributes, etc + # # should contain name, id, and role + # eles = @xml.xpath(@xpaths["person"]) + # people = eles.map do |p| + # { + # "id" => "", + # "name" => CommonXml.normalize_space(p.text), + # "role" => CommonXml.normalize_space(p["role"]) + # } + # end + # return people + # end + + # def people + # @json["person"].map { |p| CommonXml.normalize_space(p["name"]) } + # end + + # def places + # return get_list(@xpaths["places"]) + # end + + # def publisher + # get_text(@xpaths["publisher"]) + # end + + # def recipient + # eles = @xml.xpath(@xpaths["recipient"]) + # people = eles.map do |p| + # { + # "id" => "", + # "name" => CommonXml.normalize_space(p.text), + # "role" => "recipient" + # } + # end + # return people + # end + + # def rights + # # Note: override by collection as needed + # get_text(@xpaths["rights"]) + # end + + # def rights_holder + # get_text(@xpaths["rights_holder"]) + # end + + def rights_uri + # by default collections have no uri associated with them + # copy this method into collection specific tei_to_es.rb + # to return specific string or xpath as required + end + + # def source + # get_text(@xpaths["source"]) + # end + + # def subjects + # get_list(@xpaths["subjects"]) + # end + + # def subcategory + # subcategory = get_text(@xpaths["subcategory"]) + # subcategory.length > 0 ? subcategory : "none" + # end + + def text + # handling separate fields in array + # means no worrying about handling spacing between words + text = [] + @xpaths.keys.each do [xpath] + body = get_text(@xpaths[xpath]) + text << body + end + # TODO: do we need to preserve tags like in text? if so, turn get_text to true + # text << CommonXml.convert_tags_in_string(body) + # text += text_additional + # return CommonXml.normalize_space(text.join(" ")) + end + + # def text_additional + # # Note: Override this per collection if you need additional + # # searchable fields or information for collections + # # just make sure you return an array at the end! + + # text = [] + # text << title + # end + + def title + get_text(@xpaths["title"]) + end + + def title_sort + t = title + CommonXml.normalize_name(t) + end + + def type + get_text(@xpaths["type"]) + end + + # def topics + # get_list(@xpaths["topic"]) + # end + + # def uri + # # override per collection + # # should point at the live website view of resource + # end + + # def uri_data + # base = @options["data_base"] + # subpath = "data/#{@options["collection"]}/source/tei" + # return "#{base}/#{subpath}/#{@id}.xml" + # end + + # def uri_html + # base = @options["data_base"] + # subpath = "data/#{@options["collection"]}/output/#{@options["environment"]}/html" + # return "#{base}/#{subpath}/#{@id}.html" + # end + + def works + # TODO need to create a list of items, maybe an array of ids + end +end diff --git a/lib/datura/to_es/ead_to_es/request.rb b/lib/datura/to_es/ead_to_es/request.rb new file mode 100644 index 000000000..e0e0c9899 --- /dev/null +++ b/lib/datura/to_es/ead_to_es/request.rb @@ -0,0 +1,7 @@ +class EadToEs < XmlToEs + + # please refer to generic xml to es request file, request.rb + # and override methods specific to TEI transformation here + # project specific overrides should go in the COLLECTION's overrides! + +end diff --git a/lib/datura/to_es/ead_to_es/xpaths.rb b/lib/datura/to_es/ead_to_es/xpaths.rb new file mode 100644 index 000000000..10a1147b5 --- /dev/null +++ b/lib/datura/to_es/ead_to_es/xpaths.rb @@ -0,0 +1,38 @@ +class EadToEs < XmlToEs + # These are the default xpaths that are used for collections + # if you require a different xpath, please override the xpath in + # the specific collection's TeiToEs file or create a new method + # in that file which returns a different value + def xpaths_list + { + # "abstract" => "/ead/archdesc/did/abstract" + # "category" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='category'][1]/term", + # "contributors" => [ + # "/TEI/teiHeader/revisionDesc/change/name", + # "/TEI/teiHeader/fileDesc/titleStmt/editor" + # ], + "creators" => ["/ead/archdesc/did/origination/persname", "/ead/eadheader/filedesc/titlestmt/creator"] + "date" => "/ead/eadheader/filedesc/publicationstmt/date", + "description" => "/ead/archdesc/scopecontent/p", + # a non normalized date taken directly from the source material ("Dec 9", "Winter 1889") + # "date_display" => "/TEI/teiHeader/fileDesc/sourceDesc/bibl/date", + "formats" => "/ead/archdesc/did/physdesc/genreform", + # "image_id" => "/TEI/text//pb/@facs", + # "keywords" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='keywords']/term", + # note: language is global attribute xml:lang + "identifier" => "/ead/archdesc/did/unitid", + "language" => "/ead/eadheader/profiledesc/langusage/language", + # "languages" => "//body/div1/@lang", + # "person" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='people']/term", + # "places" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='places']/term", + "publisher" => "/ead/eadheader/filedesc/publicationstmt/publisher", + # "recipient" => "/TEI/teiHeader/profileDesc/particDesc/person[@role='recipient']/persName", + "repository_contact" => "/ead/archdesc/did/repository/addresses", + "rights" => "/ead/archdesc/descgrp/accessrestrict/p", + "rights_holder" => "ead/archdesc/did/repository/corpname", + "source" => "/ead/archdesc/descgrp/prefercite/p", + "subjects" => "/ead/archdesc/controlaccess/*[not(name()="head")]"], + # "subcategory" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='subcategory'][1]/term", + "titles" => "ead/archdesc/did/unittitle", + "text" => "/ead/eadheader/filedesc/titlestmt/*", + } From 12b089729f86a46329fce8cedbf3706a74ef6357 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 3 Nov 2021 14:28:44 -0500 Subject: [PATCH 031/163] add EadToEsItems class and associated files --- lib/datura/to_es/ead_to_es_items.rb | 36 +++ lib/datura/to_es/ead_to_es_items/fields.rb | 244 ++++++++++++++++++++ lib/datura/to_es/ead_to_es_items/request.rb | 7 + lib/datura/to_es/ead_to_es_items/xpaths.rb | 45 ++++ 4 files changed, 332 insertions(+) create mode 100644 lib/datura/to_es/ead_to_es_items.rb create mode 100644 lib/datura/to_es/ead_to_es_items/fields.rb create mode 100644 lib/datura/to_es/ead_to_es_items/request.rb create mode 100644 lib/datura/to_es/ead_to_es_items/xpaths.rb diff --git a/lib/datura/to_es/ead_to_es_items.rb b/lib/datura/to_es/ead_to_es_items.rb new file mode 100644 index 000000000..ff6ec3e9c --- /dev/null +++ b/lib/datura/to_es/ead_to_es_items.rb @@ -0,0 +1,36 @@ +require_relative "xml_to_es.rb" +require_relative "ead_to_es_items/fields.rb" +require_relative "ead_to_es_items/request.rb" +require_relative "ead_to_es_items/xpaths.rb" + +########################################################### +# NOTE: DO NOT EDIT EAD_TO_ES FILES IN SCRIPTS DIRECTORY # +########################################################### + +# (unless you are a CDRH dev and then you may do so very cautiously) +# this file provides defaults for ALL of the collections included +# in the API and changing it could alter dozens of sites unexpectedly! +# PLEASE RUN LOADS OF TESTS AFTER A CHANGE BEFORE PUSHING TO PRODUCTION + +# HOW DO I CHANGE XPATHS? +# You may add or modify xpaths in each collection's ead_to_es.rb file +# located in the collections//scripts directory + +# HOW DO I CHANGE FIELD CONTENT? +# You may need to alter an xpath, but otherwise you may also +# copy paste the field defined in ead_to_es/fields.rb and change +# it as needed. If you are dealing with something particularly complex +# you may need to consult with a CDRH dev for help + +# HOW DO I CUSTOMIZE THE FIELDS BEING SENT TO ELASTICSEARCH? +# You will need to look in the ead_to_es/request.rb file, which has +# collections of fields being sent to elasticsearch +# you can override individual chunks of fields in your collection + +class EadToEs < XmlToEs + # Override XmlToEs methods that need to be customized for EAD here + # rather than in one of the files in ead_to_es/ + def get_id + @id + end +end diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb new file mode 100644 index 000000000..b5ca56d79 --- /dev/null +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -0,0 +1,244 @@ +class EadToEsItems < XmlToEs + # Note to add custom fields, use "assemble_collection_specific" from request.rb + # and be sure to either use the _d, _i, _k, or _t to use the correct field type + + ########## + # FIELDS # + ########## + + def id + get_text(@xpaths["identifer"]) + end + + # def id_dc + # # TODO use api path from config or something? + # "https://cdrhapi.unl.edu/doc/#{@id}" + # end + + # def annotations_text + # # TODO what should default behavior be? + # end + + # def category + # category = get_text(@xpaths["category"]) + # return category.length > 0 ? CommonXml.normalize_space(category) : "none" + # end + + # note this does not sort the creators + def creator + creators = get_list(@xpaths["creators"]) + return creators.map { |creator| { "name" => CommonXml.normalize_space(creator) } } + end + + # returns ; delineated string of alphabetized creators + def creator_sort + return get_text(@xpaths["creators"]) + end + + def collection + "manuscripts" + end + + def collection_desc + @options["collection_desc"] || @options["collection"] + end + + # def contributor + # contribs = [] + # @xpaths["contributors"].each do |xpath| + # eles = @xml.xpath(xpath) + # eles.each do |ele| + # contribs << { + # "id" => ele["id"], + # "name" => CommonXml.normalize_space(ele.text), + # "role" => CommonXml.normalize_space(ele["role"]) + # } + # end + # end + # contribs.uniq + # end + + def data_type + "ead" + end + + def date(before=true) + datestr = get_text(@xpaths["date"]) + return CommonXml.date_standardize(datestr, before) + end + + def date_display + get_text(@xpaths["date_display"]) + end + + def date_not_after + date(false) + end + + def date_not_before + date(true) + end + + def description + # Note: override per collection as needed + end + + def format + matched_format = nil + # iterate through all the formats until the first one matches + @xpaths["formats"].each do |type, xpath| + text = get_text(xpath) + matched_format = type if text && text.length > 0 + end + matched_format + end + + def image_id + # Note: don't pull full path because will be pulled by IIIF + images = get_list(@xpaths["image_id"]) + images[0] if images + end + + def keywords + get_list(@xpaths["keywords"]) + end + + def language + get_text(@xpaths["language"]) + end + + def languages + get_list(@xpaths["languages"]) + end + + def medium + # Default behavior is the same as "format" method + format + end + + def person + # TODO will need some examples of how this will work + # and put in the xpaths above, also for attributes, etc + # should contain name, id, and role + eles = @xml.xpath(@xpaths["person"]) + people = eles.map do |p| + { + "id" => "", + "name" => CommonXml.normalize_space(p.text), + "role" => CommonXml.normalize_space(p["role"]) + } + end + return people + end + + def people + @json["person"].map { |p| CommonXml.normalize_space(p["name"]) } + end + + def places + return get_list(@xpaths["places"]) + end + + def publisher + get_text(@xpaths["publisher"]) + end + + def recipient + eles = @xml.xpath(@xpaths["recipient"]) + people = eles.map do |p| + { + "id" => "", + "name" => CommonXml.normalize_space(p.text), + "role" => "recipient" + } + end + return people + end + + def rights + # Note: override by collection as needed + "All Rights Reserved" + end + + def rights_holder + get_text(@xpaths["rights_holder"]) + end + + def rights_uri + # by default collections have no uri associated with them + # copy this method into collection specific tei_to_es.rb + # to return specific string or xpath as required + end + + def source + get_text(@xpaths["source"]) + end + + def subjects + # TODO default behavior? + end + + def subcategory + subcategory = get_text(@xpaths["subcategory"]) + subcategory.length > 0 ? subcategory : "none" + end + + def text + # handling separate fields in array + # means no worrying about handling spacing between words + text = [] + body = get_text(@xpaths["text"], false) + text << body + # TODO: do we need to preserve tags like in text? if so, turn get_text to true + # text << CommonXml.convert_tags_in_string(body) + text += text_additional + return CommonXml.normalize_space(text.join(" ")) + end + + def text_additional + # Note: Override this per collection if you need additional + # searchable fields or information for collections + # just make sure you return an array at the end! + + text = [] + text << title + end + + def title + title = get_text(@xpaths["titles"]["main"]) + if title.empty? + title = get_text(@xpaths["titles"]["alt"]) + end + return title + end + + def title_sort + t = title + CommonXml.normalize_name(t) + end + + def topics + get_list(@xpaths["topic"]) + end + + def uri + # override per collection + # should point at the live website view of resource + end + + def uri_data + base = @options["data_base"] + subpath = "data/#{@options["collection"]}/source/tei" + return "#{base}/#{subpath}/#{@id}.xml" + end + + def uri_html + base = @options["data_base"] + subpath = "data/#{@options["collection"]}/output/#{@options["environment"]}/html" + return "#{base}/#{subpath}/#{@id}.html" + end + + def works + # TODO figure out how this behavior should look + end +end diff --git a/lib/datura/to_es/ead_to_es_items/request.rb b/lib/datura/to_es/ead_to_es_items/request.rb new file mode 100644 index 000000000..27f14d072 --- /dev/null +++ b/lib/datura/to_es/ead_to_es_items/request.rb @@ -0,0 +1,7 @@ +class EadToEsItems < XmlToEs + + # please refer to generic xml to es request file, request.rb + # and override methods specific to TEI transformation here + # project specific overrides should go in the COLLECTION's overrides! + +end diff --git a/lib/datura/to_es/ead_to_es_items/xpaths.rb b/lib/datura/to_es/ead_to_es_items/xpaths.rb new file mode 100644 index 000000000..bab2fa935 --- /dev/null +++ b/lib/datura/to_es/ead_to_es_items/xpaths.rb @@ -0,0 +1,45 @@ +class EadToEsItems < XmlToEs + # These are the default xpaths that are used for collections + # if you require a different xpath, please override the xpath in + # the specific collection's TeiToEs file or create a new method + # in that file which returns a different value + def xpaths_list + { + "abstract" => "/ead/archdesc/did/abstract", + # "category" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='category'][1]/term", + # "contributors" => [ + # "/TEI/teiHeader/revisionDesc/change/name", + # "/TEI/teiHeader/fileDesc/titleStmt/editor" + # ], + "creators" => ["/ead/archdesc/did/origination/persname", "/ead/eadheader/filedesc/titlestmt/creator"], + "date" => "/ead/archdesc/dsc/c01/did/unitdate", + "description" => "/ead/archdesc/dsc/c01/scopecontent/p", + # a non normalized date taken directly from the source material ("Dec 9", "Winter 1889") + # "date_display" => "/TEI/teiHeader/fileDesc/sourceDesc/bibl/date", + "extent" => "/ead/archdesc/dsc/c01/did/physdesc/extent", + "format" => "/ead/archdesc/dsc/c01/did/physdesc/physfacet", + "image_url" => "/ead/archdesc/dsc/c01/dao/@href", + # "image_id" => "/TEI/text//pb/@facs", + # "keywords" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='keywords']/term", + # note: language is global attribute xml:lang + "identifier" => "/ead/archdesc/dsc/c01/did/unitid[@type='WWA']", + # "language" => "(//body/div1/@lang)[1]", + # "languages" => "//body/div1/@lang", + # "person" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='people']/term", + # "places" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='places']/term", + # "publisher" => "/TEI/teiHeader/fileDesc/sourceDesc/bibl[1]/publisher[1]", + # "recipient" => "/TEI/teiHeader/profileDesc/particDesc/person[@role='recipient']/persName", + "repository_id" => "/ead/archdes/dsc/c01/did/unitid[@type='repository']", + # "rights" => "/ead/archdesc/descgrp/accessrestrict/p", + # "rights_holder" => "/ead/archdesc/descgrp/accessrestrict/p", + # "source" => "/ead/archdesc/descgrp/prefercite/p", + # "subjects" => ["/ead/archdesc/controlaccess/[everything after head;persname, subject]"], + # "subcategory" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='subcategory'][1]/term", + # "text" => "//text", + "title" => "ead/archdesc/dsc/c01/did/unittitle", + # "topic" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='topic']/term", + "type" => "/ead/archdesc/dsc/c01/did/physdesc/genreform", + # "works" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='works']/term", + }.merge(override_xpaths) + end +end From 21a55dc8457c35ca2bb63493358800d83f9dc7c6 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 3 Nov 2021 14:29:30 -0500 Subject: [PATCH 032/163] add xsl file for ead (not functional yet) --- lib/xslt/ead_to_html/ead_to_html.xsl | 60 ++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 lib/xslt/ead_to_html/ead_to_html.xsl diff --git a/lib/xslt/ead_to_html/ead_to_html.xsl b/lib/xslt/ead_to_html/ead_to_html.xsl new file mode 100644 index 000000000..d0aa8f923 --- /dev/null +++ b/lib/xslt/ead_to_html/ead_to_html.xsl @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + +production + + + + + + + + + + + + + + + + + + From c39668c4adfebaa9404535c0c0bad51f12a435af Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 4 Nov 2021 10:42:27 -0500 Subject: [PATCH 033/163] remove gem doc that is messing things up --- datura-0.1.4.gem | Bin 91136 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 datura-0.1.4.gem diff --git a/datura-0.1.4.gem b/datura-0.1.4.gem deleted file mode 100644 index d1216675f068ebe213245adbfc7813a7c50f5b6e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 91136 zcmeFXMQkNN?2nW@ct)W@b(YI?PNj%*@P5hnbm~p_A@#hqK?z9%l5{N_&~n z&S8CVm0V@nE|vT%mmMuVOf5`3Oqr~G!Txs<%YTT2g9Gfp@&C|&Rucy0*soQ@{`e9>f>F_@b{>%9PP5yuT_CK8aFWdi5 zosz`DfsOD5nLh3#lbD;&S8xNkAF}FET@3O$@H<7}pINflSijr_4K``bya)Bz7 zHLQMYd_C`~wkp0xp0p1)mr6?P`IjdABT5voWWj4>_7K!XSv?W7;OdGXm)+A!imsaI z#IBB$GZGncbF!Z;fn9(JpUhYNi?2DdfzJ^$eivJrvZIv7gdt^4ZnWLC1dN8chLA(d zE=n;6E&o${>M~xei1fE6loX=89NfD!$Mg zR=A|3*?AsTSMFWdx!j@qF{q~L4a)W=f8Z1Zs9t`B>3}|a&(MBUUxQ)R)(;kMIZ(rm9X!Wmaa zvtc-iH|N&J-U7bQt%Ns@LQBi|aYPns9PI-2Ey~At>JM$~uQJ-oS8N%D-<;h2AJ3@{ zW4AAK#E#{!XUkdvIFSStk+4bC_rP!1uuoK+K+2o&0d~|Mn(V6xu(4v{-r9DV(`Zb~T|aa+SU&ZdFavby(-qi>TU7UMoNj<0WL3Y#?lPUn?D*dXef59fEAVY5ONj`vOz)%Rc5htCA1GXWVY;?b2Xo=7v#%$jlhBxf`0~2l23X?=V60=eldgX ztnADq$wmupSZ-|V+pb5h-L6G;+Lfaf(yX$|)_$=3+c~#-z;$UbtGC~qja=);NwCxH zxnjq4%%?3*(CyAkAo9-_!r)&Bgh_?vezWqi zN7W;jKet28C8*09%zr*^MB6hJtAu7RxRMa}IN5ASf>hsZ6l{5}McW{2AqiVMOohPm zf<@oKYR3oAuv>{7>Jff*&NWiz>+5G(Y`8R6L*6L`QzbQcA~fdC-GqIIsu!W<2O)g)pXR#H=VdYV!9oWxKZIJH?X?vA z{dvCK?_F@S?XNrD*8aL7)aKdNc03-xU7jvH8fb6-8|a|UUGHSz=qPS3&IqcpuPk6O zB`ERAsk@!@>$@CMiFjwEZ;5ux@iP|anxCK+YAfj(j=V`cfqBD7c9atS?htb2-dC@@ zY=IG+{vMeNbew#3cBmy0{MJ8C<@)?(pOF9ly?q`y;U>5}8DMrE`19_yfW`N#W3EC- zh|GxJ_v3p$pj^Q@i?hyC>b>>F=aQ9{Lp`TL-drYGkEROcms(pTn z$3Iy{9%ybsw0}LRBNapJ7x8x@WSHyMpQgCh)G1Uw-F-O%ueIBE$N7skb}a#ntZipN zZvwx~8;rm;t02vhMf{vN1HHA4pr_{3d%&E%93{*_d{)Z*F|9R=sU@c8-IljwF&_?V zi|rrMb3SQ3+I^Pg&-Ce%tLsLX*;?lc@`qQ9mKMF?d``F@`a7=g7)En)R1sHAv)=u} znN<=$srB1;b;(e9fIJDvrOHgqUpbHqO3&qQ-knt?({mDo9fu z{Fw8ogTi)>8b3B@*s5f`vj6>DQ?=Vr?ga6X>0ijoOs(I5CZCe9g6Cc^ zSd%{UM2Uh2nBmyGq^z$#)c1=2yK;clTEzKbFJ6-&ZVwmH?yTrf^kHLf&p(lAYjtBZ z{UAteEYGJ{Qu0G-Z~oN3Yq*me151HqjU>jl5i8$t4dC+Tu+9~9$YKKNzpg|EHmEk- zYI+_g%8%%d^#})+ey$G`QEo^chCX~A!58%#Xu%is;ZJ>TgyTS%H}URlw1ou5g|e}E zY3+}^1v-{poye`i(y!{MufqxB=m`b4#CHx+b%RPRTQU)i5i!^JI^xiR9m%{9XbRyXuv=yR6Z&VJW)p_Er-L;f11&*-n-Ze0p{vJK@if`phW0DR6N`2 z6>E(kJnKZAn(khQ?Y`SlUY{@@ClbF5R&Aa6S_`GYTL-dV^!Taixrz6yp|2_PyN1M=pYD5?j%0qahjuzjP9v%kqbyj4O_BnAAa2*7v^u)U zqOHjr-Ynfo0!%(61|OwPrxt-B9$&r)djNMaKV&sEl8ZhLrF$Gf3LIa`KOK)wt zti@#snGNNpR)|dqkCiadUvmCr&i8#6mx})qHW#@20#Ef}eB8P9-!E&6{+UuDo(Me> zU*D?5$RU3|M+1BxUW!hY88CtvAj8g6-#fupose&_GWHL8OZk`TiC`aicB59Gh+01| zttpvs{Ku0HHCvr$`I;v$5(o}8QZ-6dQ(HP$o>Bnck;c{<&4zqP7QcA4Q42DmxwP0jk@1Hy{4P9Z;(F%QSVA7>wG}{}dOWnFhsNFQ$hg0T_F?W?dKJ6Au+-uH*`07w zVB6@U@o`v$>PJ84X<+gM1U=+PPPhX(Yd{|-9M1sMA6CjeGBU{ZXm~BOOap#P1m1+B zDODS?*T4@Y?>Jl8`3uiHQAi(0+9k;HWzs$zg4Sf1$7E8F0X=Kuu!vcNkJjLa*@L_O zw%OPRFC(kD-1tvsDMM7g`cbvMb_NG`AZD<9Bz%6k3}u!Ao8{CvD?WQrgd?5&qFfc2 zN@+VPw4X^z)fZ4RZ7XI5!T_Bj3){oe`wT8Fsnen1V9;*pC0M*es#YW4qQBvz&F};*VcYwiidal31fvcSA(oG_pnrjXW(xWxk5b^BUD+VorEE*J+y?E z_f^P!1H+HMT2hpg@owCiHkJ_Iz@OB~gHB~(P?sNa`5y);^cG$EKNI%jKv~cdS=feb zULg9Br!8OjVgV?@bv8j*%Si^_IleyD>rTkVUe?uYEO@aR%BzcT0N*eiczV{AwJSk7 zkpzPs!mH=02f0gye4~J^+iya=-El^+t=x}_B8hKf;i4wj$FF^Sl65{hXUGX@ z=+kU4hLxL~y&&B9${%n;@KB~d(Zk(g`g5|;PR@ryc?g%QfjlOU?sp7QW3feC8EN@pLxH z23M*>nZxN_&gkQGZ`}2Bkj&+XFY;>(Xz@P1nSF8gQbdX^4o;3%Y2e^6j`^IjnJt(t z#r$}uCkJOQ9-rBI$8j%7TjL+arY5bIN=>HTQRtq6bV7`RL}kv^Ei4B>1-ppzx8*-n z-Lt0OqtwsZiMen)@6x*-&RQk1FCnk_Y9^qnWTx0yxy-2S4(NUa%lz(O<0Tinao8em zUiKl~Y-HzFUKifG-4S@g&WlG-%zeWur7XS^T5oPXGIYPd^rarRL6gQ}erH_{X3L9u zN#Ar_(JV-7Ggt{%D&TjA;V|{trM>PYh;WF%P+TyFygG)_17K!0fM64YoQj%xEX!n2 z-m0@>yhPkpz7BW+9Y22;wKr{~a{}~w+iTS}ojfj2VvoW@b#nmQkKO2NVu;cMk)?K7 z0DWTZo0bnJX6Re1h-!>9g8DUkeE`ScM9~?M{kg$`bPqJNfnLrAxDZZE3{s|L6GysA z__@M5;p0&3QVNXMrp>$+!Xl5D=UvS0^3f?vlNNUf#AQa??hz~)@9CO&!=ysEmbwdo zpsuPNOr#CVOTJoFw->*oL33LtQctnrV~ngNiKS7WhbCuJ#0+DKm{)|V^5kD!Yy35# z-Sq3pd3Y1Jb0v$|z$cPB)z!c)aVBpMf4CW}BZ&-DPzAIydjVdLc57@Z7*lj97*k_P zGp~rcRe>hQjmg-pGmPfrio>Dl?j4aSN36}es@?}?$@law zjQ-unx+0DHm-LOW(D8XeuEqWo!}Rydj^ACM>fpEa(T88b$!kN7irn{l{MNUB`uun0++TK&uSJH_<7@Ml z^Zvf3k*^;^HXjYj%dgh%Slav_GWYW*6&`1j(VaQ-3BjRWQdL`z zBCRLcP37KeL$}JI$5w1T>}F2=*zo9)4U-K0;0Q2*9jdC0FvB;=;2kt*%t%%nu6X{> z78l~n6eJ4tbu&wgs?r)N%0x@fH|I_s&t zLU1zlqT#+7k*-utEgnJz7vh%b6>8n z?K^uvr4+&IdOrhb(w?*9#$@E}s+Jps z397@r315|OMrO)jEHBq58}dL#3Jb%42w~kqD6_1&eIT z5N1i}$$BUzI%Aa_VB)(Nm5p5%RW&zBA5|LfW|DES9 zu6Q&999ixZK5~Y+)W9Rm2x=L{1HcCPckU}aJ)q-MAaL!#SP11Jf7r@Z$AC5KYh)>P zW&BM>b%;f=8wc?^ww!xOgyj768dE^8oh_qtXE~`b(APtNH@q_SkpJVViimEfqPlZs zu=?Y^>Wi@!PWTA#H=h20ZtHvwds&+WhIfbC3N?GIYbJcd=mKLi3C;jB3)n~;h#PB| z0&5sjP0{`2mTu9Fach%|-Mpgfw9tY4CcBqqPG-bieF&ak0G$rEZtuiSbQjOpcl)qy zQ%7{sx{Gdf8qiwHymvMg$xoyAZ0qR0T|R49u-rYF!%@#h;W+pSC|yK9m+OHfhZmra zZrbqd>Lh}Z)IHVduoYN4WNG>d@ixh;1-12`$0Dam5iXt?0qGOQ>DDRnXcmQU{5;Y4 zM6(pl5sukkuzj?I1jqn>A z`+h`(>U)85>PKQ;I_KTK!}1dYcfF`$hl@kHYe-1*&5$$xahOZd6_3;yxEZ6zArzJX z1Z9fmVtpohKullT3&+%bFb$bUr!i|j!)UzNNmgW$>opk$mA4P}73_X9EYnYV4VZZ% zQVF+o5UC3iX}wI{QG+SFJhUN#aH7+#OuR}ijn5%IcI>GOecOV*p*+3dXWE$80Dm50 z7K46uwGPsBORL@&6Qxx$8VnQe4>YYh^FJ%?x-`*uM^ym_Qshe|czb_(J5gmrb^peiGdtf47AGojJHPDVJA=bM&8Rky=B*b-pBOiZBCVYCM~YRam)uM}Wc^ToJ5)g>)C@i6m_H|pZ79gC3}z_l~UjCevrvFLb@r6eTtdpOM~jH6mc z8Vb7YnFqd2zn|)&zX{}Szr8)+2}ZvQ0zQmUJieY{!KnDtWnYCm(J_|sKg)B*YGLrO zfR@D@8A-@kg9m;zS&~94SM2K0*ivdyG_n=|2_3l9dCHB^xuECKjTBuZwUVaK2 zQAdeI#$V0FC#pnLu7q*!&9dtv4A~<>uC=t_c5N<%%OObWM^-ZT49ss)B>_$F4yW9wOszPQ`}l>!ZXB zNu&D+^5(}yo~U@-1$bc3#a$wR=HRHZwxO;LsZm%|I=daIT+&PkM2u7147YaUx)jcG zmOFhUickpOvz=N5mmSa_$Ks=5G=hz}D{`&~6On%&%)q!T5W{-FF87o%_+1o4p`Z3e zvd&Y#@zwA8^V8-dQ5pS98#UpPP^*jei=5IadCfAFTLP~0>*>9S9PLG_viBSNe4qqO zA}Y0L=yV$a?&VMGjoNBZ#Ch6AFoASF}UVW*T8>J2Po zN(>QgooHuKDAs~n#~@xi%@#LB5+V1Bzlf`B1wDsD23Zt*n++$a&j|aY!OUNjcYM@; zVDG54MH9nT>xa}UC;AFsF$ttL9I5XSe)XeQ(C0$H7`s+GsukNLn(Kzo2@%SGqA=4k zi;?5-!1p63K($UnH_Rw*DGLtw-nNr+d8~D<>)ws@5JzQ~Fi_74TJ0l$!S@W|1Cy=- ztz3m>A9TRe`#CVnanrNkha5(NDFXwBe54=5TvZ08ohNY3eLVJ&m<=aYn0;8Vo73l6 zG9z;MuT1hds<@9H+s-nzO+GS+)bSs_DCU(}#b*NEJvI)Cc)V6*P-GXe-nUcJ?X0ik zU@f@IDC}%*Qo~Rv75MQwz)~=O!!AsBZ$f3<II`6yPYBm=d!ks<` z1`d};>cy{3u)b7J>E6+nSVYV1lj?;HODFJ3l{4>{42(b9kT+Q&hE~Bh44A6ev0YDc z4!zCtgmFu*Qn&aEy>1+*RLNRQKk_86SCxes0#pTyP{g99U%}@I8)33f-6# z+O3jmHUl2&q>zt10tAs)1f1e^rXC~D$M~dqP2CI5y(zfI7^@E>UKV{PP zlM-Di#BwFNdZd7uN!rl0nZQEzf2=d^3bcu;Vfc_*C2-&VFd)K134M}+&wb!gU?TnO zHetX$(q0E-C=(Q&z%7qDJ|;# z(nmqM0zU5fI5{HcLdacbPigK#8 z$jB8ikoS_(7w;@tathqS=M&L+1EV6J_I(``o%twU%Ta4{d07&Y$e>fdFv$87euHq~ z8{8vd)ehZiXSJ!MavBnDEt3d+;LK(LAV~?3z(pw>FP%9}8pUOXk_t2dWRQ=1F)Kqj)&c08nV;PB=h|F6|QI+a( z2~SF}W#fy2KK;qoXb=igh%nnK_9|tQQ;hNJ3ULeDBtC%ZQp%i~KDg*s^O4pGjmpaX z{Q~;egQ~v!Cbhz2&}t?lF0J>&Xt(nsb~BuGpOS#E=?RnsM}H)j*=|a~ChnAE!96P< z;WL{Wgb){}JzTfKAcqeaS8!i53;-sDX5vn}yw8#$zJ+*oaq}a7ipsO?8s*l&L}Jd? zs(;#U*U=5mq}_R!MWij#h0g^{P8=CAnZIi;4olmWtTxVrc>P&#IpyQ_c#EJ zW)T$+(K@uB+yX6g%ovEMBs9XYZVcKYvvE}X@H@Nh zQF~J&qaap)>}pF1Q92_%l}vC@dc5hR_%22Yj#2e$VB{wL$Sp1?AA z_ueWTxk4c5KV^gAH_>DUe5)Tdk9E+q)}&IJH1x+-^phH4#TaJ^I}SvQWrdYm#w)4i z%kZmMT}qAVQ_GRUH0t{}ZKMy+(Fp_{jrc&dWc8h;dWk&))fyCg|1byWGC<95V4=JX zhB8X#*J3I34$|koim~?fH+NA}z{;2P?}=~vi=(;oI9(BMitvi?_6o}9Oum`Q;SrK< z^eC&!dca!@hu@GdtCb3Q8>wSPdUdSZ=Vg08aZo>^M7z~R>?<@Ry+XGUZ$p7@t|d5; zX;LYDb5Wu_du1fR#8-f>I7Q<@AbII0g&BqPUQ%q+|B?V zFIZ`*rA0)-6izup|K)1^tz8F82^+W9vW$byof0tij)FTg`R|E7FkGYmScIM?RGe<& z@Y+!UO*U0A`4}RCT$*Z`+&5)y(p(67?jlO6+(-ccNdiOJ(g-XMpT*Hp3Gv7-%y4MW zFXw9Pz`r1<_b_=eY49&9ZTENd?Zo_?AOL8A3-UB)L1eSFFsM%FMnoe?#^8{q1gGg) zpeIk2jB%27N85TXozrKQpkGQIzDU%!2DRghvlomRF9qQDRs14*;m!*QG*Lg?CtKI> zL9!Hw@5p0@4)q%IV26Ues15Pg7`g{*K3DJ-X{>O;Jsk;Fq6)nps0e!j zZYKhIgDej+B#VXH=(wn=hB3v~z!jWO$axB5<#ablN(KWU^NpHSFv`NL2{4OVAC_r0 zsg#959BhExfzo>(OrJNI89X*EI2jWTdF2YTh|^(Vn!4+zC{)ws(@X_585h|wZt6~1 zb^`5H^@Z7^Y9AtWo`rMpNSI3U%Hdrz=Eg%;QN&~HywZ^XzN4WPP=0(4fFhZLG0F=n zDcpoB7E^tgZ!Q%3PZ)G|{@=YI{T3h140Fu%-gGax=BVerJ~Ms(G?Hy3K44f z=gmsi7EZNiJqibHyPp*i_A4jJqcy>vRp@|NyM+=8RiZ0_2mu388=^a#MXo4|2nI(~ zWjq|KqKRVKC9|U`aaq3EOu(Wm3=R7-%cjwz4l%Ft?TV%IMJN^8Z-|Y1xyJ;;J8Eh2 zJoduwaUne}p$dcCvS{q#_3=!mcFpVIneviyfWF}uLHYNHl!M}{v!UT<=h4S}{IQ+% zX}^B#S5EP=+RsMR+3hOuPJS?ZN%EV4ShV)>9M$d)nsYTDT8n8SCmSh1e8B ze7>9Bv1>5^;mkzp$Hs~wGo<#wz>RVl%QL)bc0JXHR5&}0y2{ChQ2-Q=;T(Ex{AeY{ zk1iKBq9gj0a9bDb6e(?sR`KSFl~Wo3?xn}(9jv{)J%QK0WBU4!A- z0O*!j&k-jOlwN$1nadW zY;>z+1`Tl(n~9ec9Xrf0p6~=)C~OltDfon_BdQ{&O2C^+LV93W4ynH;9@qQel@_B# zh%KwPCND6E!-E`r<8(YqF%rjP^WyGQH-6(55Kdl^P6g8!EKceNIdDElW-%(ddYRF5 z0_#F%l|o!#XpKvQ~>QjHu4^`e(WmfA5{dY9K1*Bic62vgYD@>`e{ltX(u&b^msg zJ8*4kykTdt3wb35VipHsrD5I~Pwj#tSY9~TSCc0;n%uF?8J%SSF%YiRwn#El#gRKR zOF=d=Ux%#O!Rt|vwxZa);*DZeE{Abq{Z7n(SjiTf%*9#clU6cZeS=8RQh&p)gyaF! zZ0#}M=vHJkE!4|mB@f5yWTnEGmocBsN4Y94f& z^T1~xM`g%zEy~0EB1H>I;WU{+Dd)`;562LyPLrh^v(uWYlO@L=&~wKI$;(o0QnCl1 zM3M5!jLMCaphgHhwi#c7o+(a|tBlbF$)y;uUPA1kg&Wg9Ml0dX8K8uckUZ5m3jZil zHj2HulD)V`gRY^~$}WmW5j@UokTo#m^Ls!{F`QTz!5)7$*woy!4}zHpR<%ro=jc?? znItABu@F=U#nHXcB^1>~dNU$mM3cl6=O}C0jKoD?Bg`o0N@MXv_j|f&ZXm+g+csodJT= zFFAE~%ZNC{C`p|lRk$s%vAS(EQs8Skz(G)sg&(99E^zkhjnH8ki^~oebHVnbN00E5 zrv)@%AO;hm*~3&;Qdnm}9?>+~2E|qt5D{!LhFFR4{5(;(Oj|%(Fx) z@SDeg$uiJLbgn0Lv-A$YK)EioI^XoQS@@%vY(o6TN%Eh{<1L`d54L5~5s*x~Z58`P zlS(5%ObPJM%YYg%fq=@MN$GE}ftVUd3o3SNhh>f91*%puh%nu?;jjxNs7|Wqi=2oi{f!@*uvJIO^JME2lThfQ;!FYv0sEP^Ml%R`3 zs-2lYvyWOcu3&&`vPB8Ngh^7Es4|d@4CV+@S#hknEg%CrYkAsh$5YWBlY~oD@1GFw--J_OnIE2}OU! z1*fINocwMYl4!*)?}qGkxd>;?hlx=<)PU9mT}QZQk8G%j(iA&ibks30vNw$HLeUI|SBP zs6uzS3W~wO6NC*WFni0Y3*Zu+6~OXziVZpa@R>)nA+)1QDn+d_<=9w$!RAx zlcNvTrnON^&?XS$1lhcMkui!4784OikSn5W}zP>mu8988h~i^{(OQ9cKJH6dBc~F z>f^$ij_Y*SpPDcMxNEgXb*5*#HTPlXNA39bM~_FM=7ylzG_{5D|Cy#>q)Eq*0$Q}S z_SrhC)<&F|?;U^sZ4o25E;fTo@6gBml_S~bl&?B^oJ{EVPh{pI!hMf%Tz|Ha)!k^^ z0K)r77s_w22m7zpBg3v?NW5?r8jQ4qX&yu(Ay7F&a zzr>eHXq7R89iJgewX(X=3!t0Co+>ySX*LxD9uy&8kAlMGfsptDB7~3q5NlCJ=I^jc zY|36fRle!12f8IpY8k5>QscrfR);4razm4+LKVDvbsS+W?H`))NDwmoyAXdx#MSxE zerOh(m@0UJ3Y z>4MLc-pIA6Q!xBq9D=44jmC?*+lhTkx6pXEz@*P!= z%p>`qM=bTa8R!bLMxbn84jGRPuDmdASU+MKKPD@0Ma`K~B27Y;^akou63q614W28+ z>^U`c9m^SxqCmel;y03fs~WU7wbdh}CcHe>F!%K!YT_gSxaw@Zq`4SP=obW$rnz-F zC5s(fWJ4WrqsNy_QJ@f$GMnMsx9RnBdtRrS_Di#!^UkgX83I?KzO!2KsnDTia9goe zp@EN4nCe*P&4)YIvMq@_m;zevZyrYoW6GXK`A@fD?E2z`1`Wpp;?xb|%kLzk)V6=+ z+ZniAjNN@?RgT2d-KGpu*zIuIHExwEABxJvR^rKGVN|$A-DdrZ05F8E0nr%IJKglZ zO*9FeQw;0@U@ccckZu0WVB<}x#}dk-9AZEoRTP&7?YLXT13O zoMWRTNnYw2mWUYzVn$nzo^B}srjF<#~Eh*p#}K2KE4V2KVgGU z$vcv(-*(c+flh7j-=tmJP@(xMKdSU&6d=l+F#a;PUnGCYz+W1Pp77{qSc$FG4#0&m z7=W^h98&AKPm?9N8*fh8tb*OEXrwC+1330l;b2}MF040P;dh_RUsTqnn(}lCv!sap zcSPnn*lw<|C17F5<(?iSHgvT*VvlRGY8oj7X+J^L>SYITIcOb?2393!2n*U(Ax0M+ zsl^VtJ1ZNr@>@O0QkbJn`HSapk~F8_6BOjGL?9LHIzBQH^)`*Y6WoXyF=0beI|oWl z8ERTWbVRRIP#3>OZLC@poH)qzFi`9)y&Nk-kH=a>10()U!H2envMCgWQ&;22&%7Fd z>zjbKBv$7=c8_i;(VR{+UTCGXb_uD}(2oZAqd1!A(c`(${p=3|L1uWCRCvh~!P#T7mXMfS zr6~AN$!Thd;m-gR>FJ^~7)ivS2!U}t{&tnx3W^UA22drRMGpSoU{KDl5VYCqhH`5^ z^Kpb(60^&IRt}Xljj5kB6A4&jrq?tTPNz2!hFZb|KsUhfK}D_PHQ8h) zMtE*c|AN2SviXe@H88P0g;5QHo+bJW&g5Oj*{?j^kU2|AQ8|?Vf`2ldcLo4%F_Mk& zkJ?I}i3;RC8W81$y*~CAc#7KVHNJ%Lzh6_6zTuD0$-CY+np(Eb+qpz#aT!I}q06Np z=grC!zhKTB^Jfg`_m28>1a1|3MvP_lTPvZ*bQ=GL8bBU|Jh#-;sH z&$sH9Me~$kD<*aLfsFX`R=z)4wEH0Te3`uI7J&q+9`A~LuxqP3R8C*(<qidqP;&g^$ksOA`G7l5SfZv80SlN zVfE*e1rd~@F22;Bb%`La8mR;EA4Kny1!@bnAPjJ}&`kObrIqLD2TZN^qOUqUODuG? z;87H4nf2Rv*%NCGo^Io3q?g~ zNX@aNOVLc%K+A0faR);KDy!Hb4S~St1Q86b3Rx5ApOCUDm*K1ogra`rUUy|P($)-5 z3{<5Bn+Hi?=hK78_hu<<6Gj?Ug3PTag3L;PuP%_9syI;d06Bkg6$@gk6*T^GB|R9{ zP}w;e;+%*qom$T{O}*-O=($61DNGd+0J+crF z7YZQTN|)~Q7FY?mjH`;hXFtPtfly7>(i{IXiU-`hZ?jtT*VUtypHJf=E)?0FVM}6& zZDDu$<#*uqjCd`5uo=pGi7O%#E9E)vBYLw&vGKGHi_fke{N39t7E1SV(Hh zKuK=-24(_I6!xgv=oiyyE3a=QwHSn_C>K*0jw01rA66Q7EnIe{1RxOmEf3GX0pe{? zh*JdsKOO+Fk=G8#Q@`bT-E4gcADizu5}7goYK8lwD>jnHeq@?l!rVCMjh!qx{h)RI z+tg~%0zHlUC?mrqK1UGp2$w8^htAi@gGuhy9pVcJ7BL{i!_SbO{D)@;wTX;VqAv6A zXNjVM0_|buo64PsqarG^4wQEWrTRfiBZ5o;Op@g1l;o+U8KRD0N@hh@I~jWs?7YqUQ8a7Wjl#HNG#$Z!Ml|ifq z?stBgcqMb1RtUafjHmmQ^Nsxz7RrE2Q^u0eWY;kjOJ1dN zf5xX9`WF&@O4BE)6eqYe>H+2nN1@cgRqSC{XBYCBM z(@DmKrck@EcN`yKTc0XrvND?yWMU__gsG=Gi9JBxGZTG#S2l-L9&O)>W~$uwpFZw{ zfmE6Uw0ZjwR=OU7;hjUXMz-D|<)ueSD$0zc>EQ{_mGl$q(`3g7NMcqSv`*C>@e_zp zMf6bWpIWyINTMfjjpCuwv=Fsu4-y_!{3(uYmwOobI4-ByJf&UNoFk*YOL5TVi@Yw) zz2HDsntLu^NR^r@tmya+&l0@lNNt(g?r2l2+Jhz9Zbyh#=FS>4svzx_jR5S#R5L%8 zJ1V#(GzYs7rXw>KqpN%kIxN>$C@UKB%wNQ$lC4UjtlR;??zG+gL^uz}LP(b&YzS07 z#SusgI;(8})jJLI)+%5$iBc6=A7EOq;5HuOWDbj-hg<)s>kT&niKFPAMlOB8 zgz8S2h=Uj!<-USKUTjxHt$N=40&^5!NBJjDg^Lv~4uPIinNrr;<&A&&pCLgdLaf5Z zgg2Or`kHJDOc*V(z%qvU%;^Q4o`VFEmSp=XhdS_l(v(g_Nl_6jMNzR1(7lVN30{s* zOq_5WQ22?rrw!rh2<1p+VGnP^K(++|j9sv14#C!kvEzkGn$9*Lm!{mq5a}J=S6fKQ ztXnT%=)FG!LCK*{L|kI^2FWy)S;%^q;25_2?68CcZ%+EKn^W=#TS{P&I}PgQ;gjWn z`GE2ABxR5rD<&1U5*00x=W>=i!}3m;UH~T!H$@2dgS*z?{h8CKh3zl-q`OtU%}c?TTC;IWKm?C#ygJ9iow>N zVvW9RrvbEgZx=!y->oJmepH@U9I}B{nCeO~{c^=hrVGf$B?-i`ZQ}nB?Y9X) zw-luvE2QB+Jl24cfS86ECf^%OPM*ueYztu|l8Vlu{&jPa8=EO&Q6?|z(DmJTAc~yw z2Vmm(K-`6mKx2tX&Dfou3CSG<3$7w>G;B7EV_Y?LlVUSS$sn4d$VrPN*#=YcbF0%3 z=@NRKyoh@F!aWHamRJ`Rdi(>-=W~0fV`nEGj0ItsJo%Ouzg;f$9vYlxr0#~6wYo5V z9B<5{6bDWuvX1m4^hOcofKg5Xr@Ut5At;N23xU01`R_!5saWca(G&N{#HWol3#A3N8U3d>nuBju8)2X-dLlaLOK0 zp(-n!>~HyboGcIn9?u&BX3jl*Ma(-r%3C9IT*N_<9h2Oh&HSv~)LFxd=h8*ZPSXAiz6DFI!$;=YB zSdEVoD5d|Qoh?fEOT)2KX`U8Iq?<&e-$czz2f52B4uy9EeX4A#%7RWTT^s-;@CK~} zGN>TKw+_mq)0SB@VHk6Sv;3kvTBAvxkS2HF4(%XQ6D6MoV2XjuQeNQxqI!@11P!d@ z%*PDC&M-}Mo-MKKV2DeqiH`=_v8HZN>0Eh(%m57t%Ai{Y$2J7h9`Za3^i3qFki~F8 z7Oe8Y&&DMB5xg4G99wC+5kWPivI$g4IACbY$Qqb8>8UJj!z_)K35EUcJiJp{m}0R| zcR#yVgNA)-gyKEek$r>{O`f?ll57(k6i14B56^8b7oK{Hi-g=ZvPA93R*qPWqUZ(o zgQ(-wHvjc|VH=od`Ls8@?r9O^XjqM>Xv`)WDc*vUz|x*l3n8~70lUIM;9G6s$OsfCCPvOAyvCjAd2yx*-rqHw1=QkYm!3>z87AAV zExpsv_~L3aq`?H5HANRUu$qfVXrzxuAZ0xLlWm%|BS%#p>6#W3kFb~~aB8~8zh!>8 z79|_&QW*vftS^X9;)08-ohjyp^ccp>SA-{%b-*C2)rCtuiy@tidkOytkK;YUMO&=0 z@&qih=q3(Hpl0YyHEcyhrw*R3b>Sm3CFr;y0_wQ5OL76xqm|BuyWW{ZQgjhRtuhY7 z;h|(qi)agfr2QJ%!tz@5aSUELJizSVf<2F7()}m7q-Ze*sBAw!h0z ziYOFy+?&b(r^FwfYD>G|2dsBjbpV4KF9d(o;GqTqpKkWLaX=JAI-u)x-s*l)uuvgzii7*=%B%z{x@TWc)DsKV_qfthemPfYaNlO`spl~U|M zYaa-V;8zqD9FLN~1LZol1~<|ZyoeUr!{QASwnqsdk(3bH_F37Klh2X}T0x-PClP%^ zKa_mOMfb;i=p}Os;k1~<713BJev6ghw=ye}vm6)=u8nNT*t@hTWAE~~au(eWLQ+N? zpvVG(mQdQw)n`dhQ|Y^U(Lg}bPJ!tiKUOZzu#o=llmctE1kye!!OR%O{O|wID%ZiCYpK`Ip6r@K?&^Pm0;z$&6YUS+vcu;UV4kF>o zs8Wg?HJ*;7_++JoM-Bs+icDT{N(UuKHIE_D~6HDH; zn*_EYhfaa7&=ZH9MVpAKC{CZ}#lAfh%}vBp_L_>gXU#g(HHW2wh{^sK??X&; z5l6+A_$Js$J$$`z1(J#V*sR(MJExbbMs@Wzt7Zc=c~`ft(bY7qK>wSk4pz8V)AfAyj}NxrmDSL5DiZ)`YPr8MK5nibOub`Bdq~ zheAP!L?lHem=0?U`NAjjuc)1lHW^WLPTjY@ZogHsu(LE)+GdBtaXX#j_FB_uVNkxk`nr9oy@(Kf zvM`Ie5+VPLiK7dr)p2ElJfe9G%3%;#Gw?baGPZ><$P2_Y#48P(nE_gRX#(6&j@1%F zd5SuO21%zI6GXU%#binwB_9=6xu;!ZNM+RtCp~=vvg&nx=sjaSS0UDy7Cwmt;V^~l@FPS4iId+#mh=z>~B3vGes{2+i z8|@N^$qNahlxj|lsyMC&IZpG9ihI$69zTrcHquL+YNQ=w3B6wup~8I4tI3Sp(#dRT zC!CzTg0Lx>16!_&_90XVGDag}H6{O(^)Ml(PnL={pA(F34nV4!oDtByg zVT)rNkkwLIr((3Js#O*mD>gR+`sIXM)#4RDr}6W2S@=?24gwlK9thav5cqXa6Mo5h zIHY3G*KBrhk>zZa(p0Mo2WME?LiWkX=s`plX%q)XXN(+4HeIwV%ybQy6fkO7k+!}0 zEV1Um7A;`((rdGAl)__y?JG);d6g#<%V0g58aCOh3s7e;N} zzV%Tua%qmX5^v{5CoQnd0aRGeFo9>Es5GaO6x~kNz6wf3ViILEBcVYl*qy^@loK1K)XNL|c!vXZkX>A$m{gi~ zw(7zrDAf<&z%ja7`FiB5m-!P;XTU3so4IkPB7~mtpT4ue!JMTro3M~l`5GMQpd|(v zTwdC!Cr&gLi)BKUq1-`UGVz2rE+*DaJ)~byvKFDf0jOug;+m{&05A2&uP z%6V-1iEBZVXEHbh5Yde>ByJS+XBmoBbW_INt2m=x+9B$?-^n4PBU>|%q^#6Ra>2+~ zPi08weGv2PkP)Ru{fvox6#m~0-5dcYVP5Q$w>bXCgwp)4W& z_Z<9{cAukoWYacfvXKwTgMpknY)Z?AabO8CirB8Ec5298GR=Y4N|R1Y?vt|(7zA_V zbA-tOm}J({edlu0uFEs32z5f8mf3BJv8`s_LbxllbOc)zxC!;(A~3x4V-mhuGNFu< z<}8sR#_!y(KGsl z(+BbV?t`K7AEWx|H* zg&0RPCsb;9%aV(eO=j_EQcH!CN~DKA8$2+8?Otj09g2h>E3#6RDo}wPkpq?i>P0vK z^5Q^`?mSy^lHFSXBxGb8HmKCXbmj@B(o)o7g)*Ur7J_&cA$>OH@YYi&s$3c_PT%6< z^z2{5|CH?UqfY;_+59gP8cO0nHP(&Bf7)93p8yGAaw-Mn=J-J}H;5J;iIprHBlR`#XUTKCD5l5DDRRv&a)N~5jYZs=&#f%*r zeaWi)z?Mu6KWx04_b~V$qC!o(33aP1ZwYo^y@@W6rmO)DyQYTRX3Br2sN`ufu%SRV zDkU>DV_`FT-m^|S3-!8WGp%}ta71CDyyx6TFSfx1WKnJY%nfjRN)eiUFf9@B*vWD` zF1EvO%C<8P_Z*WxVq`1VHfjG{2}l%pvuVzX zhyqh=mYqQM zDZ^-!y2#2B;g1D98IO^I%}$*&og?w4tTAIK#EcfbB=CLVlcRYT>hUWDp~S6f@t8>* zfE)-##6bhM%tpknflFD2qoC+ld@AG&n=O;cyfj-**^?pcXJyjx@;N0p)j&9D^2W=U zIUCa-moWqpj)X-i?$Nr)b@DlK^-l|`74ObXjs+HJ#6KLPNrpQT6qkjZH-~`XW3&$+ zq)oQjtK{R@{vfpEni#>rEE1Rqp^pZwXd^aarSiVYL6Z?G@l9pF+E`Y4j#5HFa#U+f zw@lH`K`Q{G<;t5tm@Z))%|IFNDQT&KAyjAr)f6QU0V)3(wsTe}C63hxR5Bms-r^Rt z1FMoA(deckbwiv=mIkMimQ~Hl^BLmoOw}c3#w=y08wq1j`6-T~l+80g5y~7@N{OoM zP=dVD;Qc^LBAb|W>CxzQnzkd7db5B)-mj85D&vMmi0;&cZ(k=HA{L|I8{ z)J)>&pJr-+!ZwI(&&uzg0zb{gXa&H7OASciC9i@FIXVwUr%*b_qoA#(KQO%~`!b`I zk4q#nO$Pdi6)|zjeI=BXf_*n$0MOB4;Kxjosejkt^QH~g**KvAf2txAkLH0q0zM#}{BFv9|87f1D zbayC1XbA1Og?Q9n5DcSjI`$uW)S8q8)NagCj5)B z7}f2Vc1ShIF3&ilm^!AS7h$I<gfQ@O0t{qiN;%?E_DZ0S06Qea(I=u^gfK_Cg^`*f zZlMw601)H37^Hx3?$maQd?WxOi1nmLZ)%lZ0!hZQt|3CJC&d_j9HUyN8XU*wR#y!_ z6t{!-Q@|Q^CU=tHp&04up-d>OlvrR6tl48{&A`)}%-U+4q|uu(In@a7nOx^Q&Vl7? zM(0BK)Xbb&Gj;0R>C>BL&J=5o*I(c@rsW}EM^q^{l~Ov!nx@fbV-_BfMCG7B_$&@G zT>>!!(2>h}PM{!3=PA7-(U~ zq9tdh0O`y|qhY#yP+^GVXUNi$173*WSGuf;)ZG{t+IKChj5veTT`3YiqnDK93o-@c z=y`UD#b(f#Eo`yG0L0y+S=wvam$Y!CMvv~n3D*ri>w7cEtg zSr~TQv+%Ymg&@w3$S5mbWlgVyhrSJ|x>ohLg5{I;4az!VPY!g^VBxGcuWwdW}sslXFC{ zPNXQ)s5MR}StMT(p`7y=9#H04#{h5oGAYjqA&>+ZK@j2oIA#ShmB>aya}z0V(M*!+ zMB`&cn}=hg(McrFE*(o`Vj`%;221TyrW{&$1RO0hpbCb^w;C_JSpxF9TGaNXuua^A zVm69~I%Zh4SCJ`r`Z_}f+_)eH zjSddmcyP-cOk>RtYdDr;&5CWKx+&RKsh6k_uK?oktMu-7z7{CnkrEiCk=BQ1hQ}dQLA_ijRu`y z^FROrX7^xeS%xGitKK}uspkGNS)<0=Y=0Xmz%qj=KsRfviknK3rz$Jn7dMhra=(5n z4j`1Qmbx%D0^xP$y-#|n9L1zex?Ulhf`&1-!Bl5Bn{j}mqQ+~cw{ioW^wUn6Mtcs( z^$rrL2loz(*n81kKAyGwguF+1*c7wKOUM=zu$&QAH0Yqs)^9yw0A!tkM{%};thtJv zV3#a6FH%&i1p#27)gH13^`-2wvpk|?QvfYvr~tmQvP0B=Byw=e{Fd$m+h#RK_Yy%1 zjx>kN>S&~+<*@cy-OxjGwEPw&^yQBt+yHYx%}sBg*~%fNFiC7W#ttjkE-!KGtofJ# z08OJY?x5XkbaomwStGjFVpV@>)adHkpUeq?=GI236``qMv(ZSejRTl%iqw^~w!tb^ z?R7x4zLen=#63`bLnqKRA|-sJ76G0`_d0-P**ane9#)p?IvgxI?{NfN7Hb2&Z($0} z5!_BxX&KVBjS*1<-x$IeK<4?>fkE-Wa=ZjXT)Amtk2^{1d#ZoXNgMl&l;3>!vWzFJ34rnZrLEH)!?r%_RjSyCA#!p}~|RTRqOe~z$fj45nb9hxYH zhb@|{7DhW!F-C;7NO&aNkg4DHi`Uoe7jJ+@+kmOHHZ|v>aoJRUlmh>()Bk8$_|e*c zY>xgPKVf_^{a@EOYX7md`2Pj2?MqBMN0?sC#oVo$;W|4vPdva>Vm|c8;{9G0Xu3`rky4u$k0u9szLT|O=OIK89ee?zsaaWpGQ(_!%f4M$0E05 zFNSN>+UV|aO<18}6N7&Y3y(kwn$qpTCEsKm7`%@e7-PXUfwlC_Y{r7w)a^TA-0ayKRhOQIp;!>7Ah%)(biFzYk}OF6aXRyr>bo9q-AyrqPK*M3>Bbo2ny8u z`{XB|;RyZ0M-pKR+Ga?=MrTQ zI6f3+^N<;~1c)RJB|e@svyzt*s<_HIJRL6g1p9P2LO5*?ZlG{sYL-R(43i#hwZE_kcraexTpp) z&~m+<6h}b|%%9Ng$e*g)$)r^xfNm<6l%7*;n2t^N+9N1$2sa_Uw1OsNv7|7hk#;N^ z2D0o}NU_Fd4pHC9!-A%xVIIeJIq8HC`zbcf@CS1m`elc2VdjLgWlO|AuyY)IS6STf z@{5ckBB7;Bi_2yt>P8^XX^Kr7#P}tZA-MquphtGcyt2~b5S_y@{2M)P;)|8`Vk2vv z7)xb3Biw)&LK8_lvYJ&g-^p?VAY4y>tMdJ=bg0+Y5-r_`9Z znJBWE@SR53AkRBZ^M?6<%0?={HVy({1&*EIZ+yIN0%`!9EF8HiY0l*g7`5@n3H#R8 zt9O*!%uH8D#!ZlC8}_5t=r!#Of?Lek&%l6dXbp>XJNkIMhI%BS-{A?ga_h8!C7($&pw7A%|Ts*1-G@SlR0Y8m=G z(oCO7q?3OeQvy1Z{L48rTL;T+%BI`G8xg~Qn)3eBNL7zTIV?#P`3nX0B)q{ zm;ne#XxB`sTzeEvc}mCpn6VJQDIt|0lh7@L{Y&t?&*p4TD9R38ttcfp8TeviihNIy zyvs)4#<_?E$o@^3chD&8B4-k5Q`;2eMmJ_R#xMgi4+@4@;=slBjLICBJa>38th8*X zt*z}Kw5agZ_Q8_dy@*pW9c7H|ENKQ3Ri>T_#JBacd@eRmb`!e;Q#!ltn zvdFo%DZp|; zfpO-L!2s~29~JSQlE)OjsWC1nJ&u?l6C+GZk0a^F%6>?pD=qL$@K7_4X$Vab$kAq?;p_U;Q(qc|QY0PQ{ z%cVTMNrM{qT!!zis`~CwlBEPNc_R33FxHOw(#*HCe99Zb@@VoD)F4n+&^tx1 zP}+iR+r6qPfmi4~JtojEecY+xCnBCpTNiRLTpGq;CF??#!<@>{8tJMk_M(!DK#pm} zWx}5JqUeHtAi&Xu0wAt*K*QQwRYybtNT6?u?>YtU2nFFvq9;~Xv-0bz+LXno7z_*C z30{iIOA7%}P_rK?DyW+hap5VHJl%2oV2kzb5PMKg5J?;0P);Z;2y`u`PR5YLv`T*X zK`^YtI%dqp-J#*Rff9Pe%lHU*G&uE~FEeom$iqxgVr6Kw9+wlq4TmICIo1=XFev}a zBOaMAeWe0FDd7U?NoPbf7gT9Ua{>lKzVp)1i~|Ad&cvE4LySz=i>@flE?m(3n}k-E zL&-`%Mm?)+A>^$YMQ$fboJAQIu2E_Gr&RuL51Ob6;ya{U#Dzr@)Oar2_i@wzjg7U% z_J8&06go=(Z;j81vu92FN@RCDn_p?0K6TDE+x+_z+idf1W4HSPJlXpPC%n4NHVYqa zo7ysW&6f7zvrfV&fjp{ZoB>dd;hP(_;+`wI(XkQ`+!5I|E21NQ>UN$ z^yhcDxwiG&$Nk*+-Z3M${QS>P?^$!zySJ{ZeBt8C%VU>Mu@1WRFPGdg`_QjV`_6~& zAN};I7tX%*<+Fd=*Sl_2!yAR0k681_%d1|x{*fa#EL`_i*P6F3i+3&ij{~QdnuN{xBS~Y2I z*OS+-yW~%kU;o}KOWW5UQn22s*m+*_i(ilIabee`JDhdqf*Tjkng9F7n^ykh+xwmS z)}FZo*WJEo`0}G~y!w|fzyGTr|Dfr(TfVSiYN6%M{aXv`Z$4?=K0E9+aMHg&aLs}% zj#%^kZ&h?(`paMLd-%eI3ooASuYK&T>4n1jmABq{>xLgae*2`?PHeyI2kT-&=jt(hWDfaQ@=^&S@NY_=3Y?Z!cPM?TP!HJnPVRCjE8t zi;E6i|Cc@HAO6zP=N9j<=#&rrliqp%#>c-N+3A$#n^$dk?AM7!J7hbbJ9+r@^i)yUvh2F`#an?v;D(cAAj$j{Ufc>Z+`H>!0IV$J~(5- z`m5WQp4vTo$$xp{hQIZLgMM+=^V3o%yzrxycXS@}*R`u3T)yh>&ED<192foCn(p_k zb&sF)@W>v|U-9a!b>3Z9c&_`w-<*^Fy}o(!!FUw zan~;0Fy!2QX3gp;J2XFg-ch&h_Thi7x&4Fwx3B7SZp<%v;MCJ&<^M!+p1W-CF7V z^VwJa`}nWyvDZ(3uy^XwE6tl5Haz<06L-<$Eymzsa|cIB_{etVBMpSgS2qvuC2*>2BMUOxK4mlnVN59jvb z34i!(m+CJy1wKv>w!@9$lzV+8>d!7CRqvfKN z|L_0(?ZS9#YwPNLuDBt3($|l@;D%@ayi@zcC6}$ee4pT0;rh-gf2{qt z*qPmjRUSKc#qN8Y|L!k#oxE_v>6fhe>a?#OV(fhHCI5*3VCR1}@BGwPPx#Y1nExdY z`DdMZ=1ost=&ZS^e)?(kKgz8;{eZDQ+v(T22RAI+cFteU{r>aMJ@U! zM~CNp?UnC-_q(6lv8{FN*LHnn%8uDFPtBix-Of8^k6YGx%jKu$w*S(#2i|h|Z21WO zM(D|@xqZIAYOg0Iru%OD;cnlYkiG34^Nrz$FI#y2#KDhdk%cq^OevkC|FC4z_eVrqZ z|MlJ@Dlf9WdFS!h#ItX{xaP^r-e^4JiZ^#(`H$aRvi_nS-#q<;KRkTKc?Vv%Zq0iQ z>n?lag^|ydXKB{LJ&0{-TgO_`eJ1?swFg!+$yV*5}{4J$wC{{jI;BefjT> zKID~EXO7%k|FZkQ*Df2LmtQ{Bf9Wr4_Rqh&YW0PWJn_wUpE_`#3IDuf#)ji=Kjy)| zU$qMC%5jV!E1&eT(QrMN6%}%_voGO`buHS;`3g<{G4ki9+!Q7{>Wc1 zJY)4U%NPCkKkt8OUc;`x{lPb~H@Rf;oOcZ9y!^&zxk~5C)~4c#flYw9{KMHH@$Z9pZDE<*Z7-X*x{Q`{H;K8Fk>l>kJ-&9&$4|Yv<@x>nHx1{N zpD(=kh=ka`K61`sxw|Jix6Rx2XZh!!9ka?GzP!74#SM+Wd-6DM@51Tv z{Vx8~_7mDK`QdZBE}C-tjQXC%w_ktp4KFRoygl|;i$w6{mG9 zyE^~PM_*sH=#B5>UOH>WibJL^e0aCM_BVD}{lU&VeZRKyTSI$%{_HP4e(KVnUOaL9 zxO3mx?Y@z>_qwI|lKvfTx$&ikuh{+f@tUf)vlE>ko%V9$f>`5sPaIh@y#B`rykR!| z?zo*(N35AN*mCLj&2f9S-7)#~X6x@G>rVW3_l%R49r4C#yY+Tn`O2R!-?eLC^&U?= zv)47v;}h>@9OCm^ONHB8}>eM z;P&T^`SYsD@nhH2-goo+kKbH4u?F+a7yV;rff0CeHrlA=jPKa`T$24!h_3moC5l z`lj>mO4UEPYU~r|?)k3y#h%I;C*&?&@$516zj^KqYxTR^-?rDibKcqQ(R~Jv+v6{9 z-+atppT7IvD+ljCx^eB}JFI%;@3*}F+pc38dYTvA^20A&v-*PBk8c0of1T%lX?Wni zeSVp{X2ypbrcc^%#HklAzG&+VdvHh_x{}KGjF-~h6iev zzw)-xvG$k`|8ak2Ty)aXKTm6TVA{~$le}rWzLI|Wq{C0Y@T<|A#=mm?OYw&D&$)N+ zzn}HQZ~pD}ZLj<7vd7mfZ+s@Yer)xZ?l^CFwSD!YC+zU8Y;ND@c3L^K_Q=_jPCMf2 zr=Q9_dc$P***g|Jx66|A&v9;Ab?GIS)cxt1p2OPzFfcH1<9^=rKYgxY&)241x%T>T z3$8n@f93fsXVXJ}>l-@c;BTDpkC)%u;TyGo zIDE|CzkJJ$@4j-}^9_~z?6m%_cfL7px2KL@cFxhuPn&u5GdHZcaX8;|_&!gh|Le|k z&p9Oh+^uib-;MYBZbuyU&-)MQPhT{x{TrwK$8*m|Cafx~JpXq~mernp!obCQul~i$ zD~_4D=9%aJm^-**BUQzzxK?(zr6JM z4_eoL=ek`lU$WxR6K`01LFG;lOuuOH0~c;T^P;ot|H#+>p=H@QC%pOP-PRx4@SlGi zc;}DJ-H*NYt2@V5y!+*K3mR_OZt^PYuER&xoZ0{GAqVer#iECPvb45quiYD7N?!B& zPcpNAebm~k^G6(IKa$$;`ki}Uy8KtU&J&+c-2LYAZC^dV<`-8zGwYm{;|{$jHMwKT zEo&3izkmI<#rM2z?fLNHx4*mc>RH!adhpmAF8R}aZDWs~zI^BG)lc5I^8EfuJN$Uo zE=z8II)Ci4#ZUeInv<`5ebU{BkALpPMR!M69$_AG!PC#w-Bj~n?FM_vqvu}L@uPjJ z&GmiD>uRK|X{B7N(d$MrTpvD41K-V4Tgjd|Y7FMRu_%bWjUZg}+m87FSPu6fDYoA;XZ zxmC`jYo2_2$5$R&cFxN~<946fw&YH~^7JFF{!90V?>+MEC*aE3Y3~`e5A3<^f=Q?K zYrPAldHI!ZB>(et>%Pm+tyuNNukKm%Z$qfBxlZH!a-dis~1Vi;l00 z-+krXzqjuF>Ko^E-S}+FHM6!Gf8gLDlmFv851&>zCi})U2aSC3%ikQlt!>_Ql{3zB zmYsk9pHBS3-T4#VIRE{7=3lq)t9L)^y}Ew=o=bXf&aJ#VJt=m=+Phxp8d~%AqR02^ zTz1Y|7iWHc$!iaH=U0BMdD%}+xM+52#$dyakNzZ=_-XBfcRVx2bLw_nbpOle{BXsu zZ+Lc5!=CF8T6Wr@`=4vQ_DJvI-Oj4tad^KU?tf*cQ*r5KJG|65{6uT-cRslGmiw+g zHF;6XkI!Cx#_Pr{GpoP!(%;YDF4yyi>pQD|x6pq0^rIFYvw!myg;Qqi_u!SYh8EX4 z={-;U-Q$lP-8*Tgo|7N-e*T9K&&?PgWRCgLu=~{WFRrZl-fxfn+w(8Ha8B&neQx^6 zfbp9TuH0?#i62~V|7hojj_qE&_P5Xf<+*Ilb<6fYY1ui}dl&BV+HdC^`MKTl&z-)z z^XxbNGSo7E@a>-#HXJhVzWvuteB<+<>v{i{?Gq2}G3!g)KfU|h>vsFr1;6{_z2{HA z>D~js__gdY$1gwcjKUkMe*C2gJ@>z`1}^oZ7JRwB>zOBax#i?>8!p^s;cqV7ar_S`oc8{I12iM*4gNmPJtG8cw=sv4wumAknFYmVC z$s_lAYSMg@ZL6>^d`L*>C%E!L3$SuMFd1b69}P~ zP`1*02WcWm5osDQgeDS@UIe5j0Rqy5fblz5s@o=XoawfP_8A}FhUcw8my3aj zylcirZr>&g+O*EA?yJ395%NY$lf!+3~UG4$8eBB!D8sb=i(`0Z<@V$>pkg#m?j$kQV^ttad zfoLu&WCj9d`rFA1>Wx0-JJ@L1MYet^dK|ykfWIsixV{r(RRI2s3ks2(kiC!Zy+bn~ zzoRTbLmVuLb7;EbI+uLDXL!X`TnPUVnKI4Ev;mgs(dMU_dJW}C0=rthz!D$|gv>eR1fc^Sk z-`N8z&~N#Kd_wSGO!%#J=Ce$r~O0^I3S-s`;}XT*=DxYNVm}Zz_bzh z0-082B#A5K&(*)nCzxI|c~d6a?ZrnX&1|wh?ops;0ZF#~L;HEp!(2mP^pIZq!>0%Y zvM`laT_PLG6B+GghcaNu+onRVMFC7>ln(<=WucknL}9-xo_Xer;T&L%pSJ)&MQ;BO zRxa(1=oaIYs{xK5Hba9mG;hZjM1|&K_jqfKL}eE^mo^lS3WoG= zYiBmp4a9&D)u2^MWW3ahth-!lO0%;mpLVv8Oru@RbwZGB_R^`r~+#gKwF7G|F61lNZuDh?zH;0FIP|IiRa{Zd^p z#ZXL!Gb6}@xb&h_XFSW(NdnA%b(2wACPWFcqU*{R}sN3 z^_kF}smB5xZ|psn#rpPUrF-B*GZa8l-RddZf^&s%ND}y9A>;?91<>I?!U3;$n125!&@G?{X zEY-f2Jab>L>)w@y)gk56&Kc<#b?$yd9&finnzqvOc(bOK{Es5AQQps!H|s$U;Uba4 zc0~5#yRT7oqWahUPy`mdTl|N0LjftqKjJVXwfxij(ej3r8u6R zF~!g_8%j`ljAso!lbZ2u=&oy^0oj%wDH9_o1vt~osD4D(8ZZH@^eJ+SHZ|jib?BPq z>ZBQDyg=@_ ztm~v{f5zYWhTjU`tGBuWSL(JlU2wZTy2>it82&ZL9Q!0X_YLV>!Ctrp$goa;4gTC>>!RH{@ij$ zW_4_cN^>d2)0TAJ&=D}a><+Nl;%z*>(Hkf`k7VpeafD3%_y^6xUlnKEhaW6>V*r)x zeYEa>c5+~(s9shs@U)R~k!!^>`GBQ2M-OTGS2jI93PA?}Q8^1w#qQIu(m^ilx}?q| z7&J~`zYXqh<7ZgkuONq0A{4MM&Cq3fh)#bt8MrPoBHWS-XD$fWAgE*nc}$cSLy*b- zqoomE{&_+W&h-AZ*ZyP8osTjwVpA)=qEsS>3g#3u@}bez5K^VQT@77kWZ+0q6fz+X z%pr-|v9{yrCDWP(9^{O&fge=PC76XP(8M&0#+Y3rKNd+MJ3w0jfO7w3ATf3MJ-@`@ zll|b$$LO8;FtqvE6M|5=QuI4~$HB?|U*_YV(HBd9-%bU(a|bfADE`f_K%AL`G@=zYdgOH> z7GqXSQ>#^Q;UGwN?3wk$e?6_>6kG*1q2_fM$nimb{0Q~R8W|ap>h10Q^}@x)KD9{4 zU&uj6rzgcfY;@;wyJ>n_+Ro11i2;8D?sDSexuR3l=|O%lapdAR(%FTVcW87pk`FOU zju))%u=AyLt#y?S&#ue3UBVEgf{9BKGftDG*i;#~cD;Q9Gvb3m=%*uzIHUiye6 zbXR0XR@OzqA;qq$s%l%))&vnfDK|qLTVg?Si}jNy$NoVotGShxqU;>vg8iY&p!5?} zYmrTCQEE{b>2>OXIMzY&=oa?Aq~NBM;FSW4-sV>vKbInvqdQ4wL`B3D-WyL6YZYy3 zatDOQZwlDBgH@O_fuU}}5fb>K3|5_E$IYD2fi>Q@eDid z<{x@&zg45oT0hv{-rn!J2&tvtzw`g)(nwO>))r?pANrz>@6SW+t2ijKy1Dr@x!AIf zVYqokZ$Ti5sYaA5yqh=Tn|)H9xM~PxR4K)TyW9PBVLWu*GmzQ^S$zO~wT` zNAh8dWFm6A>Qp}ZloVSMr#!&T9QP}vLb%j@$?5?|N}-mkVnVl1KRUL);h4yzRs*X6 zWz?c886kiV1~|lsm9>Srz$$|pSoeRu;QzASLPU5FPg+l?cn>AI%20J1W8PT4T3uQF zxUh;hygMAK*BC)%A&yCf4CkQSN=`avu!-8hqih-o!QHh8OP~j)$QZ2ySLqQe54CT) zZlYZ3{Ncdk@a{ldrgt2GqaQWK>BX^+GtmBVj(d&i&hw5FCD?wYsdw`&nHOgHjL8n# z2VPr~Wh+k~n3)__l!l{q55A5X2G%f*RGsM-8nZH9y^#blL89+9GIlOR9f#17cg*-> zsFQj-W=87WZH%JLN}AqET->ADGVz}AKP|J2_hw{Zgd}cC;yATxMyki6!dT7dNh6BL zOLF(u%i+~ytxr!I+`lxhuC_M6!&gJ?&Jt(Gg2BC*tH5Lu#l{NV&_im}mRO>yv^5%c z$hO}Qb7N^B5o*H}#W3fgkg(7ye5I2RLB_3lU$+L^3NF=0iRSPDHX1@BO26``-HB2N z!MCv6ca=h`sP!DPba7CK_{6QP|0ejI-}L(!n-plAgHC<8Ws0uiR-aaacNgjQo7hcd zjmu`NOBCo5--oMH%YCU^cvw?cgl3_E42!I0LWOgLu_;bJyLyTN=Zpp6R)Bl46i!V- zCNZh--uwTTRhpww5Kp$Z9+I9^1EB}BG%OYYoBa|DpY|0bII4%z_TxQGJFGSn1&#zd zyiOw@Ce~K~1DXx0t@m8iWpYzU-#{=xjswmour`^^2(ML&2(=pI2zDIR@(bta4K)_z z#nNmzbd;7|%m+huYNr{hxG+SN?7P03Gx)LbaoN}94wEmh5?vWs`(Ai|rqZG|-djrL z)lt)L_Ju9wokf|x-${Kzk)VLTp$w&r*i0Qz;eE|y?qJD7Im%0ap=vS?)zz}2ch0sK`&yW!9C#xq#>TP(T7IL*975V7$#-tty!DJXe-+_6Wny#< z-*~4rU}q_NRpYo=p_Oz0z^vm}n_ES5)3~@)w)4Rmo61Ea^>PM_yAtFvd~$#NQA_VH zQum{*EgC%5TrAW%u zlK4p93HZ&zi4FJK>3p1cplO*QIj{T9@{m#Y(M$`h<*`U+ce2XS&$O9juKWEiowi3O z2b)GB?TLM`W(TXv_erdGM=V=ITGr(Zeh23T_4a6mbZbi{XFe6Ry>I zze}}*$nfZ29sKB$RrQXHf0M^_Df=L%v^Xp5{=tj@?7Bt?e_w)N*EZRj0VoJVw`m8$ zx<`Ime{*&tz%gH?>vZv+-M5f$#~e9}1s?m-QU)3ilfDv(MyREnjbLXYauKnJTFa#K z(){8Uc(!xo@Qsmb$#?{0|5ei!rMZYBLR=TKIka$}-@Wb-2vVhq>D0IEgKI-}ndE-d z6^ewd;*nC-jbr_Bwsn1#F3BA_QB)Ag@7f|3_AGW?i82<>*RleBU;G-YXvwk(x%&~> zfFkd=Tj)$!lF43|aT({m=02Ercdt~Vvj+j71(}pBglehc``x%7a~+ZR;I^M+GvwSR zQZ@O^j7g)k7d5bTcAh?PevdKFyzeb|%Uou_%WGAZNgvd8NJbKZ(6)?&+2G7RmvGo& zb{Su!=#vJ{I~$H0HehhB&nF)o@zL>Sb>_8t)&Aqdx~p57`%cOIvgDw7m)ZON;yJaG z&6@vEd%}Ld9j4%JmOg~&kPz|o>Q^S$>Sqpk>PRNHIovYYNbGYdo>lU{$tI=q>YXvz zwjUXIl!8iybNKP&N~_-G*D*OwgJ^Do(@1D2&7nQsMvdOKZ;!Yq%`w@722YYJIsa~R z=`75z>l4svlVqAObgYqpvsbR|3CsFX;56fJFVEGml__ru}qzO|vJt(M?mr z2gXo`_+ekW8?Qd9vsMI7?<~eXSVdjYv9>8WV-}NhvU8nVvH^eE0F;74dPe$O`{M=N z!5GJ1(H?|vLw(Ui_IyE(a-V>`km;22#c;m4qU!CtHM)Ttmg{i!S|y^ltFSnB8GU{F_+kuHJ9^~YG$te@CICLqw$XD z9xh)EnKH36s8BNl>lWJ^)Ba@nrR^d;6zXO70X2}2lpgi_O>W^*sNWHI%%an4NXb#x zDEVH!L+&wDt*+YDv?fnuN`fXIqLi37mcf$+!5SO^XFA6QgAC1>$~-} zV^B(_qX(noM`%IVSn_srMV6wIj@ecf!39tKj{^&c@3xRc)d-{n{AsO`u%=4!VUdv%o5ZIx9`(0ozAd>_=9KC#{QjJQ(BaBz}yHU|+I>?`i=oQ*P(1cfnpl&RDH!NC#_05ejo;8st~ z_d>kDO+3V)DSOKqob@ozotLaUy&GqutXW~0<`qVWWo#F{O1(&gM>o7_Aq6uhj+a>E zubwk4a@0GN*AVB^W|Y)#Af9prg&CCQ=D0*|W%YYeCHwuVVibLn?mt9YYnl2AYI=Xr zUnHy*GVxCy)lb$vG(L0{F?zo^&G1<3RN>+xXQx<4{k110@9PQTJNEpTkI~Oe z&tG~=#f!)+@~JNAG32tas*dUr;D9EZ(B+P zpFBdCt};-yX(T#_H-KZPEODh=TIudEfQ8fB-cp{E5DKrLUPfeYYb%0m9^zif;T!|>Je$!E~4Y1 z1{_xxw=uBPT?flZ2^fil-|Q?At>@$66?KwI@yj%Vts6aGX#-VV!%{Y0fkd!mGh}Ds zO%3lZ%1s&WwSl7POSm!3(LWm1f`%qGq(A=vqgJ`Xnb+Q8Z*s3-{8wk*#(ZyD;KJ-a zlXTm-&YwGwkh<}^CTLStDt_xQE?h4Xt%)$DhA@Y2gqPjX`e;>H+J*oLdHTVqrVyaz z$q_0T6emt}8I2I~KY}3He=TQ^d$XY!_?bh%HxPLV`JonBS3O^1NhCjM|hWTmd-nmOw ze@(N{$p&ua^St2Ti4Z#Qx4{e5kv4j$Fb0zK68J{;$*4r1;k@jjiJBT?XnOXc>ep46 zCHTg#?XGyHog}G2xrG=oxuJYS?g%A(q6ej-yeO3ZigV)JmS$t02ffMwVLk+_GIXja zXUR0w{jz4yfC|&)TOt%Hprc{hc5Z@GZlN*kC4zPD&fgUAV$@jE@U?^T6%VEi>!Ii} zE&jO3H;OvshzsX5gu5Gh2*KTCuYg4j*bjr-`1CIhf3O}?awGbASNO~U=#6-}zNhH? z8=ca8x@YwyMw@pNKZs;wak+mcGZSy*iE3%UT>O|Bti#9ZHGL7qG2HI+mCZ~SoNje> zw9|=ED|DJI8g-mob^l!IdS6;RwDk>p1u^qeKBYpfi&<@C#LJNWGNXd?q%=eM)hT}% z4S>}pMbQM%pMr;zH#Fc*Gz+|XSf&SKIUjf(BO4(~C*qqX4<2k8K72UFrO_34+q9JR zrVv`naqy-HP)KCo6H`)U%Yz|4Q9DEhnlt?Q>qbah0yk3 zV_j3Y68J|0(TE_L1Rygz?9q#iE$msFe!Fb9G12OFQDm{Y7mrQM10LlZ+#j|MKj~y~ zSA1e+)dEruOhLSMiUlaHtQA}|oT_JTc-Dd!twv1!6v-%VEi%PWC5B0L<5IUIKA4QD zb-7>oyH>uGgbS{TQpmO8F zWIgpdsra{EyCkK3+mMj+wnMbcv}Doe@P8}FbOngGAueZ`O9fXDpX+7AIc$fhd zJTM}@KG4&1RQvy^nGAyc$MvHA@(mYr0((ziAI#E(7vRc#Xtihr6~Sq4;k)A!I|WBbHh@I*KqUhv4w?B;kTRaJKg+wX9yY)ukgF7{Z!)fMr=j#mncHA+Mk-L|<4YlN6=d*$ z!1cM*VWpNyLYw=aL7{>KWuWSk=Bw5D;Y#&P2)w?6!0UnHEoA@I2God$XCnTG*Av>^ zynGC9`Jd94nuRq%W+3L8Zu+lwz!gq~w<1UxEx(^C?%$TT-d!DaY-RilXi6=5Uj(p7 z+1tgVCuq4J0tEB^P?TEX87Z*#jEfas7;;P&L+f99i^q3SX3UvNQ^6KIv=oK-Fhv?8 zR>rSGIkgwcj0NdPHTZe1j<6+mnF{Q_KgPGZBJF1vRIo+(>i3LE8&fL=Z3 zVY>=AZ<2Z^#3d~TcDqgq%=yCfDreJhRp^Qq8y7b&xFySY`glim60_CgIfK z%+EaZy;~(Q4S?0ZvmB-1a(OXD79*GRnN-6ueFG}Dn~135&e%@f?uCQ?_e%)d1zowO zvqlfsWz7|B5B~eyLI5NF<**3kIG86({cZUTOl&L1=b4rInod14D`6zBS}wk;6_ ztCKO3l0$hN;PN?>wT?+gt>ue%#5*08Ynr!vJr;hy3C(}KnIq%U{K>8>5a56{0vK?^ z5$yMzDw8)^i2{~<5TjAstdo_Ow#9M14?NX;PDnD|LOdk84Y-r+?I#bcxu(K*MD{@q zjQ#TU?QBt3$qoo9%x{!nfl?0r64Y5@n-PrG+S=|01_?caLpd^i!otE*&zBe>)|5<= z9j8OP+J1JfQ}AyrvaYYQ9nMdWtbKew`yF8Y1R}+4zF+`K)h*Lb?OL3(XRG9Tmu%q< z$GFGo;g-WBz+~;eizaUr&7)e0u)_pJY=APW*8bw^mBTxPtS5(Cj-Sa%Knqj{L3l&= z#ZJo>fsiAow7!Mvxv?aSeP&a4KVx{u?j zE6PgiasriCAxWr#t#~{3t~=E~k72_fyX35aCDJ_PvDM~Q(Z4m#4q{QDsMQBXm*xEI zWnlZ=M6_KU?9}n@zFhNpQw438ZDyB0eov0|F_PMb(=v4pn)kBfdqR|EGpHxk}RphU9lImh=+=-VR;0&@JO1WI#{_LwI7`qMlb52|H1MZdso=c5LH zdx)=N~J675-OK}s#-H+f>dU4S$0`fg8WK5Rq{ zxFkkojL|RE*i5IfMG+M|uuJ^RI?Gt~G+8@vOL@oPR+ssBQaEcLq6_DeX`L-cv7_?l zt8}%Nw;Yb2g)Qz|{8JvZcFlDXUw7kcxo!JopiQFd&ir;8YW%s!t_P+4ZfFEkHh(+z zHMB_v1OAqxnIa!lgKhLV>3To>?#kZ%-lT;{`U(E?*7;+-fow4qk&wyLNxVde|0yvG z4FueDd)PO0h4`~&lqOG`3;d_F2n3PKQ9JMoWM=Z*rRUa!TRa^hfPAsrNT^NBG)r&( zo)VF}z})rdd(Qg**o&0X2lQ+T_aHBU1+iC>6nKiTmuSmr8!_ha@hqr`W-bCSj()ug1(#8&jujW~6JNDMb zmFkB>VSj%bX-t#4ixsq?N=D zz+^Wlm2-$9xaNLgazpbs$~cY!1@?q{k~qV~8@io)7q_a+(6z3;mQMcN{+GeK$m_aH zXSc63b4#Vv59m%n&6A*pl!uA!`{(?L9vfqJ9pmh@mO_~Q$1lQ6s0=+5-C(BsJ#yfh zo;nae=3IE0%^X=Z=h@CWuj|~?9I!*<*(cFC)!&3=-(!SW?rYhDD?4`cA~_sNDvAp6jn1&! z6(uMGE9BX1uTlc(BJYO8i5-9lfrDW=1wXPqe~`n2#y>#nh&7ZXgWue4`uR)zQI3vc z!`@6c^gbV7IVs4V2cniBBAe|7Gs;4fzg1~>t6j)9L{@o^-tPgJS<6_(qlvZg%^raQ z>Amsj!|Bg1wAqg8XfRv=z?+rQ)T6inD#5-LmjIh)Rt23ghP!AYk3XWY>BCZ@7jnaCSXPgt7YJ zTOP8~_K0}}A-bF_Q>RRD{N<5Xp^A%|dcwn%L^={P%a8j#USCwPK>Hopup&WaG#mO0 z;$+lrJZOP}pluI$zHZok^K6dD0r@7!k^LyG@$CNkDm>i6`DDS%hUlzthi6%Rw1xk@mwoD?+8C+>ad4dWtnr+Bs5pKRcPCDh=_`Z zSfWHtGTjK*6vl$nVJo{#lWX@}dS%sjibCPR7Q%D}*FUcc>?`6XXdt$49kZQBOUL$B zhVA^Cl}P>oBd#gsH;0Ow3N9n>TVE&$zv`HidZGMYuJE}t;j*;8x37;8 z2C*W$Fz>Sa2rgENvxfJ@D{l$T(ydY)a0wao>^r}^fj4^RDo~uO4_f5=;hh$4BIp{< zgY#PmJ~0GE&lWDvpvTmO2DL z&2S0m*TPSB3ND(#1dJU;wXyohAmFwbx^?`H0db1VC7B@}eh?1Z7Z5vd0E|)|qRy(^ zgV&(a%)vb8z6pr=pA5=ng9% z5Q-NmCI7ujvHbJrI;$LX?n}Oc-}wJT3o}$3RfR`$+4d2g z5g&$7GTEDmmnNq9ov83DwE&%oYOC2)C>fy}BW%fs%@j@*v1ac-e30k;w;fAD0>Yy&$bJJ{;AxL0k`DUw=n$@x;fLLfM@Xl zM&3F|#c)j$!25GC1orVF7Q%0ZK8gj+7*+Kv$FcL4;Fq1GCz)-7CCwu@v+`e<WvAF?9HvWZ#Ca$qVjiK=V=d0^bl6u1O>i)o;t>&nZk>)@9<~DW>(PDm zl~)~&=~ncRgrL^rt9L1%n3nI2)7+2L6#hw!+(Ry5q+@6$MH9OvA2aNH`i#4m zn!be(_N~649=`y<@?YNmIR3>=UunY|*|`iyS0*_(vZFI>Wr;(l9tSxasLGP1XYlvJm!rXd`{fryYxUF?nHXLS0&P`DsSVN1w;j-J-ue4;?vs#x00pWmZ`|`TTEy#m`|Bf{%Bf zxY<*O*AB|Boe^@JQZB=2QTx|myY?u`i{Pj69C2E5sK3M%0+ zWD61!axRR69!fdSP+oTXNfhU!1t~%FpS&M$W!#s!S7ZQ8@G(-1t^4*T^~XsA_vLUt zE|h=jGN4m&lm9u(2_FqmXIn|3tiv-bNiW>^{+yLh9v~S-pX`S6{#$FYDnaZ#olS&Poy4_1lz0ojD0~pII78T}_y3T&V5+p=PorgXg?pq(UE|oj^DCnr^S0GOCUSBrqXOpZ z0<^?*R0-Uj46Dlsr^=>+Txx(oR5ouUqF{ae-$W(Aj|}L2H@B*k4O1hfJ>@@nMQB|= zY4hKp%R&_5m}>8XUt??3Zg&Xpa*@ts9qHyHV99`P++@cVL-iUp%tr}XM8MSL-jAYSTHTmwIWd1KuMIPs z{(KBg4*Nh@TQ9xE`vI&_Z?q2*0l+R5mlGF*sV0|?UUn#I5fJ$Ixc zWZ=-n`f_4%#)79|?7=V2VMogsXWgY`7e#K>?-?mWFJ#=u+K2q#DSK{h-Y@J{=D`xm zHA zp|{q~w#^Y{$?@%Wx3AK3XH-@sS_&&T$CW56BGDWLUEndrp zn_pk@7X;w&v#qmw^&LamQeEgT!LZJsoTGiTMYY;~*48r>)!sz(8z=SW{Xf24XF5nP zbPN;*oeH{L!p7u{Df6(&l z|3d&-tHz7&Ht}d^8g@ep`X1!AF%qur_1407 z*4KUiRa3L&`1*U0@{}9o3CW1y%>MVZnME>{HLKR=J07Foh1)VfGYg`~J-wMM-{$GJ zf7L2q8>Ov$1h*gd6DQ*Fy9>tS9iKCa#Mw781p;Jl7%`Gxg)%_DthmmUwQo<@*qAAP zOkbv`l+AplMQw8n>>#kFc{}!zl@)#<7c|ra;57phl2p=Yp@kc^0D_wNQEqCW8FYHM zwG?`QAhVqi z5e6v%W?-F4@?ddLq^uZb6Ep_pDlDPL>X&z?!Vm#A=snO*&OOZ|g+Nq%I!my6lVhu@ z13MwrMx?!0!eV=Vq*`-31^cRKL|rFN_-nI6Kz~$cYq>oIo7C-3)p-&UHWWFmH-$e0 z`lpFmtqdT4nb(R{769wl3AQ?Dj^l4i2<6I`z5V;h{R?=SVmg6-Ep$=@H}kuL!+@n* z(J9o~3ol{4^M-9n`C7Ld%z|NU#Mq>Rqk)#bg3TVJe`4Z#yv(k-6 zo1KP>IwY>Ks`8oU#X_5Qz)ya%9brG&MNtO|Pc}#0O2b36k<>rmCd-Hm32g&><7Y-j z24CjTvc<<|s?QbSN3twpI0o)Bj4#^F{oA)~rw`dTnFFp(IHVov-FIi%dyB%mSN4c_ z{qyz{Wdv&PMQ3>-LJiLd8Rv82=~typS-#g}W60@W(29i>3gR1eko2+UqQmk} zjGrDCu&E-QS$+$hd(IDL+%cTjp?li$Q?3qpuh>x#Mwa%HgOkBe4dvXfLUSv{k zDpI#gf`;g(!os?9DHcA0N3_UNt4dxCaM%5g)yg(|`=5_K0 z#cho2JH^oRQ}yt6Tb}pj#Gv`a)6@a}wWpSrE;9lZ2K$)nQwSz#E3{0gy<28&Wrnz? z(QD)$f4=?^aH5!*aH~i9FJ(I__YV!JJbv;ZnD-%0Q~7T4?}-~zw#*A=;kh?hKe0>r z*JWpl#3@M(AD;A?aSuNZL*;+C$um)>1+K-0SX#Je_MLW9OjV{5W;=gQUDn+A)bR8G z^o1ug9r}65nyJ>_qt3F9Pc`C%Ea{@NKdsd~(k)+Z+c2QOQhuVwAza5?nisRtC{53+ zFK6-b5qVX|=swp4;~(In=g87 zZ@QD?mfmV)Mh^XCN6HDpgDAwJ{q8gQM5qln&siti!If&?jYEMxCO{-#;p%M4G15Bb z)BHSw3m>-!PWgwve40Tb)~mU2OYGTZ%9L&#Yil#@C>? zNf|c=^Htjb346yY^8=h;55c}Do?+{hq~6LT1*r`*H>lFI)%#3iPIZ`KB60Wshx&b5M3xe`7hx~TkK z@^oGfCp))HbZGnCM`u)?kzO?43&UCNYJ6Cw{~5t(EpfqvC0++eWP;*-A*K!6_DH4_ zO^2S>ru7D*pq40OxaUm3a<(lvFU12*>F*Vh>%&v)=2DyDs;u&PzFW3$xdq@@fif~O zOP~LAObBIQz>)9?ZxEiNJoD~T$#_yHcIFvvUE%UqTbHYMKYmX}=f=xL8zbh&$)}mO zmPkQFbi8PG{q;rbbM6Dge(I2VX79LxF0j=QY509k%)Ty@r#{Pkh%fvHBC41ZmEd$z z^JNN=!t8QjO^Ys_lHbZC29XerXDp7+OpAbYk~M@mbmF62N0ogAaizOqG-Q(PUlUR* z9@WV1eFqo2$ub9!3zc=td`9G69lnPEs)G9GO!A?q!o_!|)_SiAJ zN~=jY9RNFZE>q@_0LrQJHI9U;C56;<gTO$e^RXaE+kiTD)KRa z3QI;c4NmhVv9Vy)vy<}n&I_eXo88bRfGP-aVpj>&LtFE^@;fO`G|4LeXg&a2UkI`W zNabYNR2WYZN1&omDJwM#C*31A*W=Wp2L=C=meE~X*24T>I^_NMG9zMI4cTRww};Nl z%ezRvlup0@Ct2f zy8l!ax?W0jE_H71rud@H{(m)I^W~_`ap7Vk>;Ks{3isQVAbgIh7kz3xA4F8=6)G#Xo~NoT!sT5r9vw{j6V&3-9E-stWbP%<&G_VHd1*IRnw61|k1 zo90#A{P8D8t!92`7$WkL5GFZUhv4TQCIzNKhM#VuhNmA&F$VlmN~3vE2Aed(;}^yw zqB)|NAq23ptuDmT=a39H}E+?(e*hoDs3XNX{fy)W{FHpM?K$5b2 zs?G^D@BqZ9i6J9cuI}LWbkrpkJVhB`TK61*rd@IT2y4HZKN>1`Hq>&pw6y8btm=W2 zaEw_5z?A}QdxpO}RH~ySA+(Ol*KcgvlPfp;w_aGfU&Unx3E>ql;b#UsSO8jh79cXU zA0lJe;wAthg#FyK&qXlNMzW-BR6kT=A7-Eh+!n?@^n<{*Xecgt({r0_hCSX@e*9T)Kmv4FRmo7En?M}7m<#J%_Hdk@r7N#L zt**A-S{18N^blUQQQgn3)+mVz>waD2)iGKyL9&}zc2dCR*r7kpeIfZm!vJX9ruTn$v2X5@nT{t(>J6#ZM@~82?-rN#jw*J z9*Q!Z#`YAb|HMC#mhRiIG|rlj?d zo!8!3_!sNX_;BXDaoBn{#7Ij;Mug*Pvh~&TN&-n^%<~;@bCL_HN&Kxw40u;jqL68G zm@q5Y@BMZ3smu815#Fma~Xro(T}JFyS=j*GdD)5t%ee*Ynb3?xj0e9I&vtw@QE;)O=kOXV=jHKjuVTitM8PtG<S(UY_ zke>lku02XHjnsK{!JRVmq zvv9H|^+?~wsno~4n5St#BKPFa9yWTH`XJ!ED(aPQlJ)t3%S{|#670eU^Y=h+#M zT1p+{C`Vq9XGGU|=xX;)^Qlum%Vbsk-M<$PxPVh2T7+h!27jD8(Q?qE8R+=br(&W? zE7W}&Q{pg*LbuLmn0rgLJBlOw(qVZX`*j_4kHiDMr-w->k4jY!JWZ=wx~Ls2u#04* z%V0ioc>K_ZV>8jmD4T{RbHZ`Xcyl|VKxXq!e>Gztk66o+uhdM(Dj9FLbNYs{;8XPH zkGp4_pn<)LU#s~l;JmxYot#Rk_T^P$)C){a0Q@2zgx zRSIBGONFpB8$awOs%)zKih4G~XbnyjeywB!YnrMEOHyA|s0lqP(61F;yNBc%e%<43 zNupWQBkS%zpX4z4A*w*9-(`E9CE?jlPHQ!%xb@z9(CSFUUN|~4o%2VtZv+d|=9IfJ z_q)Vyp4s!8O+|apr<+UMAJS#Alc!qo*zO(A>FW3Q%Zi`TTGmz1BV)) zuDG8)tQT1F89R(?sWoCs=^o+Q^1U^x&*p2Wgj9$YDY-bmXhg*KNpF2l=vQhc;Gj4Aw zPFu<-gk!(HJ%jCp<=K;vNBTo>;t+)m`jV-4L!ef^YIYrD1XWRjBqmyPB!M?A0+ChL_^cI%r%?vuQovpF(X#! z!$E!wvCJb#vU3RKdX1jg;cKSOFTEr}kK6>k=IiZhh_@`Cw=Ngc>S%pvcx9(oC71Gf z$3A2~d`$E9w(@wU|-@BPG&G{5+E z>qFN+u=oDo`qfYT(0j{&eDTZo-@9|}@>{N2U;o-*mQHBZ(5X=D5AKmX3Jdh^#k{zd-}b3gI#e)VJL z|MGwR4z`gc;*}a^8J7P;otr4){k9#^wVGbz`yuLK)jo^ z&%g527oBqp?|SR%```WIr+(^*_rGcE{Fi*k*IS>d zfBKal%>BYge(vLI;V0kWeBsOf_UX6JJohV?f94PV^VU@JML+xQSN!xRzWy(M;%A#Gh|7r2Pzw$@_;M&i<HJ z+ZU#OyZ+qY`S6_=z3*i&y72b)7n}e67uNp$U(bHmbDw+D%EE5|LVd?~dOi-}|9wKKo~XR{h!!egEJ1RPc4b_Q`MmkG7lpN8k3^GaGOCrI+QK3y=KS z-iO*BSeg0QjmA5_@xOlI@#~-ZTmNk3&%eJD{tx5-@2B7OeXSq8_MXON9`%)19f$ zKY8H?F6{pNKl$@dPksG+zxQ1){`D_@?j7&^$VD8?$Nhis|9$c^|MpeR`@ZDgeZ{~0 z%>Vew);pg5(_eh{!s~wV{I>)@_>n*Q@Yuip#c%(AcE9j}uUY=!UwqRCpZdvH{(Sd6 zUxH`ae$$73y!fv6Yi_>!Ee&AAKl-7u54?H#y`Opeo1gnvr@#2<+OIwLx$pSL>tFh# zFZ<@7{|~=<>cipgS32ML;s5LRU;M};SKt4V_M6}FCvR(f_zNHV!H<9Emw!#`Q~&3q z|8eTYzvuqqi~kX-Mc%yfn_u?c3ApsX_5H72od1faR`Q>-a^L?QpZfE6|BZkBnU}8r z(;xl)&%N`je)6-w^X+rr^R2Bf{qdboy<+~qf7=KD;POv=wI{yi{S8#? zz3*q+uc-V;;{)H_e#Of__2aib_(O9WcmDJz@!tLX4{x6Ql8?Rr%`0#7KKX}V^`Yx; zeWbOI(F5N615o2!N(%^&h_zw8y?`DKqx{q*0Q^Z)CM%J}+sewttZ!MBfX z|EIV9{wr4B_OYeuh5WnkJoChz6IVX*)gS%!nIEX${HgzP={MiA{%fD@eDvIZ`mWrU zfAH7d^Ityr_u=OFrBB+Q`nMn4?)>$M_gsDSC9}W#TYotIO&|OC$A9T{Q2EVY?d5<+ zc@7wbAAi&T)_&i)@2kIT?NzUO)i1yFH@E+fZ{PpIfBVbN{>bvlZyft?pZ(_l5MF8) zfA9k0`)h&q_`V+|<^`$k ze;9lHwkscetFi=7zTQN?#7j$ zdF}_)n7-Gu7R-E+&0zxuN8cr5rz=q<{*1fVgS-HO##Sjm(UV&BtGPn0?!!)NPb_qiE?Y zb&{>3nM!IS>5?)Fa;haiu(mO25vH}^ZV|RvPM)!`>pBmu%$e~P;NpZz^c|C21(<@6 zvkf)PikxkC%Z2gLC@`tf4(GzdoQs%ZN-{Q-Pd+O-5>+0KXJHApoQ_KgWNG{iXneF~ z&WvBXwyG?rAEK;SQ0FQZ z##)pelfZRD%z;vsT$sVyvucy-rw{hJv4h)Ah-V)_gkeWX9-$ECVw7zhxJ6D8rqVwfu-DxV zE-1`~w{AJGAT>;HgM;ue7h`Cx1^#WQs@tZs+j4@y--gj_w_?abuuf_0g?az?Oo?$deQeBIrxaP39Q^{Ez>3Go zdZ%qnR_h@`wJ)y7#aDMJne*y}GMBu_iMe>m;43Z*G-&CnW0sYyJLb0P02bmwZ#|Y( zwtP~CJ1yV3#@b01f>l+9@^2QmMq_!q9$Ms?z2?uSvA2#g|vK0Ky1t)Q)7L0 zH|8I0HsR)_^mUjJ&jCULbHMFcH-Plv1ohS)zuz0%n~46&cX-HMNJ(pY+QO374ZkCq z|(NmoFwx!OO9ZHEoR}ciI_;vw)RoPbQCnT1KgH37{x# z06j@f$9l61xYk^5+Onx>8_!DAcC@UxMiB-X8+!tdNsBWALJumJX+@8_?fUHoZ+55K z*@Sa;DexOGdZ5OePOxL+D|m_74RH-GUO;l3zFl^316(NU2N%k^6t-991~b}r>zIbN zPt!{6}zX?pxZ^u-m?qwImj$n z3fm121_Gr^2o_fXB;Zyo>sfmAEb|E4esJ5`z_Tt042XP!gA?0_m^9YWnSeXnzSZ`C z@D59sFfY`T<+qY~+rzd~a~acNG2%slfmGZV^ji-jR;20G>kXH4!4q7}%=rx8rW4dT z*CwnH@;z{bOUda}m-X1#)3^^RR&rvc0RJjl<10@=M~=nkaCN+pbxUwzna+5HLTTL# z)YQ&4V9{E@5^cGSouXz9yObw%!@bj^#WKCLHjtgeOSc?r6vxM?QpUt1f&(HLGz(&Q zTf(=!({DDsH6=xG+MaOWoRs(Va1R{;{Xcd3^qI3M`v2@m{&yt*b^&L>?d&${pW8%U_1ZG!kuWlFBBNR`-Bi^{tV@$v9*6m6s=(_qL=gdw%4S@9U(uJFt{`*Np z{P1SoVJgeA_KTSyg^ljU{Q-))?gLfSx*v3|dnCrV1}Kyp><4o@@LvzN*>MBA8+i8! zavXr+VIBY#aHkJuE(gGclH!LCFciKnBP_>k`(Z}nhF>EA<3qR)DRpx7o-yS80IlP7 z8pm$A3FsxLb=&~2A=wOk#_$2t3O;(GtGm|l zLSUzk-NE!*0N`5pIO3ED^kcWvozAhlG7;wEcw-{mSG=&@aCVLzm2w;jX;RZ4H>jI# z-Rm}w8`SQdj@vqRbKP)S8(m;$jvL&8+i;E}ZBj$)u46dgq2CQ^$8VQ-8=IZ*s^6#| zKVpz!Kb}zETAV+Al3K4-^V&{hzP06s9dF|}a`u^nIBxVf5QoPP(-6?4DL!@=DQp<; z7*Z}8@M9;(a$CpnI%24g+g#Ux-`wznont68ynIYS@Az%6cFeZPZFh(4QI6Y9NtRp3 z@p?aQGmTbA#|>sQ0!o35536qL!NUxHSFs#$#O(ck&yKgu^VgKod z)yM=iEKinBq5J6l_y0P7`h42|Yiep_|9K?-Ul#dHbv*RdAjjR=w%2W1Td@3NO{d+4 z>pHBUVS9j1C>_tGUzX+7f!+4tvJNV80FkPwSc1NWE)zNR4pXLzNTYYbZs7HOSw$)csj55a=Vv<^Z zM?CXga*jX;=sJ^R7WqP>M?Fs-x)?aMyPJ9APV8M@)1v)OjHWp6aq91GX7L~T#PlzP zbZs0wd?F8+YOPrI4*Z+Aetn`|-`jKBer>bb^cpY$ww_FHqXmphwdpjll3TA>nzz6P ztFlmikF{HcC9bkt$Wg$Ay^(j%eeSG3vXVY}zGwA&@As_e?)_e=ln0mSa|jhL(Ypi_ zh5cF(`S3%07+70Y-6RENpP^8(LGP=5Z$hPbo(S>gcNzzr72*}DhlHbRKK%0FvN3y5 zg*RNkN&!d?G}kq+vok~~zuOwRTrfnzz}+x5Y437CaCY2biUqFQ8C(uS(KwAN`mXjb zjE;=SL-erR#KRpCrv+G(P(et}nenlZqZ>bPT25obfr^3ebAc0^-se3rYn(i+ht`MBC0zNG-@?DU9(r!Zuuc>pSR$TkTYdWUV!v0I;JJs-pN819cE@ zMWH(=Ko@#KJ0ZL_$0)#L4icrvhX7VUslPN%=?!kCR`1@zd+AM@9!Z7z%YUYhNd7Z< zc5>=`O8zrBivROq zo3QK)Zu6owebKTLkIOxtce6`pKL%3DZ*64P=>2XY<5s_@d!&yFtwf-Fp^J#s--&yrHF&2W)772y>{``f_)}wlF{#I&?Gr zC8k3)GlW=msCI^xy&j;k{sP(|nz?%^?oiDSC+fX7IERu44|fcP5*i<*sRzT)4mqR4 zF}_1JHjJ$LAgg@10_}U-UqeZ~hw5Yq(fE+)`u@t#kAD9QCt*1V#(p?N_d`Sbf|TzM zH9Lcp07Er0NK-IWBZJflLp3s3=Ww8A?iJ$>)?N&U{|BfzhJ@vQ>Oqb@QX5hQbAYxE zrIG5rllu>o#%Ji2y$`CP;rkv+hjf6U45w^5ScAb~wm7W%YUl=s)N>u6z2OvMLpM0I z=4^QK^8u>Zjy?9h@7ld#`K&`~X^@_9s73}W6NlWG{WXrmH8MagIaDJ9beF@;>`1|R zR0__a*JMHvU7x;((GeQha->@w=~hSoj!yqeva{MD>3k1j|9>i#|K;rI(+yv}YxdeTczW&H++}>U#6;}afm(*9B_REpYhjuw zAm;*iz1!e)YZyJBlR{yJD6c^%)a!6`cxQbb6MSJ_u1L!k60n?Lj%q=pnEXqHoQGSx z8#us4Hc9L4Yn1NQRMIsU+MrhEDOp|MVs*!Sy$Y3r`rIidacaZSe=Vh#>VjW zTDrVAe{=hN z+n-cT+QJi0%-?td6EpZ5RlfnmMz^_!_sgVZ@=5#@mlVqJ zv{@|Sg^vxd)snn*Zs1U;Tx=}_ilt(mo;rW-m2gbW*6@wl7LIFer$d@*tl4n4+(yN^ z=5GMORjJJ1czi)>I$Ms{pll~p1_pBL#%pdYJawaJJwCfUd#z~AEiNuB7Okgd7vUM! zELvCQ9=r7fk9Y3c?9%f5<)yjV#miS=?yt^WpN*e0D(0EY2-0+*-Um2Wa3j zZT_V(u@J!3s92kwPCKlW%e8v2iCPDz)NM6NZoR7x@||V_wn?~Ew9qs`<>?72anTrS zDu!z-$2I^^)9Kbdzi45OQlFQe+~OJcnn8K1XV47&fjpf|r_bBq=W_ar$7ytzt zE;R5ER$et_;-+*nv_ta|QdVH#j=|&|)f%d~QqbYkSgTmpqNf59Rqg@QD_`}>(`Bk* zRPGH4E#|_l<(s#b^-|I^xoU7Lt+=f%FYsGXwX<3V4AAa&$}5O{$sA;rFB7$G*W68K z%Y$u-DY2r=sGO4ybCr2~c~V_C=%mQ*t^s2fhM3?K8=Jpz`P!{3bJo($xy$p9&tJA4 zpT9P@Waa0c=I1wnsF8PbR|*Ps%1!Oo1ydr)!#Zm@a&-U0y;gHWKJR=R@fmgRIJFME zCqBz^IE$x|d6X*veF;d%C2UE_h}3J$CCVsLCvAg+*YGiSZpg{@N*!Xza!3UMGC?jJ zX$VCq$Wtms#VF^WT=M2I%Mpr$b;VoR^(YZc6_YNh(Uwb?EOLUcnlcu_tp%mb?^p&J z*b1CM4PX(~fbm@#AZYLm!*~23Q=>ku(Q9Y|KVv_nJp^$;6EX^F!T4^BXg~up3TnXk zP7Op;e{*(m_WIoN+~ShPwI@^(C6+w^dc_r=z446NDJrE}$oWu#agi&KB*l0hXj^h= zN`Oo>=tkgo+fnx3Uc_g#5u<)EODofNPPgM@lGhpwGJI>z@94W{-N)@Ef`S~$8KQpl zBaOlJlL!&(M+Q2;=~j>!HfEvMajV@xeu=@C6Knvs1A@+Y*x3Z?5Gcm8r^Yn0iiU=h zj6$1+yW!M!OvZtZa;$T#DxEjF8x=gx&340$gQKq!jdRs2b*1zs(61oX=(lmy{Rnhj zsM7mFgi@2L!!k{9R=3pa&A_jBc`syl3Z>e*Ol2g~dA`bj7<0jWd7t_iApbcvd3Gux z|2ci`+(`a&B>yCilVz*xiz#t?Ect@)P19)sY7NqFW7MfVHr52R0ffsHdOJo*4@YP1 z;XeQ68YP^qV)D$xMFj)S|G6{glIQ>2DJVWV|3`TK$6Nkwf5QvBaea6>0rAsKp<4qN zb_tM2H>4nC2lvS+8OafsvbC<4fSLpx(y4ZwjWB-)1(U)_cbtY-%ZL0MH(4zV^Y*5< z5&(91-SjvY7+jH6a+#KM~lci0LzbYZQ2*xfzu}Wno7&sbZ#TF((vk1G{@>u zpjD9}c44=v?7GCDdrc1rc;IV8CAC6o9n)qbi;GgGma*43)jV7_z;u%U5m`ja4+|TP z`6g;3b3<0WR+R%a<~d7yG=8e6*E7KT3OPRJ230HZ%dF0DjS{ArXU@oeya2kcF+!={Gcm$8b*OK3M zTe?cY+O_u9_Yz}UFItaQTK-zYY26MB6eUd?9g)Zv{y5f#k7Coh>&(RZDJdXelW|tx z6DS|ipqVqE>X^F)SxRCk$!>xKfa0vRgL{M0pnI^sn|>G0=9;VI`LI#QnwkYN#%^23 zjQ|g@xzNg^Om6DbAvr(^kuCm}{81cq&u# z#NKPmWwivT&9EMQ)Y>UE+}1{iGRvPCi)5^r_jzKHG+(W{hsI?rn4yEFZ~)_@Lfw{|;n?S%TYap^XoiJr;t&>n|rmCrhT77TL|S&(SU53B(v6X%`R zh<;#^uoS!=$h-Vh5vIY8_pqZ%!`J?0*%k84IwTD~x_WpEk zcWd9a_Diq#y;eRoe7P97sf_)Zqz#>?BMKkhSr_o-#nQKsFE>G<{ROsGT(Wgx8Ca=DLd0INq?GtEJh^h7ly(shA`P29JPQnC-GR; zwQ!5#bs*<8T~dTSh6`J|p$+zS1G zmgzBJ5DN&!%#c4G%5F%?#&!_`l(gWqcC-bi@EAVYGUbs_5WOml-}RRLxloZlMB0;d zgNPQjtShz|RK*MIRiNhtB)b?{JJFAmh?uy+D_*dIjjl!#wDmpB7&EF{PxHh$J(!uG zQ;m=SEqkYf%*3}WE!-&auj+;iG$2sP8dMoc7v{>AEm*N~yIfq@21R<&vv{{bmt_f; z5I5qHS%y!bhqqVt!fZsuzt>`vJ9fMx>Ar#p1YHT3`%v87meMqq? z{Nlc$j}kVWhbP8(@V%M|vO1~qNpM8@Z{QPkM0msLN^58pQx)48b(~@ktH#;KGoD!k z=BvsRMKecHfygLT&0@?nCJ(>(TO>y_kQtz>Xq&|<+N;_tA8#x+vk*HSGOLr)mRDDN zFxjFS?suWu&e#BPK>41Xk#s2kXoXmpNaz#RfSmxe9*GHn!2&Mao@KQBN`?tF&G1h&^SY4|T%2goO$XD9WZN=%C6V6GFED8s` zbkZ&qQhc}}`%3i}V}R5Xjgd*@A>v3vas)rjlbmCVP~?o-V}LCKg(33rp_$VYYKcp? z-JO}}RU9qAoWq-a5{Bb6uES#1PXLpGM>lp8vh8HOu4))7XReuS3rW0_Ff}+J$ z>j;Xqc63o!Tn`MeWyt%eQ2E5`1Yy11`_Byv*K4V3JW=}mVHaFp5sQP1XE z?g`HxTVIkqRW(s$i_2&aEmIpv06-bju_>j3=Id4hv7-4S?SGn;nY;~nNli<9zuRKa+UGT`hJK$@ zTaYwAM8vFh96?Yjz+2C@66VEydNquWdzCWGknO@)cA1FYFvKR=bA-fgudcL#Zr=h1 zu&H&A?mCi?lZ>8Fy&yB9ulXJqfl>76!UdEBX#!q^*8A4nQgjBaGF`M>D<2kK+v?j2 z891tTt}b7{X0$Y96UZUxCg$hoAGd&G^wv?(f8e&rf>gD&gj~kJfe{YY%_RsP{k)*~ z1Qz63M2_L%hqY*_Ka>Rh!pgV1{9y4CdH*gUd?%4>BV`6gK+5*T1aPfRx7%aaxH{4pXlX z)u}c7Fd@O_c_KudflX1MO9lM3*B6ZCL;V)amTRnozQ3p^UMsf{?+jJQ(VfH;Z!ZE) z5}>Wd0xwbFbeYu@y`W2H$TO1sJznk^1ch-|a%;vtM-TRNXK>#eU0rn>7*nEzqt$LG zLctEp5lId==~@#<4A?8CrB4b0*UeN`B#9!N{v>-K$Yu$d5Zb>;KB^P2ua22Is*8MH zgn{H6F_NK6+m&Q3BN(UY#IUP}KOU7BUrlv77ugn9va1``#~Jl+!Y<|uGts?^G$~wb zXzUBMpkALlb6{(P{QIORMo5ymBl<=V2pHi47q zIJeyvhwo5=j2>+C;lnJ%CP{?Q=w3sqt#o?I0CHB%VQm|b(IQ1~g236iTR;Iws$&2e z`Sv0Agc-z(T$a?2%4opw!()s^EIpC=Bn05r{dIC0@VlWx^V;}Hi?*P?({@#7ER%CA z6w`WQ4g)-$nottf5&AA;iI@|nnz;;-OH5#8id@Y5D1LwDBnq>&X8J-9&rTL`a)6@> z>dACMtYU~%%r*%t$PwAVbg~(D9KDI8GuUV`jabrcYikXDL~TniN|eB6P{|Gjukz`i zLA$6PC;^p@krkB4L=Aa%L_^KeNn}7mbc+KpiWWCU^{z_TUCZ34;rjNZ&y2 zTO92xYA0%7W1adMS`7Y#0byj$28&H2?9q;tP;E54HI$rVg|fDZLx${bgz=`2oZ_!L zld7pEF*iKO0?rjZb?C5~LK+OojDD$6V601dTOqQ(G_kezSnvQsbX*;F6VOcSkr;>4lO887=mIg zKtOj)BE8d1C)-d;#yXnv=_4GcnECGWl#LU9DE*k+5?MfCC15)u-Gl1S0IH`1c`U<& zKe2emNX&f|V?#2gBxaYH7^la0qeItfJqkXy?d(`&gpa=CEu)d9&*rBcr76!Z)!SV>pZxh*x}imUUkl=g_P{ClMdy z^*vNbsn=vFsGckZ$@cGMDX@X`QsC2U=HYp2_nBe;8M~hj(*Sgk|M%I`XOi}xXW{k8 z{`2VUKOk+jAWp56(nmL+uPFCx*rv?Kzm-O7#T963JKa#`kne6^A%gA zh#@ixPkZyl%ESfv->TprCD0-qH?8fvotjvBu4wvXR}A1P@Qo^0@I#E0D%4`bw)U<4 zqEQ9rs7HAm#=dqJUDd|jmbD1$cbS%tNUgjY6h(=2(D(`$ya>&TDuUDRSrvS2?UUzl zz^++uz&{~y(t9zo3?7sy=-TwOWlx0Utas5`1;&IHiZ~=Iw+o;9 zCF`<;NaUxed#EHDZ7A7SlDcFox7@A)*S;7(5`HJAI9@DNOhYj(SXSiTeuldfd(A&= z8+4ex3H9{`1@-_wWse5L4YRc}jv#OB6bTZ5w76kVl4;o-D1KI!Bsp_>rTS0F?!GY{ z44MHC3hjnIf(%Y3u*PT+?!xn*tiSjBpU}kNFaZPZ|Fct5N&m0&XC_DV|MT$vCsIBZ z?gDuO<;2`&yH7%sE-6qun}OfmU~5^4KWoTqd!b}%lr2pUh?V7b%TS_Va%8yY@OqYe zBZ40AOcuZ+d5g?23lsm-+%L>E&K$H^UC z8RHyFRdXF;EyUI*B7{tGi2p@z)VKZQLS;|EJ;AZxOZ(S*=P;>svw@M-W2W3X^egHi ziKCu-a&GY%YwpG}{MC##{G4B$v*vGHnR}WH$c$ED13x}LckK%Bw2O03%sq{vZ!rWZ z1@fMKa(4dO>|@vFD5U@L)%hiB;UGy3w3*&4++&h4U6viW$Y ze8ZN^eIW*<>4Azd2%nEJgXq_Z6M0qq+pO}+&NHiJ&5w;3vWJKNV^_K(kpE6job+I64oWmHVbHJ4e9<)KISsYyOROH9m?q`MVmKmJu`d#T8ZLa zpg}@>0=Q6QY)<5jv{1Bw%3L!&R8?*t6l;f}U-QV|O5A)sQEy61#0YI8sY9N_m;$Sz z{5Lk;s28mb4Jqn|SUjWAetf%;O-J&4m>$Ap{xxPWBB{ApvDI|itBMXZZu!fBgl3yt z63PV~8Zd8>N@Zl)eVT5xtNTbb7+VFX2X!n6m`5Mw&l9kPTHQ#rVv?!dPTpvN`JjiB zO0DUMN@x5z5V_iK_<(Va^B|M-wW(mC-FAia?$CbBm{?<o+HL1@jPd(nH%69qi;d$Z*-fpZBDhODIAPs!H3C(68}vGTa9qsDZBp#VVAx z>BLRm*Ie|eaDml_+D5iBLHWv;OW(NA4_xo{nf#@APX%rH97mzkrtGi21R z9_}U6AVV|ioeJ4}ySTMyZLP2{4@JMFbuS1zhEcV2W#*Me)hNZox$RmftG8(vZ>wbeAz_sR@l2~hV35naebSalIfi{E z)I(D4b-7(tYyahT{Jok_SZ%YPn8D@=vt$Y8tFtqcXaz~X$V*QZ<$$Dn2282VA(M%_ z$tokdiCWt)_i+8X)Q>s}<7l@xg%lkkO(B^P7X&Xxn9#)7f)bOrx>4d@sG0*BV+@V! z!+XQ^$WPTO96S#r(OM*~4cmT;Vo(ur9B&%R6E{Yel&p`$jg~hvdH7c7LkJJR{3^U2 zwJK(On6{5R{Z#%?3T%(&eX*49rjs8xT_mju=Tu|I;Z&srInY62B6nt_2S$t#x;!h| zlQpNKPQb~ovs)xK%rTKAW^)bMznPxs+Eddbsz0FY()0=&p6CQeq(h?9{7K@M5z~Lhn2{ z#IbwvaojF4xP_Ry&x}o$ic?0Wq6j4pgiun^@-|xNm+rv~L_8Vno;v2CrD)Qv9itT) zt)TIB2Uy0sWaXpHfV*3rYjSWX(^*eO)epnKS!%NQI>7MWd;c)Mb{X^+z3 z#sMsfWh)?Y^qGfU)Rx#zywd8-VBD(s6;%{hck$Mzcu}wiH`sB0y&K?M6?sxw6XV3X zE^^(bBgZNV;qR)I(6+f)k7H&tG-@g-OG`6`0?a*Jl$|(@ywF%*x+L<~k}b2hXX?jG zG(-8`_l!}2;V@~n1ULzAu%XIZ^4%asds9SQbTy+|YM2}z+$ho0;>nQznwe7%RmM0# z{(E*R7619{xwDfa`R{|2|6(4Q(^_Cl;fB%epN=!z)iq*)Mp=s`IYeXfS=v^pa&&!mn^$EsqAYWX%qYXwA|xlG2BM#E=D02OcxX3sE^o)g z$EG(p4?!TMX^iXJsXDZy{a`PtNzE>2xH(=@lbl`B@bkN*COx~P;S-mP8iIcMI93eW zO3)W00E-a>#i$FNAR)28Ij;>ca+@C9$(eB}aGfc32PP+q+`mcc-7#ViL=ivYTO(zT z)~~S16D9}{_f8vE8Lg;W+N~W4-T|r{9c#!9^Ge{v{6&;-$8_H`q5~njo%9<{dq>)H z9GhH~6hSqLsB^mt*&Ou5FA-^wwiNKl-ToQ>FE}Xq`$!8OaTRAUA}tlt%|; z4Ti0q>9H+gk(`L_q4?RXVAyuG8j7MqL?!^5SPwO|l);;rQW)T@Gr6dwJlIwt0&=@v zeLp2Gwr|cZU!|!tgPoPJ)iT$J86ze`W|{1lq`6ijgt^1Sdw}pHP7biEPGl}7SwLut zV820su_nnT!psGSA#o+7k0ldEY3YSvy|sAFsGoFOPjTsXp8*$%2a616Dkk%edsNtW zE1pd#%j-Ql6j}Qjez$eBxM6m8i8<}ijjdyhMT$%Ob?;ZyZloc?Rlu=%GWVLBAsOhK z0m)!3}xhV15rRk%G`xbh}*su^Rhc((r2%aN{V_3(CbzqGo z{@vS)MKBrH0Wii_F+fw|1;)U@7ZL(vtRM0+{jW{b0YJ>e6SBodXfJ8=Nen#Jh1lW;_q^eQVBe0ZnBqrU?s;qj!!AnP84|piluWbfY4U{7mEUnI zEFyy0Z~GixA5&2xASfdVJ-h~~c@MKQ5G#%m$n3h?1}e%i#N9bcb%zn{;Oa({q$0<6 zJBZ#%Jc1OV6YNB>3{Z}b9`Ce!?=&0v*j$q|KOVQBhZK++;SriH`sNa3ogwKWTW-;3 z^$c{?ci|A=HNU>2SFc|nghS}h1yqjWhfZR7k|6j>v1o~OT!nz49HyL*^<>yxgY8}AHc04L zND9?MBd6o3q}&|vo66Rx0O44=>vF3lw_q^>9%o+3Tk(Rgb;FL|1eVVlkAXEV!FcSH zt|bn=CPVLVu2|wGu+xYpj<}G2Tr{o2K*%$R7(|B8=#2ELfqWlcB4g*f+e~GfAlui$qlAbb4Ljm$q${ zV|Tr@9e5o#Z{h?VNuiKJZW`#3Gc{>VjBrFrT}6`HMz~~SndZm8_&ff2yvM^ zZVe>_DYo#EAzV=mcY^EHgufGbm`Q|#1M%&Y*{hD;BZrB&c5-8hY@n19B%j+&gyIry zApld@FQ`ygkr|eU!B&xuB-luHIVc6y5;!cN4bulW% z(@WP9LAlIOf6UYwX`3`=2EyR-(vyJSTYhT-5k*o!(0`vZ@*TLC2q%VjjMkE$WUVDp zm#Z0Rgb&^m*;EYdBObStk4UzcTwkdcGJ_!|!2sq`_e6?yAFzPfV)C7X2K}uDMjB={ z#f~;Vq3q%gE7z!95k>Xh^kW9m*?XwWg2(LhVU{QhBMNBn2asT1^eZMsi_kN4pl~w< zEu9lDxacW><*vT`2qRDZpuQ2A*iR6Q_pltGq z*8(LyMJl6!+FFziu85&h(ccGkcChCG1y9p<;)RTuI!e=8)uy&ahR@7At8(qvn#i`< zoa1#OtN;s)z3y8R&@KFezx%eCEPZ>^gC)RtdfODmcH32wi@HyUVMc|Lg_bc;xi#TK zI|90j`Ww4)fJD915m^cs)1XlUfS8Q22{`)gp4D-Y>bQ#@#L+{N1d)w2Gz2a0+x>!} zFQA$LxD))yL+Jt&44pa}T*HTts|KHd;?y4XNtVp-C<9iE*W|M1bB_{Bq=j%ggzz#& zGeil3PI#3-=4ocJFB609W^z0kaV$)p-{{EBX0X_6wA)D$1Q~2dnziT=-5`VWh|z|! z2^V64(wVBn%s))3s;HQhcU{TRxEXCDpA2R$k9OtLvg|VOB{W9K;8!$WmC^3$CSU-+ zRc_OVS(0C%17CY{M^)P&1!_1#{~xXY4f$&=+&Yr@Pv<97`rq^Crp}G@zek(@H{8l} z=28DPZcQ*Y)Chr%o|5nSKx0aQpQo1HWUc>+oerXTUX)hOhQVC*C``d?Et^tZPgf7 zuQQG|czCY@Y^U$-Jwwoxrh=SH>oe|?(r1YRql{podgP2T_rI3+54&4mUjQC-| z1&sXAj>We-nXE~qC16-3o*l>6*3EKJ5>fzdhAA$fZ<0Ncuo>I)YF=jt#uWlHqY_=> zY?CzLhJV}N@B*(=fi^0Y$M}V0-Lv+96^O(MD2S+gTY1`874(evT2;}dAbV|y2C!(i zkqALSYso8P2rc-*ULvjpHqh`;t;6qG3h8gJ@Xyu#Sft;$&Lk^}MD(7xf8J>wpIwS% zI%Gc;dS3J)Y2`4PCYL#ZUr9SaL9ywVHl)o+eBuLx=(ILa(Z$C=rH$=zbX2ivyW3fx zc%=+k2o=~|M13VI?dlP8<%<5MVt2?AaG3I9OwLh_y<=O^LM#X)atpxc;O1AnyBL-T zZw*{hx7m_TaG#S3Dinh@B)WK3+C&tEtRrdp7$qMX^;?GOlnOqxaLc-WYiZe9cyexW zasCR&y@1g#Sx;S^zkJmK0v|0rug*R>XFWEzbaUY~bJo?l*(>umo>-!?vp46}MEl#g z=x&&oaTf7WVJtGn-{lRH+NC_IJ;OMipCBiO+fw~v{G)y*b=RneWTi|8kd;lb)Qx9e zrxAstAf+@FTU9ItKcZ5r3B@?8Tkmz5h!wuOg)*j>ly@XmOwaP*iN#2RRT8A+?F$hs z(-d9;%kGa~WKZ%x2LYw*8>` zWJ+?vVhw2%^Bu!~?Y1X2{zle%7`#J9S$8iGwzY^)&T5R{0Sp7!=Hi7i{!I5Ye7}QMH+PziyFpgiPH%LDGU|h2t1P&Z z=qfD(%Bq!gl^l|y%oroq!7);){SpbKjh&4R%SmOC6$uOb%K3(z41q5;4LG8a{N@CrSV?lt?JE2%FEbwUtpQ8Cf)vCS66k zCMHgbc?yW8eC)8rOe%SjsHxD1gP0MZGXiVd4{np87m*>FIJ^@!6d@_zS0b6ASZsmY z_R-$q4krCZJWDC}!fn@WD{m#OP0x-rmo}=`ujo3UKhSDQ+;xOK#1S^l4dx=|gg5~<>rM2} zL^jzSvKd3paIVebJ~CS6&tco`&qk>>?8Oz1{B&)tb;}}sIhkE~aT<>V|e@va5 zOvis4#s7YR-T%w37r*|Ml?B>#BH9)wSHiWhH(97f<|G_lm_v`NEZGgta>)77Q7h8) z?r;=16;ZqeO^+L%Bjowtc1V@tHp%P44Lp*f90?_IPjmrJNSrLNV)j>e)gZ0muyh6o z8c{~3%2C8LT2pMWa^1?d7QnXd!dhTNeH?)L7(_=5MGv42`k|8pxomB$uTfBxLm=>9)0{QqFy{wdf>L&)|Im7M5%b!zD~LY9_Z_q;$TPlp3n zCT92ubxe*Ox!GFdSX7*8cSOgkZWg-g#8qDf)rs7ddf1Z)atAOme2jw$ar()21@WiE zq62S_tPYM+(3{o&Na%Z{_&+-Jhup^i{D0>B*(Clyd;0V!|HqNq|BoXg?RLCIsB^ws z<1M#S-t2VR<|DC1G9F<@hIAr$374fJ#&xPHtk_O9ia212^}OWXa9ff#p)7CN&8?zC zlI>V|cHt{1br&%(P(r>)m`@ZnifPg;v`4zIFZ2@moC`xTzAH7oftW# zFejM^&8$!vW6JB_MN8~ws_PABW7KY-wq)%n$L+L=gV8zbu>eBn= zllCfyb9L5e)NDVCrG+7;QRoVB2XwU-8Tqnqwa^ulZ3STh9b;-73=kB7K&C_Hfa86r zKwpvVyKoSh`$BIR^1f6ng*l}ptrk9(goiF#=Q#}yq3z@C+hK16sspQuYHOz@396E- zM=j<3z}vt^s^VkX3^$_Wnz46gz#g8W+%A;fuUN0uHm)sqn+x~KhQ$Vdx|lZekDg?B zI;x3OWwM+TbJlIwZ38>Jv7y{NXujCWl0BL^@fM{l2-}VqBs?1>?IiNi0ed`qXgA2Q zq#~EbXzs}SR!>-Fs3!qEK9gAv#Mmlk@!`coTU6BAh`T*mCRiIlf3_?x6kADnl-KFl zIHc39#d*nepkLf@JAjv#mv2%XH7@l&GICb|{sa{66@?Y-B!k$6D9NH^R|(CKiB_4& z%~%Y@99dJkOi5vyHx)A)EidSlQ}pLp9X-A%-rz|m*bw3uk;g5>)}`iccvo%b)jK^5 zj*f2P+J^7nj&kZ6=XpFdV#RP&VbbQPeWZfDn%f!4F?TJ z(90#1FfK8?&GilBEK!RoC<=vP!InJ2&;@f^jNuUz5K6C+7dSV~(N01>XEW9cMy868 z9-c+xJ-XXaE9OxL=qW8Z=CUQbTPmTpe7qL`4QXGF$=xu5uMnR?H_3D&1|z;nM&nmO z`>{ukQLO7ng-*~L2AtNL!waE+4AWcnu(Kt2&3(e0i-gRZQM=V;-0FrLdzvr_=H*~d zqdJA7h6R@ZSnRk4k0laza-*ngo6b$4$knP8Q7*siHC>7Hfom?y7uzIJ{y@F_aj0by zqh_OBfccU1V{=&zu_W<0iBU+B1&JhxF(h^<3QHWVX?G?AQ^>P6N}Rz3+d~dOI|dFnMALB_#=b<7hd$p>Hl-*rcR~k|8pbz z&!eUPYkmW+o;SMV8QI>)O_9)eGK=3rI~J{333V_xU0R}J2Hd!01T=x2&aV=|f|a6! zltCR|5-w<84hkmXHd8T+<4P7LoAK7)7PmG6U0y{x8by*5UanRjpT9O&trieDmu&u9 zFyK+(K>y<*XXf27t`?D&RQW|!O!TA9A{d})IEL5*myMF$;aqaOwGMj61kQH4AM)~q z$(D}7d`8Dbqu=<{7@Zc(=tPbcHoGC-!z#;LOzjqm-UKe2ijD_gy)x-liz5(jBcs>{KXB?AYxBq&>JaCmtlu0?zg8H_~I z_T;S`pC$4DJb~53r_!jaAvVmD5<8TaI=N95^k&GSBxI~2#PDjdM@HFb8&SFFC+{IL zjgqPE1}X@ZA(GSC4y!sAjED3;((0)_%U#B+P|WY#F29PW|10h2$lT~#lBlfqlJ?d~ z_LJ6>#vsd%ZEe&yrzX2@oCD!BjJQQr0CjRKqx&1U+73|cDEVZfUq|c{{ooDM8&pW^Umr&t4Ppbb@UQcxqGfL1x9+we&(IW|THSYCTf{DJ2?fUIkk!S`l4I zNn`kNjM4qw@A)yS1ph>slG|lZMnN5=jldPhv6K|yX|9KngwdUvLo_#MBhNUeRxi;B!zVl_t=HYXu7EEPj1McSdlUWTDRV zOLo3VwHzjjbltjJ&mM|NV5UZ6zPt|uO89<84a83f8IF*wuEisXjUES$cS6K|S~>|p z5Oz4HQ)H5&=SNXwu{%cGd<{ovA*U2yZBk|GLgI66(0z<45lwdmH7VW) zt=~?PV~rT*ZBhm3{Fa<+B_zQIPV@=86zHp$y1LXwVn@r(y^fKOA_3y$n7WM<6N%+k zPOYY;s-(gq-4t1QVs4z48Q23xm?!fZ6)ziWQwSo&&1;Pvj-6eI&yCaWI25eZZH0ai zlH|MA;bs!g`AM*o@xcv8Hs6 zqQQj5B(S2C;?a?ey<$xPew{cq3I8omzOpiP#@zJKF6@XrHeZc2?IbXBsn2Mmqx3u~ zMkyyce%;xL@?B82ZhSNzauQPh^m_(44!gj;!l%$jb6B!Yp>oMOaMYByPwe~cm%A49u= zgP<5Vz-YN{QBS@~Evx6y9E@>NPsbwS8B2kh8&Zhz;EOWqq6{|{(_c`4+3s0Vw_(}S zs}kQ%KRL6SOdWl&zWMXCe4a_d63@MqGFt$xccB@7?da{7AGxpQIdHQ>&q8*08 z%mg9Ty(BO*dQI{iYMzELk4w>i`UbAS#tS$y<~`DcgZzKsWt{$-nuPz2=)a@*H;@c6 zwgJnw=xW1YL$~lt#mc+k9%VVK8bz&%71>9trBaEngeWWJ?mi+Y4&CK9>e^~am1v{S zC?!ZzdQsH??NMlfbxsVejw%Z8!##cbEWz#LSZfaO)k?351xow`Bi-{Qtt)VdGQgKx zj@O`rXiDa#w%^KI;}e_XMff{`N-_L)8Lq2t2jeo;J-A(*9hK@)sZ_@;42&K>?buv| zClrmKsQ_cf5-KsA`#LiIew!Yo!E+frQ2AHtJn1+`a;I8* z=0(TGO@OC+8Y}gI+caKE3Qdw@rr?WIE4pgc7{@2pd59dy(HPva$(Qh3Z9)=sk($dM zSw^=pOREe3sokCc1`Hr~ZUA@-cQzVQV8E=vbBNv{$pq1Q7>_T%hK4$&lcX~k0B!_C z65JTrsMXu>YO!O)=20ic%)`ky21_IX#^^vx#(|Nk_4C*MJ8}QJVHJo`=Md~c2HpQ> zlKwwar_N4}@_#=Z_rKZ(JuZLA6;iM!9E-|X6P_g`KunUKrU4FqsrS3Td*9Ai{cX0s zBrX`;h-b#{?K-A`_Q3qmv%~%KbQq|AmLBA5^)d*gPo=kW}vR6m38tfS@i%N;$V&u*X znW*V|{~rePKaBtH)cMr?fBMYm{(o5R|9;GWbpH>y?MEE{h~w{jZH!pdhYSCshVMw? zKb<|7(*K`7Kg$30kl=sPsLRR9yaKSmNZdO@{&&Yp3Y?;av2S`$$|Kt=n4FGdSGgWmp&F|t zaRQj(gtdjNNiqg71dsMOL#cDoiYEjnliO%vB#u&2{mAuZQ8{d+&_*S2=19qP8pE^X z;qQbB!GWa;cPR)!lF;J;YWFx(GT!19YWWjf5`W%AB*HGGD$zE^^^&*%QZ9C=vbrNT z`#zAV>>!Dv5okVx#^vz)Qz)fpct{r;GsmYur;@cvp`~;TAXIFSR30}3#Wy=kFOkPHZ116EZEXgXQ=TvqX zI&th=5u=r2$}EP}NeTou#ig9qIOTW;(Tw-R5hoaP?(JEO3#=2`K!&sW;lq{kYwNpb z_9x))WN}J==4UF)^0fk=Q~QOPa?FGw=B}>^|ysx8B2AL^Uza_Blni zF33l7P+$ufZMB1D5BcO26?(QFM;A%c6pHdFu>iN+fNju>L{bi%_#hxhY7GF|+<55m zl{6R{%d`&>L#7lO#%y^~Drt_BfI7oaq~|^}3@-Xxfn;r3ooT1Xfr8z-#d}st;*aZS zpC--)p|}V-wcBuH0BAs$ziK4uMh^ujZmqend)*T~PMFzs_R)&n&q0@?P0|sF<3bLe zA}QB}pp!@gz3~VIo1@Ori%I)|MQ8PnU=mDHVJdlzq}vimi;Cni3IJbl7~hIhW1+jb zi-0Mcs$zd)C+p760+|USHYT%+hPOnvFTij-u8Nj%Wa7!*{2v7(WrM>iy7j>}x=VI& zko@n=WP<;jnmRuW;zfM0MoF?!o3WVfa^YCY&4v;U% zd5#q;geYPfG&lr94W38yXp9Ih~3Al@%VBfN|a6?5V(2|he5b7QW zffj{_7$~k$Zm%mCD2nAG>XDAF#u>$DtbE5R(|#Nz^F)4~y0x|Vlb_kcibG=~Fxabu znNQr`d&|K8;$e6A9-xEp|5V!l^X$|J{~x9QrxD(JP;RgO!NH>-Vp(ssi!nwA?*;*& zDKiMu5RRf5(Wz`o1XfmwlHD*Clsr{Vrw;1@W1>{_t=cxTT=ee*CF;WfgBd>t1EWht z-?D(oso=22)b1N28Q@bJB?Q&(yRn9h`~ZDh$|XcY9I7g-#aOxijgB<+_nrQaXY4yV z{C^g(egglWn>sV{|36CnZ^rtT@O~J5|0eAQ71__JVJ0Eqy(<`kHyzs#o)$${H%%0l zk`TZPZS9HQW3HWO zAr8DjR3Pg$Q<1|iam--+Yj~R*&2m(77W*qn?rwVf)f0@u5(n#2#wLAR1A6Orm|&SI&f ziNTGc8M~xUU)^n>Gb?f_7vQ_waW7tYLl?Tgc(QWxLYco_D9bBbC{pCm$U5Xsvyqo3 z^peBtQq2bf9R0$lPMxx2tG(2vt+d|7Kz;;`tIT=*^~kXveP1zBo;@f)IW@rzoUpTe z?E^QA2NW~y#?)-)iARt)!V(ILI)VpQZkHZQASaFqY?n(X)$`KHvdxhp>ru>PToT$0 zw_6zEY7?`U`LJnQb%0yg$KI2l+a-YZE4f|hZ@C)qILP|?2RYpmhkb?!nMVcCCXbt9Zjdz#i*8c|0q%s z2#LUc0C`^GbQFn%%MOKpNykD*R1dGBkm!h05jSr_Q$@nkm}4a>T`(qJTJnvRM_t4G=C5+;s<|h0GWW#lh!diHmrg)XlUtzfi(ha9Jf} zEPA7=h|uvtpom2U;igL{3Zu_KB`gx@;B+Qpb*r%S_AZvFh=zUNPDO3kSpihk4$|MF zxPFb&4oDskoOE;K*s9v61tVG={?-(N`kg@~T=&9I!tD{9Xhed>z>+Meltw?XgJC=E zcEjTdEt4QBGY+Vl-oA6#qMgxu4l97pp~S_dPIrC%LK(jv+BD+QAMPwNMpx6Qk<+G2 zbcCbKi=4oXUk{P-B-AA$hhmGkdP$U-9S74;*|Nw!r4woKOLmWpAw#X?9UU=KFQmj& z)GIl%2_MY=Wr$IZg#Vj5e=5oUou3@}|2!!9-<_BkEkpjNH%OUf>VwOv3w&Tf&d?{) zth@di8yJHQ&s_mB4718rmlvu)grP8T$(p}m@y{#si*uKk7Z#uC6)kSK zUNQchmG8CyQF5#_w7GSx%U2e!0_xt1qF>>=rn7_E2jCBg%>&W`P+2!@>=dXO4%mhC zKs)faD6pj1VR4d{*=yHS6h0#Wnuq=2iKpZ_X}XU78_SpOGoS z5U?=3re8;+K;90NBnO3p(MZQbD~6|Ns>V3SO^xzw=*aMTa>^H?9~a9^sf9*btPBVr zpP##S1<3dt%X2rDix59Pu0N9W;;YxDv$` z1C+qZdM$=I>u^P@$3ygtxVI%s3{-Hh)m?`o4HC@|MSugsq~p~v5f8F(ezV;GwjUeX zNsI+<*AOlJghEnbUOdbWNZ8KiYx!_x?XOb$Tl4|9j@tDF4fm z{Oh}=F*ze4Il#)iYvptlDXFcEX1+}z{>?C{D)^Gk;+^OqbBcu)$wI?!Kn=Sd?RJ$R zw^}b+RS*7uo2<{P9W#BdG6`p#N$fo~8Qs6wzTlx;{EZS-OA-<DDzY{NkJ;*vIKJ|!Tn!y>)mqQua$GV(ARz<2QG!SC&neLcHBE1 zv*T?<6(TZDr5;|R>|CY{6K=W&7!OmF8Usmgk=DwI#n%#A`;h5m2mTHoo^^hdbEWJ z0Ljy^cOjQ4nkbhDXI9mB9j0bB)6y7`^NnT4WMqi%5XEQVO1dPS)%FIy>3%pUr8Pw3r?sQ_|5am;!Xz3A6Xe z+D@-6Vv$FowMdn%MK^SVtyFe#Zn}y-zI}j?l}ZnQ-*y8}8UUCSCN!x&Qv9{xb_hHa z)%rD$FcFcisQA($`rA9qOz}dq|4>E_1j{rFk;>@t!ty9xN@2Qy>3y6dVQ3QvDFoWF z=JuQfew`Y9*d;Nbxye)az)Y%2;vO)bWL^cvdwaEzu4GuXSu?TLDwd5YI+ID=qf3l5 zTTFzD$4x{{tHhJ!1>-s`%DD*GAlRX_7Hg;_i0ZeU8h*l6vW@ANBBX-7p~DcHk&8+o zJxKh8S)Yu|%T)8i1tqhU2d|i>&nnP39_|RIVM^LB`^~1`0-C7QLhOKQ?kd9O=CKkd zo3bs=otBuAWQ*#wX+D9rWdbd;=9;;MRDEV;NZKmXiioYnc#DF_`(|Tm7m*H}lDZMu zh3(7e99b2WR0KvK_J;9URx{V}It@25Qh8)$tpUDE<2QO7Q2`zD)>ExyO*ol=b&{?4 zZD8(tqPYlx^tb`loHDW1hK1~?X=tEW3N5(M_jk*4w^`Mxk952n%(&_BW^~9zIiewm z3c#o0+6HGyQmWA_qz^Oc4Kmtdk+_}P#Rm4Ftkm8Yva)n1MN=R&NNOB5H0)y+D-bg@ z-Y+Wxr!T-6R1jKPXDs4KT>lWr{|zC`edYg8oj#L{|BW(%5&!={`G3}gvK}yS*IhC} zB14=1FDbfaDy-PkN5Ny00#jpS1;xJbOva*Nc9n6x1AQn={kQ-i-vydthphK6jQ z@rHR7c}KsDm&j`*+al66_HbWP`>EYrVZY51;YJvg;k9?FV3`1DSqAb97kLMDP3m(9Ayu-2anPr_Y>A@_$n!`Oi`E zf7!cwWd3n{%s&ob{1JH*Wh(r7tGrbCLWUxcxlN{9D3A(<%R(IwFJ7i42m>y!1n{_q zJZ=pV0WObAO{Z<`T6^+)55C*Ts}fh%XB=F-VC`G`_sh0L=X`%27`VvqFj5FcqcNo( z;=7MQRp`A6s^T`tz3vil7A1O-r@sZ`byO36MZs8f&lfIe2@D-OuTExC0v(6tYY9dhN!s0<&1!=`#rq&6!?G@yf^eU5E6agK>Hwk%Fd$cokh~5_(6%r zK}JNBP}T|R=&UU*6(vGwVSvq1c6X{sYBSSIKmk?Cfv!Q%Yju29S%xGktcHDG&73G= zvRki;pOTr#9TPNC(p`>Ys^iwHnA+V|$CD>sK#_r6X)cVP5+&0!4hSxN(TW`t(YfoH zjvbN|m=y5TV^aeRNJRDsiFib6AI2;9HRlxK8R@%}6nkyQ znxCJ4oc++66pl6vm8*W#4&|aNr&cNAQd3O34%?J0y=}FGvsb-d9qRX~nnz=KW^hhO zgjnt!2bnoyTgb7TB9~zqluSxRidx<6C>~1~YG)13OOe=8j-{yN!6Bs_jhjXw5_6zF zV$~uLOB^yDQQ%lMa*xc^STZaN&bD$;l<+;?+;|S!YWGLzT_N{K);b(cZznPOF!;Qk z#OC{1e%IR0FWdI-68e%x<5gryd#``TG^W*pNrWV<1`R6WE=&jS-f*zbqDGSXhhSX| zXb!r3M`{N-vVuHP8_2t`fP5j^Kl0?_ZEtP6T1wRpcvo8;XCt&4-fh>q;9X?t)1A%? zUe}>tv|{WHg{M&g4s*Jv=uC@Wcoo~MdaWw&pge+77};VTx-BMs%ib4aleuCy;ZoPc zm{g8O3mB#qr=c%`1k&vV;9**RrY}ZT)h2PuF}DWgz9OSgnc-t+8#);7AxuQ!ylvpw zA$+OmUQKPw3Zi0RWJ`K)^8dp!{~W^pGwuI*dX)d~fy)1n#QYQ0UnA?!hxPpT4t#sa z_MfLy{@ZWe$2k+ttT0`yHG<3Lshgc0e^>ai~$uMTs0*R z(B5`;Uj) z{9nQR0ImPaK)~XGx#~J~Hz)(iihW#h!&({f-DRBHvQkv7Txv~^tNP=sE0e2GC}{Ta zAXtfR#6(YlyXA#24b&AI2U1cU^Ab%|i6BGVyVPlR%5L3*zSP~in(eNza*!6CE=+q6 zgQ6V2@!|Fv=Th-2zgOEV`k=3C-daQTkb*gKr5Ls)tWmy%NmC7%3D`YwLuvt3ofD9D zoZD_oY(*P8Y!bwAzOhMEuL4`-G_1UR#jRPdvWu4el-EMb(8*W6@>M#9a9m}6U|_R- zL<}mc;dVO8NkSUF(;+{Ha^2gSR3sM{i%&l+lIPbkqV2)pqNQKC)WFrO4~={a-YBOAHa#Lw#?_}Api(dv{h;kyE$g$VRW zg)d%73=T+6_p$Le~tXVhWGz6@znFi|10BcyU9xCxBHKVyg51M=JMOSmNL9;p#ESJ6bH%z_n{M>KV#v63;cjgAjYg=n zG}6x!+#E??hb*`T55*yil$=0?G(0G#Cj5T8Wl^Wc-!<#rJ+RW#Bi=X3Ny!Lo4UIK+ z1Vkd%E~BmWEYOo4I?o`bkDfKkCMxKv^bAbT-JhV?S^htKNlGE2sAE1oaO3Q3BTxw3 z_NI@yy3mFg?_#{!@q!oA^t+F@ePj#uymN~Z#B_X|qK?=F>K=Tb64jn(zE8s}UvcCW zbr@bzrmX3(W5eHFjQ1gH9n_KPLAo+2$CjadnMSTlFR(E3M-@eW&b2G{AoI~a^4|yQ|8wT-NdEgE<-bSb|AXp}k^j%bdj9pd`|tmACguM#HM0LW`tv{X z|9Lq4e-5Oxc(6V|N8kc9@&I};`2VQRjR9<_#NnLh0Wc z7tJ)IRPzWtw_N3Aa~zYHjQ$?id9YK-Az>e+zT?U z{>UKo2n<5Qqp}EnUfF+Q?=0s(nCWE9=2H;J{CJaHN{N_9%5__kLlF{nT00h-bXr%J zuV1r5uT^tNc+mhF500062X3uUG5UGTE#Rxr`9XqtAaZJxB zo_(!jvV&*#34oa$xf0p6%*d6>NXE{J7MU6E@kk>!-e>%8s{aqEj{*4q{JC?<_>bpL zof^e|I!gNw6u{i}H@v{hvd}1%%BcT^gWki00}9lL!A9fr9%b~#s+#5?WA_@@@#;L= zbDU8B4r0VHGTG>Fvhmc_*<}kRSkE$R@t5^bLl=X4wA0Ee95G@@ZI2*6BRLj`TTLkK zVeEvbf$FEGP+OvO_?iu1rTN?v)q5Q3NqCPU<&MHRdUA0VAzf4~N{n7u!A4i(W+J>A zO`igyXjTK7U{N8>Fu}0tdVwBLQLF9o9S?@1f{0as!8hUcS`8@nuT{ag7)7gchh!%? zpfLguC2dS0Vb~COQ?w%2jiM!@8{xq83%tKIsRfTi1`_^?tGnr?{Yh84<4K@4LhOv=k8iV1`-re+IAG!^f%ISbFY}Dc74j5y(?>a1i zC$<=Ezuj#RZ3d;}5TK+g3Z+j_!S;ux7L`!T>v$xm%bT+@y}NJD)--yXSjuAse-$M4 zI!xHKbw?ytN*o4{)a=q@V|J&FR{5oB6<(>YN`EY(VWm(Oig-FkHAZelZuRx7?9p1( z4GO_eD@lPEv^685VrlW6DySMg(Py~ zOUAq!1+)CjcFnMG38nwlxobD)7L!{498Y!Vx{>e~4a>R>mob3IBVbewkB!De)sKZs zcfy8VD~?Z!+yQQ0!zM=-9@iTM>s%11#j!{%?-?aE*P#y_HA*VP9X>*6904~c(8!4S zpl{aKiO5xC9osI;-93!w$E*$AK4jd~X4yU*R~-wAkR6#60qmm2M@z36JwnCkkZ~21 z!(&gfTBVs{_%SFF3HE)(A;pXWwA^2*%Pf=`%glLVBWLUeQd=rqdfRo|6l7ys{MqDq z^X8*cRc>9Kvl3(8>1k`0p3kWDs8}ofZ8hmc$B*+yYevV^Wg&4+m5s*T&5V^?6Sy#u z=X4uh6Jur-4P(Jq$LG>oT3&=pc}&hZQXtz7Die73(4^L7A&VK3I=!Z9wn>RHXts+H zY}59uZKcKoas#FOh`v>@Ec~g~TTsoK3)D(WVTXHJGKp(A{E}G|+0sSKjCD&%4@!P3 zinZLX9^}4d$5xyV?V^iVz$NO>F%`sC$90;BL6HDu%cMvLtP;e?%FB{qDYY87O?2QH zuLW4cVG)HM5XGWlR3L$8%wRT~-bGy83vTnGbfRIvE!2RiacFk}aT>a$1tbcGG%sP; z6{;ZWjTRr~s0lbXBodV$BO-+Mh^9;(TXRJQ9$X+P_#!6r8g;qG<5tSv(4i;kw;6nr zNR4T1OG0wd-$J59)POwM(+*FfceHK NO5*AFc<>9mouV*ur)@q$rHn%9*}a17ia z(wK2vF$vNa7tbKg)Fl#2+NFqJaFl2XV-+aA9j%Yt187XD;e~yXFBD-)i7X|&ZyF87 z0F6clq_LD@{E!-)3L3^B@J>M)8t@&cuV$mKh6RLvJ&edXt-H_cG)w1QFjt0@;E}n2 zF#r{UG}b``qxzNzbw((m+w$I!u!u=fG^S;TvH;E_6H#C$!bAnkwfK=eZ=&%eDRelk zAF>pwxBRDiRPkTV{r~KJ>vr2V*68^gKLtnWSJ~f^L|+njn#OKiTWNg8myG4~GMOwd zTB2=EBvK`*IPRG_YyO`XI6uzQ&66DL4UhytfCMQ@c3XROH8u$X8^FfKzVG$zT>Zc2 zkNW@R{5(x9cqnWD=1s{}3ICFl{S&qG^#_t+ZiRDyNb=a*D3TNUF?2%|A7ts9d_?7M z^e$04OQqI2g;b5cvpIo-W3wf9{VeBoW`9TrN*rEk?ot$oFM)qSv5d@EFi2qIHdkam zb>Lz>_?X+2a6@T82z++wj)Pm~u8d@hNgV7W#`ba zL&C!bGZ{Pn$Cbp@4FQ3bNZoUX&X~^P3Ra8hI0{2%G=dl6TU?L}Fh--Y%DGjh&&Uk~ zdIs?!veB+Tlsisuog&Zo6+y(6M1N8JNUwwm5&eZ+=XcxS%9ea)R z(?^?u?0BywI_%TzG4`H|^U%;B_Vq0E@Uz6jFMEuhc*(oWEstw}9&<^9lMlpu)D2qDCzjQk+f zQyxc1o#s`O6ujG69&ONuqVK6Hn=~kAGl%(<#-s*!Mqk9RjHDU0z0)RKXiGZhG^I0X zAiKGP@@5n|;HY^8{7@0Fe0U2#TJ8wZ>9bkoQ!wjpvE=K_M;|&6eEO3)q0O95DD%n} zUS1^FF8RfajS^q_tR3frpU`gRaSoJQ;mE4HQrXv8;0F0#2=Yj#%VHT|u36Lrdr3|3 z@Gfx`RgKKw@JSZ^#BG(@e1Od%)Sn6ApMZ)m=d*A5uWp=(0&zi;z7GQet>-(KARbIP zLodFA$1^|uH@xetB%<@R2c{q*Z^SPA6S^)a?ueM^jLB`2GIc3_6JC!T(jJ3fOAS+| zB%z>EfEg~3GU_Zj-$^6R<38!@lECQ<<81IpC%WIO{~tj&s43o@--oIH@A|WC$^YZ| z*7J==`F|Nd%pi7vC_v8m5B;;j;pqwJO4oZEy=Se*1OK}}|DU6AY-eWA#>XhB|F^mQ zJa_&#*0&zd|1y4Feha1W!}_MZx!ENxy3rkB+&_2y=qdm0XqbLForPok za5S_N=N5aTw_(%v?q~GU>QZxScS^Wf@3Tv0@;Pc{tG7v3Z$`dH>-*J0*Y{o*ckA94cVoTD)vd8$sp(=DCq4>5*1n-3*VMQH zL&6$>4DR{ho;Ibfa1$qn>C^7BepbYBc{-D9Hl=3;G)|29EuwIAbzw19q@|;Nn?kdT zad7K~tzGrvrh|5wl^bqD0BRLK>kgfWpb_n9tz)(S^Y8!npMU?~FMpW@@i%SqH^C_a z2c2UwybP8J)ddD8j=G@yPKKj^lQA5tz1GrTTy8?q?2;iLs+TpLB>>Zp>z%bCkm2k! z2z1ijqQQ3jMwi;34`{{lDxnk-#+MG<>;We99Mn*d$+(O^X<0mLLWmH0tEc@jhZ&6g z!bB3_vcjm(&*Z0jeB3=a0OROnVjXnwnYDU++_CUQCw1mqOSvi?k$_KI`7Nb1yR*@G zwkE===;HWT-@$fgv%G_Y1APZy=Q<#(C2ZyD&o5dUe4vxFcU#u6whcfJEe^CT06fwn zbN=D)KRtybaLQ+oTqmncJs+*MiB!4u?D_UrfBu?K()`MAtP$F}YyAjXW16#ZG@D%J zE?;>JR>R9Z6DFk->?x|?x&7RTq&@kvLQDIY-u*%o!MNO7!Vl^}A>g+uzF zUb_2Sszps#izmOw$#Q#Zd~??rv*0p5=^gYIez^+Up}$f02Anq!dD)+pXRI z>G(4zyffVnv4c(0!E5(2tlz^H>ERd>AJzN(8Tou~8hYb;pFhXZYl6NFt2guAdS8Eq zUyo~!{b!i>?0Oc(^}hd_d_Qof1O*AXuGPiHI_ctc7zedG0PH&nKFt9Fpx=QzEboDn zhi%p0MF*DtooQWnR~D%M2d@=y)r5bAKjt?Hl-B=Q-BIk83BGchGc{|EqFzh zMkog4^-ALzXq6a;pL@-DRoco1S;{x{dvLfaY`~Y&Wq=B-0LnHyExV<+bvz|ZSBvK( zBv~_w=FE7uR9gywb!VZs+5=6O-ClbS+1XduJ|q$guG;yHi`nXv@Jg~l7V1CnFQ^lj zMks^YbT*99hX->?Uj?C?MiK)Q3E#9&mlo0#W3h1X=_10`M-cJRAH8F_pIn~>YTjXB z8%>McvAmw!!(f+*;{&VUeR6H`>ny1rkQr70{K0hW#R3d0vZwv`UOScTidGILuaMWf zbb6~_B<;u=LSg3iTDp9Amju&0o)zseRFL_=W!x)|Kff{iKg&~#*FQW2xBrH`0gCVc z&F#&tujKqM+Z&Jde;)Gvzhc#K0id4{Of$UZORPZmaP?O;jQB!D7SDPRZAbUMCE zH8+thyi9GJwhh6t?_Z&}?N0kD2;%k!UN>@V45S+QSk4)79MM}q1{t%8&r z5QOhyFmiGgmko?xPG$YiOM=FJ0uOXRm7F?*W2WbZIO`YSk-P~ULhq#w9p!p2n@Z`X zRqM&osL--(bd(voA;(T>#m=xh zRdIO73so|KN-K!7LHUEA+$cP4t(;L!w(Ude@U@mic5+Lt)Q-xUrDO+%^>h@vB%_0( zGv9i#dqLzW0y37Ssf0|0h@|eRC-x!Wk4j;tSA-k%uThHUHEhh9meE6njvO%$QH=(i z0LIY_`7GH=RETfz4IrF#PZ;M&9hD-D0m9Jf+rMDY(+<#wFP+Ksn_u9!9rgE^Okyp+ zNeUz~>wv?cf;*{vXLp+a!I4uA?S=Ig)h-y12D%3tXTE@OGu~{4Bbxua`5&CXYwvm% zviPe7j8Vd9Ug7AiV!{g3Ka6O84L;4d3ke%!yqrK`6py*D1uAoL1Uf**uiwCD^-4Grm{{sLYU zF1>|(#FwG#e2iF@LC8-$NkdW>g}28m~U3n|9%iFnAcKqNQ#~ z1AHU!HB4G+8hYN<1tzu1x66}bS-;COZ#HygZ`v~Zrs=9^VEp*MHeVZdm0uO9tKL7R z*Y?$HJhuO23SH6q;n$tUda~DTrZfOJ#cp-?2Q6KdrRt!X>Q*pyv0x`nK7Q-@qu{nh z*$A3&9BT5+;5JAdda`6m5J}G^<%Swr*7PR8;)htlHFV?njy+N_A2~F>aHAnqRd_^a zKvAI5yUAaY1As-!ERjS z>(IGm_K<6IerG8*UtKxFsDqTBT3tf)2zM+PyDyhDC3$0ooTga<6l4P-V``ncXVaTYF8G5LI3jMgCEx zM-g{IF^G}(RDQ)*6iT@p!Z3XKbYpw{8&St5svR?!s5cppOR{FpNJJD*FK|zI#x)vA zz4Sh(nnC5;m*=JOUNy3_pG-X!n+Z3RW!PPg-pQnJ?1b0qfERUt(hZY!Cz$Q66_xr; z^i-^A&J7cXzulmq5@sED+FURpWjs>`~^X1d6XWx{l zluJ-Y;$QwKC{z`L_zh@TF~CInyuj>lE$hqc_#4WXPEfw`Io=eu%XXkeEff)3dXsCl z-9W;jC>&OC6D}qtb7F#Vf2F7p-^?bL>fppCk--cdidFu`my3v6EQwii(dmr@MMTAx zutZV`3%4g{C|JI70uo9B%l|qt0N#_z3&!TzP^~n}bZVZL3<+Udeo{>3vOy5hcf(XYq4L45{XJ+V04U1HyKX+bz*SK*hOB zs?lobU20}2L-JztxWfD?P2v7^v1ejpw#Cvpn8>Zm>QE>7c5j_K(9(}w^2fyUA-%Zk zTUo+5>XI2?nl*|niJzwDW3tF`TKLC?)_*sNn-pR)CZzw8`XfTj!$|E*JOo*CVu!89in5kp9*5PO8MYtECl;3 zm!3oKogva`Ga{FDN!F_xl~Dj_aKb8GSJch|2vhyiGQ6a?D@s33qf0>DG+?kI$D<-o zova(BSSS+iBAijsaXi<6$P*_-Z5;L1=vXl%!Gu4_aS%>n!7*UB0JIV~875w~{uf)o zf3+%+Vy6UtQ}EQMl(`yGMVh`zgl(2MD^`EVxrDk6jF4iHkxoLVkZ6rq;&7x|d7^5# zWzbZ+ZM~kI%`vly(!nm^v*u>5V-{dVvfS)gW(8|n)4H*V%O7F>1=1~8hMn5C=719L|K6#M@_-+K0ZU5fwP zSl{~U(f|L^|NlYy|7Vg+EM%5R>T_cCwe(J*WxuZIedn^W>8wITQhyT)DB&fYkX`B( z!zzTJJ3#DQqhS+@hCw;64zt%^-D2U?ofh0yC$kCO$1Uo*2=RB*)~A?3f`X)k z6>Gs+zq-ZJ4j0*X*)!-Pc|a3hlaAPf79U{%e0Ex~4)~s_lgMjOCl3~b)dz*jDB0|y z0Nc=NQlkQw24^(7$iCwxv(&iu>c2&=u$=Eb*FY^kkpV8~UbnauBQRawYe4OW+Ot!Q zLH23+cV!*ma1TlyD_*Bx)*5p~saqhtk2iGRV+Gw8FPzKEkcNOHEM=+5k6X|n zfC{2H2M`taX22+B9RpPLdlij@DuyxLZI=*juyXw8-~Y!d8%5Ivz?z_o10Ag_9&HeH z4h}SL=0qF4Er8I$>ZX8bHg!WF(N5R~>dZY49rj$X6wIcT_2CA7>xv@HV$koq)1!lhpdbBo;JQ|85Ql+(-C}Zb_Oju*9MF))h0Mhui(payz(r>c z%+7khzE_&eow;$w? z0Sa7GF_^r{NxcvKqyBlpKFl|+PAD6tqB)HW3c&e*3r{$t4I!k&x0rDp4Y6E%W*ZNI9;f-D%FQB zESSw<;Z)(^@P`fDMDok#vZ#XhX~X&_L7PvT^q(#I&$E`%Mb9u3EI)0DSyLwzGv#nZ z*Q_e|QB0|ns7OQ>Y@@S%S`~dXlKvFh1nk*L86XtzcttIf&C|ngYdE|VJizJ;mDEbIHdx7PC5%xh;_m&Y`} z+2`Ol&Ej4ObLB)?B!$-fP0Ot=WGmnF_YMXN)ZT*U8R0l@2)7tPQ0&cC8moS@k<^|- zmo64|PI#4@O!OpQ!c%lNEs7Aj4nLx0F;QaYA|TX z(V}6=WROFP95yt&yB4yA5YmoId^*_wc|fC~4>?C=bb*@&Q)@Q0+;QaILNIpEVi{k; z2wI~%0P`cnPN5LCgDBRs4zHk_<2Jf*ZSxX zUS@?a03)xj4a~Q-=s>@rxJ2(AhP&rM+Xxt6DWDxB)j613uFa z_-yV0w-f_D*AMu7?g5_x{oww7F8#eN{r#0?DL~xTgSg#95svDsu1*AGD>1fuDzu7t zySZ$<-C8!@K3g{4{$cUdAj6*eYLeEn_nwm-xVw=Dv*a>NnwDM5)6gBd;LIrA%Wx$u zV&wb^@h7UR=dA}NT)#QQ+xqZ)@91!Uuu$Q8;@qV?VXDBiF3A#D)6fgZUw0N&t8F4P zGYtI5iwL(kEKL4+>xWj_4VZN+CT7%17yc-%&#BJ*Sv5zYUe%n;WT)-L&e$8a zKUn{?Lt!JTh`K(oix?vyj_sm z;LYK|60K9Q>%Ad#)eXreffWRNs}J002zxfWBwK~q3s*6Q)$QGalLdeUE8^omWO{zx47%%qp8SF5;erk$ul z_0+U$=6i9u9!8}%XBfN${5?86>095O_4ocVSQ7q@FEa#Gon@MJemO^%%>FNf~P-t#{-!CQw#r7?6np`6BNzo+B+j z6!omvLR7|GFhg_ejRWD9kkh{i;YeQ@%B9M6PG7;Ng#`;Qd~reF7rI766Wb3)EXY8- zcG3*&vvKw7Qf>S-ioK9r*pum0Gt6mB0m?9n>Uqjd3Os02-7Af0!d!K+)JAMn3Nzu2 zCmHpIO)T*cW9Ww8AsURm(o+ph7JICZ4+r~Jch?%c-8&grr)O6Gc(4RgmvPMzsJn;G zG?KXKFqk;LikEK0=Z47*HEbYyLxq|f3n#!@5d!92`{XzB{)YU6O8xg}z#F@t-0@D^ zX|KToTzZ$|!7!M*iFE!c7|$k&@;YUAOGPCk6HwKe!|28HCe7H)FYhoe?VnIA4t%|V z+nnP6T<0Tlr?ezDtj?|V&lQ?mG-8Y?uFqRgVe=h0m+&fd2(Ja-tzi0IAKHws;eTxX z;2Q^H;Bi#4`S^AAA?y7bo5|1c=Ax*X{u zygs47jh#yuh9~$8TsHa58~FhML3W=d(lf;0Id`!Z1~<$v5SIRXYwzU1I_rPGC{r7= zUv-m2=AgJhyBYgCO)75?3`WSz^@q$p_Qo5HTt7A3LAH9$7@1eW0}3L8*(@c9EOf6~ z2$|e{;b#C}3;PEY#?UcNi&Xg4as+_4duMy!pY6SUGgtufCJrsZD6m)z^AID1FI|?A zsgtt8q={oe9$m|Yn+yebn)94cDHhC(iI~P?_u3hwCn07*pO&Ei)XX<+klq&U9Zp8PT+tlLPr{#j;RNrk0dGW8c` z>^1^Yov86fsqr;$lDZrR!;jdj@i~=0F{Crfk*b9PPs@O)AKsL#Z+57pdOF$4Kak^K zSpzlK*%asbhI5yC4SqEHKmJIra!95D*}DQO%Oi}XsK3?U!;yQPha+c+wDk5r-hA4ku#IBdU zkRn%GX%Ng1bg8H`8iy@2T<^brR1^O5mOrz&%}>FU0xYcSrpN84nXYq3O(+>%9j_q9MD+<(ukCd7I$fOe{HOCi|SCCLl>#YZ$<~3lFL0pd69P5UIS7f-t zA@OsgAs4W1BaJVRaxv+5YzfN~^@)oq1z;9T_H6%lW=90u*($^KD) z?`(Mt+RPu0T_>DNL}z`NyNrp!bY&Uh7ic41LYj&{P$Ft={%2t90zbG8o$1Zp1@N-N zaWI9lt(EgZ7w3Q8Sl@WIA?1I5w)xfOWB%vI{Lc?E|8qThUdw;mvWSd3I6XNrguZM899@k8KWSKiz;uWj&jQ@X|}jhWd#*HbA!Tifn!m! zJ*)y?vNKh{sN1(xNG#h4b@HM`?K)t*?rb=xUO4Lbi*GSqQbjZL(>0 z*_?Dk*CuBvDdS>1r^PlhEdB|gOdF57smY00{geHl_m58B_74vC9yTX}(h4#nY2t;ZR3gX{^#NgyKI5 znwIz@0&%ooaj(;oahzC8{vgxOs#RLUSqLcs#&?$IkKE7HeuQarz*H20b7eArXpRv> zKy&0^O$Yq;>X$3e9glWk&A)Ue z({HYRfnRsXScL7l!33af_+!l=J*L^wQ(a9(_tG>u5Sk0=v4Y$0K>x@6K6mlAQvKih z^Y!gbN&mP0Z1Yk7_o)AS)c?WH;qlwkGhN{GGPrJUG6g1ziqZMnkQfuAY*%bU(0?}^ zg*QnHI-_&yc@vUbwqrqBVyaL;*P~nr`qPP{r=ZiD#ixQXG?@PUgb~;owXJ1RoG0p;%WT!AMAY*P%=v1z?0_ z|59YNMlRvVLgX<37^?txG3(yQyTStLv9;>5N zAzX^gQaK^#0#ait-*Vcm*6uXbep40-8!A5Xpa8(d7-|J8JEiaAo7v=2`F;YYPY)B% zATOEDo&L~2J3Bn+-_Oos@=TvFwccpvj5$gYVLFF|_@o*AGtM|kWJwunO3O%t?6Qm! zB`**JxL_cIv`+-PtPgN8&87v~Bz^<{;L=TCfWT1iigiWWK`v?;C<+Tru4(;DQ&ONv z3M9wDz0O+G8BjV&gjE*8hl=X#!jGM6+9{~2lOz|ROYU3 zm;#+{8rD=uwf(4tFMwACNCG!jA%j=h%xOi}bX4IxM2^_Hbs~%I&WQCxHWl`X390F} zz#STCr~D|;{juaf&`(ho2Q(2;6XIW4{qOa!HgfXc25J5%|2OBZsD8L_maG{&gPiTKD1HbMCqi=i#6Iu-9HQd-lxir`a>> z+g>PozSlQd^(4&tIgSc2l}CGpcqJDn43e0zTJ^eEomtwWzygdU_X(5J_#Ul5+Ih#x z=!{7bp(G?%ZODeq=6y-S*-`cMhgwamqVB_8IjfUA3O(D zxznFqVtjCzgsZMKcIzX$XfCHK|ng*O)sE1dupL59x{`V_Yc`@TsX=+$1oAKJ&ch8Jm zM!J=oF4D%BlIknY8-4xf$ik3tiFf|%&G%_B#-nWlltw90_P@R)+O>!(`~5^lJgrb2 zi8mloAD7A&sgskm(3*37wU`z8jkTWPu$)(QG;!m0`C+igmY&JY`isexuFNUzp%4qj zNVG_NdMg^psgT-nOUD&KYj+KZ`5tWl(uhLr=a4V%J%`Dn7hk+@L$upV*XDf`V#3xq zH0^#VS7ZiH1nF}fvmy}yX>x9=ViR^rkLk{_821q5oz?Dit3cQWO#NIg$S5X6kctPs zY&HjNB32O8yqxnB>=gSs5!pU`@&SyKB|22;=JGExvu9hT^uE_@< z1-E*_aTA{K92u$m)_3p;VSqph{V>GfY7vt)@7Obxd0fBqkDTcKVn(mAOP>fteV4Va(KeZ@^A%)YUo^*5heA z2avtjY12qhH#-*{WVxCKQZRhR<_$C4EVM$#Pvj0y87|(kkpcyp{~a z=@hveMuRXn*B(_lCf@9!8I|wkWxQ}(tFnKU_aSiDM03O=eLGo5gy^;FT))_v&G)>h z9vK)_4qw`CGNturWr$1A`K$iiQra$5=bze}Qnumx9M@kW5#RoBc|efFBDbF!@W%8c zUVk08&+8}o$+<>IZD^HumJKD!a8%+c-t=P|?q)P;>e}`z>|6BZ&N!Q+^4}r04>p}$#(yJRBFwP1)7$RQ^7(>f{$_BDVq^jFd&)+iki1_4FD=$1j592q z#zT{TfPbFHc{*0kYEHpZ ztp^s%d(5L3twLD;kjAp=aY^bBa007KyYDy#4?Q9pEXuqH@u%i{WW#bknf}S53yMSB z9=y1UN~ii@QrvrA4;#GUR?QGH-y=_Q=8yjCMESN)aI~IzXQb|$S7pIA~5Dkch!l-GLDQ= z_9sXMX0vZu?kE??&D>3PLYwSdw+m$V{=%fC)w7Y4*Neb%K+OssF7L&Si*%!yh80mc z0P~ItA>PIxZdx0(Jwm%-!nT%`r5ZP$g_$)Fv);RL4dYwJeHY8}0H4$6!Pe?qHg$3`bCkP*K zV%u*j8;X{rE=DP|Vhq^=QwI2@wMVvebYD*{ie(OOd-};PEt_<;9nV1$h^(H`O)oFQ zK8J@xa)2zNV7>k*A9~BI+&Tp>R^W48SZi|eg-xP-%~i{ZpsMdJNhSG7U+srKV&xR! zH-D4QN-ikaiim;3QYDbD-XSMn!0JsTrPEk?&6F+#{RFr6HlRv5@fUjQ4 z#binqw?VwISu97LX8s-R_F%5YKK<1_b+tgX-@>H2=x_k3<6v&JO}lD!cNM5Px3w&< zW-(4*>w{l}CN0ezh`YygNI3X_X8qSGEeyC1DQg}G<*^~D#gBh-<9957IUiTt^fFO1F^DX~AY@%2qs0K5bN8!`jGaQ}V*7deDCIBO zlvoQX5{tLme1z13VL{>EYF8z`tYQs#LpEXUX_DtgY=Y|aoMir?2*W3_7Bu>kG`8JT z3$NE_us3e+0)(gg>-w%%eBaE%HzPx}sY>I3`7eXm!yBK&4X#|DVR_Smb6;C5sMT~C zsh2F?4$i+SvUx_hM%K0^T|4YG>t-?mV!r+6N#Jad7jUj}ehb(lgZ!eJWZ|>d^&W7o z(U3)u7+g?8c-~-=wVCBBq+?ByqJv6Zf))*eYR0V`O-unbNJm9w{in!W{esoYWJ&_$8A^AtBNhk?q zKm1iOQ5S)C(I?eL*j2w!1-yz(j{8-M53pS(oKcLuxP z6!w+WFM6gkPzUc$PN*-C->L}B4yAH`v$SW?>~^!C9uBymYUdcw6? zrK3Qf1DBgdHFlaKwV1tp1)o_KGsOpZ6Y88c3<*n;cTsSGCawHKmX&ug9Ac&NS61;K zbJ9tR{Dzzp-i(4X|B)w?@qfz|FdIjR&m z9>le>ED zy$3U6>5`WpNv0^?kK%m52e#K%?c_?9J1hxs%Gb?nm>W&$@feKjYejA&3mg54ac4 z2(Ck1DP?oR&kZV8uYgH8`1bbF*WtVq;T+Q)Kx`kGC%&GGz)FVaK~#2)@mev+t_f|B z%K(_-6s&Tv-7*_b&y_cDrb+TXoIOFqKAMWl`u){kc=s!4^~q~AQbAtjxr}$DCryoG z8frE3s=<=D19WkKZqL_Klj^k?lGWVDr=STR}gi zgJE%<*2bfmJFasit~~-&_iKgoUh9{rz&qe=9L*bs#Q~e9XEWb>%(IToN;TdRHQz=k zjRD$p;7{v*yI@~@Mr*Z@<#Uw57%F0rs6b>T0TT7IW&~?w`2+${fNWY6F7{+!G0$(0 z$%x>lQqJeH&MUJ}LRqNanW*v1^TN#T@$~cY>xQ^g)5F+Y_?|DEO}Ofj5&g*VaZ=Y# z!gzPWI4EIx{9&eYL1pMqXL5W$Jf$=rXX_`V_b;@K>^EWLeGhI6>wzra8&Mbpbu1?S_gF z6EZ_3Fklz&BBlM!HWSO|%z|JhJf>xI4)dr6#$1Nb8Ny{`TG=iFe_{?oFMw@w z=s9FSjfGr7iJ8XE_IHPkEY3e{&1dngoCJiCKbbz!uxx$SB z078&Ld*5uW!&2x?c5B*@LQ*uheLH##-80ge4A>@1dGHR3o|-SwN08qAzy3DtLYi*N^k zCE)M4aj|ra7tHra#TduZ4Af}{o5^)uU#S9{EnWnh+Q&hSjodx zbt~lGd3@rhM0{ow_0HrEMIIjT9G*76x)mO%g~^cYHBNf<&04B#&7VHsyrJoImIQJ9 z@yvmoVqP@xbBrOJX)7;ZVW;90c`@+FY$*tT_G5NEs6cA;scUc2BfO`rW{^qKopz{c zSC_+1ZMwuH(@l0Z60bGi(8WOgB1qG|yXN4d`fB)k07{6Ga5VJg_+q~>3s_Dp-4-n$ zyzS#lXm{SdHPEP!JsPJvgye7^sV>&heQG9L|2;z%@HwURS1`U_;@DKS7-p%NVvl9=>Al96t*$55!iLRT zR4g1@)d?5VutQTQPJ_YvTO}k>IAV3!u^qEFfv|3TiqDi69}}>bz0>k~a(-P`@ck<+jOUK|)7$0j%tc<*Ik)iLq&D{I~Q zpxPJ>tU~*!L&UqG<`qzs0;h70?~-zMNc7bJfT>E_n49$T4Xz;pQ5i2GIeupi!WqdS zaH_6|P^@=m6Mc5)oR?JctT>!DrTaI=sYergh?c%aE49IBn>W(Rk#YzsM;^qA zqh8y057exHP82NtUew5-Gc*7FA!6#MN=q?oGaWP=6%k$!eST=&xwTjHnc3G!J)*X7 zu(<21Q^8^U;2dHQb{Bvi*Ku*v?i>7RN`3YrwlVu~W_@yOrP#D0TYOZ%gz8>e_4jlV zmN^YGN)#*~2}b4X&UKA~l*a)qw|rR4S~cIKk1$xOY1n0Zl#4s5FvHFtGr@vtkIL2# zCPwkyT_s6)gubonNVv-9vuXatA=R>c&2W`E)<5S?uU4k{c`M2csz;~!g$)XM%?t|l z{(O-3s~$S;d2JBpSO7hl-FJId&VRztC3;&~sAk3Omhv4>=OzB`w+)$Z7!|GPrt}k` zLB{z#Cu8H|bK4Kvh{dVOiD>x(cC%DXc9xvBH+6||PIYY7y7r|-1qN@iC>env^2qA^ELg?SzKC1fY8O)t z3%_sBm;-0=cItj2)Ba$5i{p+^JA`BqGS<_YqGaCwgmUrY(1jEF$1=Yd<{iidBCzr99^{F!d4CQ*6@Fi}6S8+#LjeRg;AsB!qc zcQ&G2;i@I)bBSv0t01oR*v~?e%$wK#PQP>8ZyYm-rPe}5-nm(Clcz)C`Slo^5&&Uct7Psi q-KdZEgz`R414M-a{}76_0~r Date: Thu, 4 Nov 2021 10:54:34 -0500 Subject: [PATCH 034/163] print full error message, not just something went wrong --- lib/datura/file_type.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 0d4d755d8..378aa734f 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -23,9 +23,9 @@ def initialize(location, options) @file_location = location @options = options add_xsl_params_options - # set output directories output = File.join(@options["collection_dir"], "output", @options["environment"]) + @out_es = File.join(output, "es") @out_html = File.join(output, "html") @out_iiif = File.join(output, "iiif") @@ -127,10 +127,12 @@ def transform_es if results.length == 0 raise "No possible xpaths found fo file #{self.filename}, check if XML is valid or customize 'subdoc_xpaths' method" end + subdoc_xpaths.each do |xpath, classname| subdocs = file_xml.xpath(xpath) subdocs.each do |subdoc| file_transformer = classname.new(subdoc, @options, file_xml, self.filename(false)) + es_req << file_transformer.json end end @@ -141,6 +143,7 @@ def transform_es return es_req rescue => e puts "something went wrong transforming #{self.filename}" + puts e raise e end end From 0df2257381217824a4bd3281e23a69dad13b206e Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 4 Nov 2021 10:55:31 -0500 Subject: [PATCH 035/163] fix xpath --- lib/datura/file_types/file_ead.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/datura/file_types/file_ead.rb b/lib/datura/file_types/file_ead.rb index 110b9d5c0..ebc1f9be3 100644 --- a/lib/datura/file_types/file_ead.rb +++ b/lib/datura/file_types/file_ead.rb @@ -11,7 +11,8 @@ class FileEad < FileType def initialize(file_location, options) super(file_location, options) - @script_html = File.join(options["collection_dir"], options["ead_html_xsl"]) # There needs to be an xsl file to transform into html + @script_html = File.join(options["collection_dir"], options["ead_html_xsl"]) + # There needs to be an xsl file to transform into html # I don't think we need solr at this point) # @script_solr = File.join(options["collection_dir"], options["tei_solr_xsl"]) end @@ -19,7 +20,7 @@ def initialize(file_location, options) def subdoc_xpaths # match subdocs against classes return { - "/EAD" => EadToEs, + "/ead" => EadToEs, # "//dsc/c01" => EadToEsItems, } end From 3ce120d6a547488f471d11c9fd5d28683101c943 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 4 Nov 2021 11:10:23 -0500 Subject: [PATCH 036/163] add all require fields, including unfilled ones --- lib/datura/to_es/ead_to_es/fields.rb | 226 +++++++++++++-------------- 1 file changed, 111 insertions(+), 115 deletions(-) diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb index 9c8f35e6c..39e4ca500 100644 --- a/lib/datura/to_es/ead_to_es/fields.rb +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -15,19 +15,19 @@ def id # "https://cdrhapi.unl.edu/doc/#{@id}" # end - # def annotations_text - # # TODO what should default behavior be? - # end + def annotations_text + # TODO what should default behavior be? + end - # def category - # category = get_text(@xpaths["category"]) - # return category.length > 0 ? CommonXml.normalize_space(category) : "none" - # end + def category + end # note this does not sort the creators def creator creators = get_list(@xpaths["creators"]) - return creators.map { |creator| { "name" => CommonXml.normalize_space(creator) } } + if creators + return creators.map { |creator| { "name" => CommonXml.normalize_space(creator) } } + end end # returns ; delineated string of alphabetized creators @@ -43,20 +43,20 @@ def collection_desc @options["collection_desc"] || @options["collection"] end - # def contributor - # contribs = [] - # @xpaths["contributors"].each do |xpath| - # eles = @xml.xpath(xpath) - # eles.each do |ele| - # contribs << { - # "id" => ele["id"], - # "name" => CommonXml.normalize_space(ele.text), - # "role" => CommonXml.normalize_space(ele["role"]) - # } - # end - # end - # contribs.uniq - # end + def contributor + # contribs = [] + # @xpaths["contributors"].each do |xpath| + # eles = @xml.xpath(xpath) + # eles.each do |ele| + # contribs << { + # "id" => ele["id"], + # "name" => CommonXml.normalize_space(ele.text), + # "role" => CommonXml.normalize_space(ele["role"]) + # } + # end + # end + # contribs.uniq + end def data_type "ead" @@ -67,9 +67,9 @@ def date(before=true) return CommonXml.date_standardize(datestr, before) end - # def date_display - # get_text(@xpaths["date_display"]) - # end + def date_display + get_text(@xpaths["date_display"]) + end def date_not_after date(false) @@ -88,85 +88,80 @@ def extent end def format - matched_format = nil - # iterate through all the formats until the first one matches - @xpaths["formats"].each do |type, xpath| - text = get_text(xpath) - matched_format = type if text && text.length > 0 - end - matched_format + get_list(@xpaths["format"]) end - # def image_id - # # Note: don't pull full path because will be pulled by IIIF - # images = get_list(@xpaths["image_id"]) - # images[0] if images - # end + def image_id + # Note: don't pull full path because will be pulled by IIIF + # How to deal with this? + images = get_list(@xpaths["image_id"]) + images[0] if images + end - # def keywords - # get_list(@xpaths["keywords"]) - # end + def keywords + get_list(@xpaths["keywords"]) + end - # def language - # get_text(@xpaths["language"]) - # end + def language + get_text(@xpaths["language"]) + end - # def languages - # get_list(@xpaths["languages"]) - # end + def languages + get_list(@xpaths["languages"]) + end def medium # Default behavior is the same as "format" method format end - # def person - # # TODO will need some examples of how this will work - # # and put in the xpaths above, also for attributes, etc - # # should contain name, id, and role - # eles = @xml.xpath(@xpaths["person"]) - # people = eles.map do |p| - # { - # "id" => "", - # "name" => CommonXml.normalize_space(p.text), - # "role" => CommonXml.normalize_space(p["role"]) - # } - # end - # return people - # end + def person + # TODO will need some examples of how this will work + # and put in the xpaths above, also for attributes, etc + # should contain name, id, and role + # eles = @xml.xpath(@xpaths["person"]) + # people = eles.map do |p| + # { + # "id" => "", + # "name" => CommonXml.normalize_space(p.text), + # "role" => CommonXml.normalize_space(p["role"]) + # } + # end + # return people + end - # def people - # @json["person"].map { |p| CommonXml.normalize_space(p["name"]) } - # end + def people + # @json["person"].map { |p| CommonXml.normalize_space(p["name"]) } + end - # def places - # return get_list(@xpaths["places"]) - # end + def places + return get_list(@xpaths["places"]) + end - # def publisher - # get_text(@xpaths["publisher"]) - # end + def publisher + get_text(@xpaths["publisher"]) + end - # def recipient - # eles = @xml.xpath(@xpaths["recipient"]) - # people = eles.map do |p| - # { - # "id" => "", - # "name" => CommonXml.normalize_space(p.text), - # "role" => "recipient" - # } - # end - # return people - # end + def recipient + # eles = @xml.xpath(@xpaths["recipient"]) + # people = eles.map do |p| + # { + # "id" => "", + # "name" => CommonXml.normalize_space(p.text), + # "role" => "recipient" + # } + # end + # return people + end - # def rights - # # Note: override by collection as needed - # get_text(@xpaths["rights"]) - # end + def rights + # Note: override by collection as needed + get_text(@xpaths["rights"]) + end - # def rights_holder - # get_text(@xpaths["rights_holder"]) - # end + def rights_holder + get_text(@xpaths["rights_holder"]) + end def rights_uri # by default collections have no uri associated with them @@ -174,27 +169,28 @@ def rights_uri # to return specific string or xpath as required end - # def source - # get_text(@xpaths["source"]) - # end + def source + get_text(@xpaths["source"]) + end - # def subjects - # get_list(@xpaths["subjects"]) - # end + def subjects + get_list(@xpaths["subjects"]) + end - # def subcategory - # subcategory = get_text(@xpaths["subcategory"]) - # subcategory.length > 0 ? subcategory : "none" - # end + def subcategory + # subcategory = get_text(@xpaths["subcategory"]) + # subcategory.length > 0 ? subcategory : "none" + end def text # handling separate fields in array # means no worrying about handling spacing between words text = [] - @xpaths.keys.each do [xpath] + @xpaths.keys.each do |xpath| body = get_text(@xpaths[xpath]) text << body end + text # TODO: do we need to preserve tags like in text? if so, turn get_text to true # text << CommonXml.convert_tags_in_string(body) # text += text_additional @@ -223,26 +219,26 @@ def type get_text(@xpaths["type"]) end - # def topics - # get_list(@xpaths["topic"]) - # end + def topics + get_list(@xpaths["topic"]) + end - # def uri - # # override per collection - # # should point at the live website view of resource - # end + def uri + # override per collection + # should point at the live website view of resource + end - # def uri_data - # base = @options["data_base"] - # subpath = "data/#{@options["collection"]}/source/tei" - # return "#{base}/#{subpath}/#{@id}.xml" - # end + def uri_data + base = @options["data_base"] + subpath = "data/#{@options["collection"]}/source/tei" + return "#{base}/#{subpath}/#{@id}.xml" + end - # def uri_html - # base = @options["data_base"] - # subpath = "data/#{@options["collection"]}/output/#{@options["environment"]}/html" - # return "#{base}/#{subpath}/#{@id}.html" - # end + def uri_html + base = @options["data_base"] + subpath = "data/#{@options["collection"]}/output/#{@options["environment"]}/html" + return "#{base}/#{subpath}/#{@id}.html" + end def works # TODO need to create a list of items, maybe an array of ids From 812322de8be8d5f17f51bba0b6514ba09dbc405a Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 4 Nov 2021 11:19:56 -0500 Subject: [PATCH 037/163] fix xpaths hash --- lib/datura/to_es/ead_to_es/xpaths.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/datura/to_es/ead_to_es/xpaths.rb b/lib/datura/to_es/ead_to_es/xpaths.rb index 10a1147b5..a1bf82885 100644 --- a/lib/datura/to_es/ead_to_es/xpaths.rb +++ b/lib/datura/to_es/ead_to_es/xpaths.rb @@ -11,7 +11,7 @@ def xpaths_list # "/TEI/teiHeader/revisionDesc/change/name", # "/TEI/teiHeader/fileDesc/titleStmt/editor" # ], - "creators" => ["/ead/archdesc/did/origination/persname", "/ead/eadheader/filedesc/titlestmt/creator"] + "creators" => ["/ead/archdesc/did/origination/persname", "/ead/eadheader/filedesc/titlestmt/creator"], "date" => "/ead/eadheader/filedesc/publicationstmt/date", "description" => "/ead/archdesc/scopecontent/p", # a non normalized date taken directly from the source material ("Dec 9", "Winter 1889") @@ -31,8 +31,10 @@ def xpaths_list "rights" => "/ead/archdesc/descgrp/accessrestrict/p", "rights_holder" => "ead/archdesc/did/repository/corpname", "source" => "/ead/archdesc/descgrp/prefercite/p", - "subjects" => "/ead/archdesc/controlaccess/*[not(name()="head")]"], + "subjects" => "/ead/archdesc/controlaccess/*[not(name()='head')]", # "subcategory" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='subcategory'][1]/term", "titles" => "ead/archdesc/did/unittitle", "text" => "/ead/eadheader/filedesc/titlestmt/*", - } + }.merge(override_xpaths) + end + end From 1f203147c8bc5471ebb8dfeacb7039c29f63fd68 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 4 Nov 2021 11:27:45 -0500 Subject: [PATCH 038/163] make EadToEsItems a separate class --- lib/datura/to_es/ead_to_es_items.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/to_es/ead_to_es_items.rb b/lib/datura/to_es/ead_to_es_items.rb index ff6ec3e9c..560a4a76c 100644 --- a/lib/datura/to_es/ead_to_es_items.rb +++ b/lib/datura/to_es/ead_to_es_items.rb @@ -27,7 +27,7 @@ # collections of fields being sent to elasticsearch # you can override individual chunks of fields in your collection -class EadToEs < XmlToEs +class EadToEsItems < XmlToEs # Override XmlToEs methods that need to be customized for EAD here # rather than in one of the files in ead_to_es/ def get_id From 9f01cd0aedfe6f9e6a5ddb94aa3d2d27a6dc2a19 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 4 Nov 2021 13:22:03 -0500 Subject: [PATCH 039/163] add abstract field and fix bad xpaths --- lib/datura/to_es/ead_to_es/fields.rb | 4 ++++ lib/datura/to_es/ead_to_es/xpaths.rb | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb index 39e4ca500..44436709a 100644 --- a/lib/datura/to_es/ead_to_es/fields.rb +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -15,6 +15,10 @@ def id # "https://cdrhapi.unl.edu/doc/#{@id}" # end + def abstract + get_text(@xpaths["abstract"]) + end + def annotations_text # TODO what should default behavior be? end diff --git a/lib/datura/to_es/ead_to_es/xpaths.rb b/lib/datura/to_es/ead_to_es/xpaths.rb index a1bf82885..b947f6bb9 100644 --- a/lib/datura/to_es/ead_to_es/xpaths.rb +++ b/lib/datura/to_es/ead_to_es/xpaths.rb @@ -5,7 +5,7 @@ class EadToEs < XmlToEs # in that file which returns a different value def xpaths_list { - # "abstract" => "/ead/archdesc/did/abstract" + "abstract" => "/ead/archdesc/did/abstract", # "category" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='category'][1]/term", # "contributors" => [ # "/TEI/teiHeader/revisionDesc/change/name", @@ -27,13 +27,13 @@ def xpaths_list # "places" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='places']/term", "publisher" => "/ead/eadheader/filedesc/publicationstmt/publisher", # "recipient" => "/TEI/teiHeader/profileDesc/particDesc/person[@role='recipient']/persName", - "repository_contact" => "/ead/archdesc/did/repository/addresses", + "repository_contact" => "/ead/archdesc/did/repository/address/*", "rights" => "/ead/archdesc/descgrp/accessrestrict/p", - "rights_holder" => "ead/archdesc/did/repository/corpname", + "rights_holder" => "/ead/archdesc/did/repository/corpname", "source" => "/ead/archdesc/descgrp/prefercite/p", "subjects" => "/ead/archdesc/controlaccess/*[not(name()='head')]", # "subcategory" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='subcategory'][1]/term", - "titles" => "ead/archdesc/did/unittitle", + "title" => "/ead/archdesc/did/unittitle", "text" => "/ead/eadheader/filedesc/titlestmt/*", }.merge(override_xpaths) end From f05e53526eaba575ed5f2d4ab1aa1f8cc35de1e7 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 8 Nov 2021 12:55:46 -0600 Subject: [PATCH 040/163] add a backtrace to error handling --- lib/datura/file_type.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 378aa734f..12a66b25f 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -69,6 +69,7 @@ def post_es(es) begin RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, {:content_type => :json } ) rescue => e + # byebug error = "Error transforming or posting to ES for #{self.filename(false)}: #{e.response}" end else @@ -132,7 +133,6 @@ def transform_es subdocs = file_xml.xpath(xpath) subdocs.each do |subdoc| file_transformer = classname.new(subdoc, @options, file_xml, self.filename(false)) - es_req << file_transformer.json end end @@ -144,6 +144,7 @@ def transform_es rescue => e puts "something went wrong transforming #{self.filename}" puts e + puts e.backtrace raise e end end From 1805a2d40957be14cf389ea48545ee372de66b8a Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 8 Nov 2021 12:59:42 -0600 Subject: [PATCH 041/163] grab 'items' at any nesting of the EAD --- lib/datura/file_types/file_ead.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/file_types/file_ead.rb b/lib/datura/file_types/file_ead.rb index ebc1f9be3..a809ab9b4 100644 --- a/lib/datura/file_types/file_ead.rb +++ b/lib/datura/file_types/file_ead.rb @@ -21,7 +21,7 @@ def subdoc_xpaths # match subdocs against classes return { "/ead" => EadToEs, - # "//dsc/c01" => EadToEsItems, + "//*[@level='item']" => EadToEsItems, } end From 8f74d38138ecee515fcc57d9772f4b0159210454 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 8 Nov 2021 13:09:45 -0600 Subject: [PATCH 042/163] add xpaths and fields, and make sure eadtoesitems inherits from eadtoes --- lib/datura/to_es/ead_to_es_items.rb | 7 +- lib/datura/to_es/ead_to_es_items/fields.rb | 128 ++++++++++---------- lib/datura/to_es/ead_to_es_items/request.rb | 2 +- lib/datura/to_es/ead_to_es_items/xpaths.rb | 26 ++-- 4 files changed, 82 insertions(+), 81 deletions(-) diff --git a/lib/datura/to_es/ead_to_es_items.rb b/lib/datura/to_es/ead_to_es_items.rb index 560a4a76c..c4778cf3d 100644 --- a/lib/datura/to_es/ead_to_es_items.rb +++ b/lib/datura/to_es/ead_to_es_items.rb @@ -1,4 +1,4 @@ -require_relative "xml_to_es.rb" +require_relative "ead_to_es.rb" require_relative "ead_to_es_items/fields.rb" require_relative "ead_to_es_items/request.rb" require_relative "ead_to_es_items/xpaths.rb" @@ -27,10 +27,7 @@ # collections of fields being sent to elasticsearch # you can override individual chunks of fields in your collection -class EadToEsItems < XmlToEs +class EadToEsItems < EadToEs # Override XmlToEs methods that need to be customized for EAD here # rather than in one of the files in ead_to_es/ - def get_id - @id - end end diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index b5ca56d79..95869cb8f 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -1,4 +1,4 @@ -class EadToEsItems < XmlToEs +class EadToEsItems < EadToEs # Note to add custom fields, use "assemble_collection_specific" from request.rb # and be sure to either use the _d, _i, _k, or _t to use the correct field type @@ -15,14 +15,14 @@ def id # "https://cdrhapi.unl.edu/doc/#{@id}" # end - # def annotations_text - # # TODO what should default behavior be? - # end + def annotations_text + # TODO what should default behavior be? + end - # def category - # category = get_text(@xpaths["category"]) - # return category.length > 0 ? CommonXml.normalize_space(category) : "none" - # end + def category + # category = get_text(@xpaths["category"]) + # return category.length > 0 ? CommonXml.normalize_space(category) : "none" + end # note this does not sort the creators def creator @@ -36,30 +36,30 @@ def creator_sort end def collection - "manuscripts" + # "manuscripts" end def collection_desc - @options["collection_desc"] || @options["collection"] - end - - # def contributor - # contribs = [] - # @xpaths["contributors"].each do |xpath| - # eles = @xml.xpath(xpath) - # eles.each do |ele| - # contribs << { - # "id" => ele["id"], - # "name" => CommonXml.normalize_space(ele.text), - # "role" => CommonXml.normalize_space(ele["role"]) - # } - # end - # end - # contribs.uniq - # end + # @options["collection_desc"] || @options["collection"] + end + + def contributor + # contribs = [] + # @xpaths["contributors"].each do |xpath| + # eles = @xml.xpath(xpath) + # eles.each do |ele| + # contribs << { + # "id" => ele["id"], + # "name" => CommonXml.normalize_space(ele.text), + # "role" => CommonXml.normalize_space(ele["role"]) + # } + # end + # end + # contribs.uniq + end def data_type - "ead" + "ead_item" end def date(before=true) @@ -80,23 +80,31 @@ def date_not_before end def description - # Note: override per collection as needed + get_text(@xpaths["description"]) end def format - matched_format = nil - # iterate through all the formats until the first one matches - @xpaths["formats"].each do |type, xpath| - text = get_text(xpath) - matched_format = type if text && text.length > 0 + # matched_format = nil + # # iterate through all the formats until the first one matches + # @xpaths["formats"].each do |type, xpath| + # text = get_text(xpath) + # matched_format = type if text && text.length > 0 + # end + # matched_format + end + def get_id + # doc = id + doc = get_text(@xpaths["identifier"]) + if doc == "" + byebug end - matched_format + return "#{@filename}_#{doc}" end def image_id - # Note: don't pull full path because will be pulled by IIIF - images = get_list(@xpaths["image_id"]) - images[0] if images + # # Note: don't pull full path because will be pulled by IIIF + # images = get_list(@xpaths["image_id"]) + # images[0] if images end def keywords @@ -120,19 +128,19 @@ def person # TODO will need some examples of how this will work # and put in the xpaths above, also for attributes, etc # should contain name, id, and role - eles = @xml.xpath(@xpaths["person"]) - people = eles.map do |p| - { - "id" => "", - "name" => CommonXml.normalize_space(p.text), - "role" => CommonXml.normalize_space(p["role"]) - } - end - return people + # eles = @xml.xpath(@xpaths["person"]) + # people = eles.map do |p| + # { + # "id" => "", + # "name" => CommonXml.normalize_space(p.text), + # "role" => CommonXml.normalize_space(p["role"]) + # } + # end + # return people end def people - @json["person"].map { |p| CommonXml.normalize_space(p["name"]) } + # @json["person"].map { |p| CommonXml.normalize_space(p["name"]) } end def places @@ -144,20 +152,20 @@ def publisher end def recipient - eles = @xml.xpath(@xpaths["recipient"]) - people = eles.map do |p| - { - "id" => "", - "name" => CommonXml.normalize_space(p.text), - "role" => "recipient" - } - end - return people + # eles = @xml.xpath(@xpaths["recipient"]) + # people = eles.map do |p| + # { + # "id" => "", + # "name" => CommonXml.normalize_space(p.text), + # "role" => "recipient" + # } + # end + # return people end def rights # Note: override by collection as needed - "All Rights Reserved" + get_text(@xpaths["rights"]) end def rights_holder @@ -205,11 +213,7 @@ def text_additional end def title - title = get_text(@xpaths["titles"]["main"]) - if title.empty? - title = get_text(@xpaths["titles"]["alt"]) - end - return title + title = get_text(@xpaths["titles"]) end def title_sort diff --git a/lib/datura/to_es/ead_to_es_items/request.rb b/lib/datura/to_es/ead_to_es_items/request.rb index 27f14d072..45e5f28c5 100644 --- a/lib/datura/to_es/ead_to_es_items/request.rb +++ b/lib/datura/to_es/ead_to_es_items/request.rb @@ -1,4 +1,4 @@ -class EadToEsItems < XmlToEs +class EadToEsItems < EadToEs # please refer to generic xml to es request file, request.rb # and override methods specific to TEI transformation here diff --git a/lib/datura/to_es/ead_to_es_items/xpaths.rb b/lib/datura/to_es/ead_to_es_items/xpaths.rb index bab2fa935..cba8d7bf9 100644 --- a/lib/datura/to_es/ead_to_es_items/xpaths.rb +++ b/lib/datura/to_es/ead_to_es_items/xpaths.rb @@ -1,44 +1,44 @@ -class EadToEsItems < XmlToEs +class EadToEsItems < EadToEs # These are the default xpaths that are used for collections # if you require a different xpath, please override the xpath in # the specific collection's TeiToEs file or create a new method # in that file which returns a different value def xpaths_list { - "abstract" => "/ead/archdesc/did/abstract", + "abstract" => "did/abstract", # "category" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='category'][1]/term", # "contributors" => [ # "/TEI/teiHeader/revisionDesc/change/name", # "/TEI/teiHeader/fileDesc/titleStmt/editor" - # ], - "creators" => ["/ead/archdesc/did/origination/persname", "/ead/eadheader/filedesc/titlestmt/creator"], - "date" => "/ead/archdesc/dsc/c01/did/unitdate", - "description" => "/ead/archdesc/dsc/c01/scopecontent/p", + # # ], + # "creators" => ["/ead/archdesc/did/origination/persname", "/ead/eadheader/filedesc/titlestmt/creator"], + "date" => "did/unitdate", + "description" => "scopecontent/p", # a non normalized date taken directly from the source material ("Dec 9", "Winter 1889") # "date_display" => "/TEI/teiHeader/fileDesc/sourceDesc/bibl/date", - "extent" => "/ead/archdesc/dsc/c01/did/physdesc/extent", - "format" => "/ead/archdesc/dsc/c01/did/physdesc/physfacet", - "image_url" => "/ead/archdesc/dsc/c01/dao/@href", + "extent" => "did/physdesc/extent", + "format" => "did/physdesc/physfacet", + "image_url" => "dao/@href", # "image_id" => "/TEI/text//pb/@facs", # "keywords" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='keywords']/term", # note: language is global attribute xml:lang - "identifier" => "/ead/archdesc/dsc/c01/did/unitid[@type='WWA']", + "identifier" => "did/unitid[@type='WWA']", # "language" => "(//body/div1/@lang)[1]", # "languages" => "//body/div1/@lang", # "person" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='people']/term", # "places" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='places']/term", # "publisher" => "/TEI/teiHeader/fileDesc/sourceDesc/bibl[1]/publisher[1]", # "recipient" => "/TEI/teiHeader/profileDesc/particDesc/person[@role='recipient']/persName", - "repository_id" => "/ead/archdes/dsc/c01/did/unitid[@type='repository']", + "repository_id" => "did/unitid[@type='repository']", # "rights" => "/ead/archdesc/descgrp/accessrestrict/p", # "rights_holder" => "/ead/archdesc/descgrp/accessrestrict/p", # "source" => "/ead/archdesc/descgrp/prefercite/p", # "subjects" => ["/ead/archdesc/controlaccess/[everything after head;persname, subject]"], # "subcategory" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='subcategory'][1]/term", # "text" => "//text", - "title" => "ead/archdesc/dsc/c01/did/unittitle", + "title" => "did/unittitle", # "topic" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='topic']/term", - "type" => "/ead/archdesc/dsc/c01/did/physdesc/genreform", + "type" => "did/physdesc/genreform", # "works" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='works']/term", }.merge(override_xpaths) end From fbcf65662e5b190350341a235f3b1db1b1cd890d Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 8 Nov 2021 13:21:23 -0600 Subject: [PATCH 043/163] change order of get id to fix bug --- lib/datura/to_es/xml_to_es.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/datura/to_es/xml_to_es.rb b/lib/datura/to_es/xml_to_es.rb index 3cbbb1e46..8cf50029a 100644 --- a/lib/datura/to_es/xml_to_es.rb +++ b/lib/datura/to_es/xml_to_es.rb @@ -34,9 +34,8 @@ def initialize(xml, options={}, parent_xml=nil, filename=nil) @options = options @parent_xml = parent_xml @filename = filename - @id = get_id @xpaths = xpaths_list - + @id = get_id create_json end From 79e89e7e54b8ebadf080eac4b4d18630802cbc72 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 8 Nov 2021 14:20:01 -0600 Subject: [PATCH 044/163] add documentation for adding new format --- docs/4_developers/new_formats.md | 94 ++++++++++++++++++++++++++++++++ docs/README.md | 1 + 2 files changed, 95 insertions(+) create mode 100644 docs/4_developers/new_formats.md diff --git a/docs/4_developers/new_formats.md b/docs/4_developers/new_formats.md new file mode 100644 index 000000000..a731bb9a7 --- /dev/null +++ b/docs/4_developers/new_formats.md @@ -0,0 +1,94 @@ +# Adding new formats to datura + +## Configuring datura + +In `lib/datura/data_manager.rb`, `self.format_to_class` contains a hash with the format as key and a file with format-specific methods as value. Add your desired format to this hash, with the corresponding file FileFornat. +``` + def self.format_to_class + { + "csv" => FileCsv, + "ead" => FileEad, + "html" => FileHtml, + "tei" => FileTei, + "vra" => FileVra, + } +end +``` +Modify `lib/datura/parser_options/post.rb` to accept parameters for the new format: +``` +options["format"] = nil + opts.on( '-f', '--format [input]', 'Restrict to one format (csv, ead, html, tei, vra, webs)') do |input| + if %w[csv ead html tei vra webs].include?(input) + options["format"] = input + else + puts "Format #{input} is not recognized.".red + puts "Allowed formats are csv, ead, html, tei, vra, and webs (web-scraped html)" + exit + end + end +``` +In the `config/public.yml` file you need to add a link to the xsl scripts for the specific format (you do not necessarily need to create a working script until you need to transform files), and also create a file and add it to the scripts folder: +``` + html_html_xsl: scripts/.xslt-datura/html_to_html/html_to_html.xsl + tei_html_xsl: scripts/.xslt-datura/tei_to_html/tei_to_html.xsl + vra_html_xsl: scripts/.xslt-datura/vra_to_html/vra_to_html.xsl + ead_html_xsl: scripts/.xslt-datura/ead_to_html/ead_to_html.xsl +``` + +## Datura overrides and new files +You will need to create a `file_format.rb` (i.e. `file_ead.rb`) file in `lib/datura/file_types`. Copy from a similar file type (file_tei.rb is a good model for XML=based formats) and make any necessary changes for the file format. In particular the `subdoc_xpaths` should be modified to get the correct XPath for the files you want to transform: +``` +def subdoc_xpaths + # match subdocs against classes + return { + "/ead" => EadToEs, + } + end +``` + +In the `/lib/datura/to_es` folder you also need to make a format_to_es.rb file, i.e. `ead_to_es.rb` and also a folder with fields.rb, request.rb, and xpaths.rb overrides +Be sure to require all the necessary files at the top (and create them in the proper folder). +``` +require_relative "xml_to_es.rb" +require_relative "ead_to_es/fields.rb" +require_relative "ead_to_es/request.rb" +require_relative "ead_to_es/xpaths.rb" + + + +class EadToEs < XmlToEs +end +``` +The new files you have added must to be required in `lib/datura/requirer.rb`. Add the following to make sure they get picked up: +``` +require_relative "to_es/ead_to_es.rb" +``` +All code in these files should be within the same class, inheriting from XmlToEs. +``` +class EadToEs < XmlToEs +end +``` + +## Xpaths +Add all the xpaths for your desired fields in xpaths.rb, in the hash inside the xpaths.rb file. it may be helpful to use an existing template like `tei_to_es/xpaths.rb`. There is no need to add all of them, you can comment out the fields you do not need. + +## Overrides with specific fields +All fields must be defined within fields.rb. Even if you do not intend to index them, Datura requires that you at least have an empty method defining each field. (An empty field will be nil and not be displayed in Orchid). +``` +def category +end +``` +Make appropriate changes to your fields as desired. + +## Dealing with subsections of XML files +If you want to index subsections, the best way to do this is to define an xpaths selector for the desired sections in the `subdoc_xpaths` method as described above. +``` +def subdoc_xpaths + # match subdocs against classes + return { + "/ead" => EadToEs, + "//*[@level='item']" => EadToEsItems, + } + end +``` +Then add all the necessary overrides in the `to_es` folder like above. Depending on what you need to override, you may combine them into one file, or have separate files. In any case, they should inherit from the main file, i.e. `class TeiToEsPersonography < TeiToEs`. diff --git a/docs/README.md b/docs/README.md index 7e8487915..75618aa9d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -39,6 +39,7 @@ The files are parsed and formatted into documents appropriate for Solr, IIIF, El - [Ruby / Gems](4_developers/ruby_gems.md) - Class organization - [Tests](4_developers/test.md) + - [Add new formats to Datura](4_developers/new_formats.md) - More - [Troubleshooting](troubleshooting.md) - [XSLT to Ruby reference](xslt_to_ruby_reference.md) From 44a379042f79c31dac61336d991d4186778a70c2 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 9 Nov 2021 11:02:39 -0600 Subject: [PATCH 045/163] adjust and add fields for items --- lib/datura/to_es/ead_to_es_items/fields.rb | 41 +++++++++++++--------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index 95869cb8f..60a266445 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -20,15 +20,13 @@ def annotations_text end def category - # category = get_text(@xpaths["category"]) - # return category.length > 0 ? CommonXml.normalize_space(category) : "none" end # note this does not sort the creators - def creator - creators = get_list(@xpaths["creators"]) - return creators.map { |creator| { "name" => CommonXml.normalize_space(creator) } } - end + # def creator + # creators = get_list(@xpaths["creators"]) + # return creators.map { |creator| { "name" => CommonXml.normalize_space(creator) } } + # end # returns ; delineated string of alphabetized creators def creator_sort @@ -36,7 +34,7 @@ def creator_sort end def collection - # "manuscripts" + "whitman-finding_aid_manuscripts" end def collection_desc @@ -68,7 +66,12 @@ def date(before=true) end def date_display - get_text(@xpaths["date_display"]) + if get_text(@xpaths["date_display"]) == "" + return get_text(@xpaths["date"]) + else + return get_text(@xpaths["date_display"]) + end + end def date_not_after @@ -83,20 +86,20 @@ def description get_text(@xpaths["description"]) end + def extent + get_text(@xpaths["extent"]) + end + def format - # matched_format = nil - # # iterate through all the formats until the first one matches - # @xpaths["formats"].each do |type, xpath| - # text = get_text(xpath) - # matched_format = type if text && text.length > 0 - # end - # matched_format + get_text(@xpaths["format"]) end + def get_id # doc = id doc = get_text(@xpaths["identifier"]) if doc == "" - byebug + title = get_text(@xpaths["file"]) + return "#{@filename}_#{title}" end return "#{@filename}_#{doc}" end @@ -213,7 +216,7 @@ def text_additional end def title - title = get_text(@xpaths["titles"]) + title = get_text(@xpaths["title"]) end def title_sort @@ -225,6 +228,10 @@ def topics get_list(@xpaths["topic"]) end + def type + get_text(@xpaths["type"]) + end + def uri # override per collection # should point at the live website view of resource From e2ab09563d81f0e65ab42ec943c2e45c659cad9a Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 9 Nov 2021 11:10:31 -0600 Subject: [PATCH 046/163] add items to repository xpaths --- lib/datura/to_es/ead_to_es/xpaths.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/datura/to_es/ead_to_es/xpaths.rb b/lib/datura/to_es/ead_to_es/xpaths.rb index b947f6bb9..ddcc4d64f 100644 --- a/lib/datura/to_es/ead_to_es/xpaths.rb +++ b/lib/datura/to_es/ead_to_es/xpaths.rb @@ -35,6 +35,7 @@ def xpaths_list # "subcategory" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='subcategory'][1]/term", "title" => "/ead/archdesc/did/unittitle", "text" => "/ead/eadheader/filedesc/titlestmt/*", + "items" => "//*[@level='item']/did/unitid[@type='WWA']" }.merge(override_xpaths) end end From aa9a9cb83b5564b581f50086ba8e170b71495d45 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 9 Nov 2021 11:22:49 -0600 Subject: [PATCH 047/163] fix image_url xpath --- lib/datura/to_es/ead_to_es_items/xpaths.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/to_es/ead_to_es_items/xpaths.rb b/lib/datura/to_es/ead_to_es_items/xpaths.rb index cba8d7bf9..b39f2d802 100644 --- a/lib/datura/to_es/ead_to_es_items/xpaths.rb +++ b/lib/datura/to_es/ead_to_es_items/xpaths.rb @@ -18,7 +18,7 @@ def xpaths_list # "date_display" => "/TEI/teiHeader/fileDesc/sourceDesc/bibl/date", "extent" => "did/physdesc/extent", "format" => "did/physdesc/physfacet", - "image_url" => "dao/@href", + "image_url" => "did/dao/@href", # "image_id" => "/TEI/text//pb/@facs", # "keywords" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='keywords']/term", # note: language is global attribute xml:lang From 7a9cbeb4252ef58d8a8935c486fb4cabcc81f0ba Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 9 Nov 2021 14:36:35 -0600 Subject: [PATCH 048/163] add puts statements for debugging --- lib/datura/elasticsearch/index.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index a409ca550..b8a3d9bda 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -83,8 +83,9 @@ def get_schema_mapping } schema = get_schema[@options["es_index"]] + puts schema doc = schema["mappings"]["_doc"] - + puts doc doc["properties"].each do |field, value| @schema_mapping["fields"] << field if value["type"] == "nested" From 21b1adc2779ad8ab258ab9de429dfa86eb283064 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 9 Nov 2021 14:42:05 -0600 Subject: [PATCH 049/163] try another way to debug --- lib/datura/elasticsearch/index.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index b8a3d9bda..a19636609 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -83,7 +83,9 @@ def get_schema_mapping } schema = get_schema[@options["es_index"]] - puts schema + if schema == nil || schema == "" + puts "schema is nil!" + end doc = schema["mappings"]["_doc"] puts doc doc["properties"].each do |field, value| From 0511b9f4e0889e25bcea989da511354aa8a3edce Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 9 Nov 2021 14:43:20 -0600 Subject: [PATCH 050/163] test for nil specifically --- lib/datura/elasticsearch/index.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index a19636609..6078da308 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -83,7 +83,7 @@ def get_schema_mapping } schema = get_schema[@options["es_index"]] - if schema == nil || schema == "" + if schema == nil puts "schema is nil!" end doc = schema["mappings"]["_doc"] From 52aba16d0d8e79f3ab1c1d247b53d1905977ec86 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 9 Nov 2021 14:57:58 -0600 Subject: [PATCH 051/163] add debugging statements to get_schema --- lib/datura/elasticsearch/index.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 6078da308..73f017ad7 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -66,6 +66,7 @@ def delete def get_schema RestClient.get(@mapping_url) { |res, req, result| if result.code == "200" + puts "res is #{res}, req is #{req}, result is #{result}" JSON.parse(res) else raise "#{result.code} error getting Elasticsearch schema: #{res}" From 2bbdab1ff52aaa3f614fdc6414662bc5e8425375 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 9 Nov 2021 15:42:00 -0600 Subject: [PATCH 052/163] try debugging with byebug --- lib/datura/elasticsearch/index.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 73f017ad7..8d8cb5184 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -84,9 +84,7 @@ def get_schema_mapping } schema = get_schema[@options["es_index"]] - if schema == nil - puts "schema is nil!" - end + byebug doc = schema["mappings"]["_doc"] puts doc doc["properties"].each do |field, value| From 43580812188ca6428929554ae1005e2099b99c5d Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 9 Nov 2021 16:01:44 -0600 Subject: [PATCH 053/163] remove debugging info --- lib/datura/elasticsearch/index.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 8d8cb5184..f774e7278 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -66,7 +66,6 @@ def delete def get_schema RestClient.get(@mapping_url) { |res, req, result| if result.code == "200" - puts "res is #{res}, req is #{req}, result is #{result}" JSON.parse(res) else raise "#{result.code} error getting Elasticsearch schema: #{res}" @@ -84,9 +83,7 @@ def get_schema_mapping } schema = get_schema[@options["es_index"]] - byebug doc = schema["mappings"]["_doc"] - puts doc doc["properties"].each do |field, value| @schema_mapping["fields"] << field if value["type"] == "nested" From 5ae1fc086ce36d59bfff3fac7cc1e2615928c0e0 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 10 Nov 2021 13:29:50 -0600 Subject: [PATCH 054/163] add alternative field --- lib/datura/to_es/ead_to_es/fields.rb | 3 +++ lib/datura/to_es/ead_to_es_items/fields.rb | 3 +++ 2 files changed, 6 insertions(+) diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb index 44436709a..73e9003f5 100644 --- a/lib/datura/to_es/ead_to_es/fields.rb +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -19,6 +19,9 @@ def abstract get_text(@xpaths["abstract"]) end + def alternative + end + def annotations_text # TODO what should default behavior be? end diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index 60a266445..e7669ac0b 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -15,6 +15,9 @@ def id # "https://cdrhapi.unl.edu/doc/#{@id}" # end + def alternative + end + def annotations_text # TODO what should default behavior be? end From 73089c0f1d3091d1f6f0a03631473c7391b7acc1 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 10 Nov 2021 14:37:29 -0600 Subject: [PATCH 055/163] add relation field --- lib/datura/to_es/ead_to_es/fields.rb | 3 +++ lib/datura/to_es/ead_to_es_items/fields.rb | 3 +++ 2 files changed, 6 insertions(+) diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb index 73e9003f5..ca10636ed 100644 --- a/lib/datura/to_es/ead_to_es/fields.rb +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -161,6 +161,9 @@ def recipient # return people end + def relation + end + def rights # Note: override by collection as needed get_text(@xpaths["rights"]) diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index e7669ac0b..34b613495 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -169,6 +169,9 @@ def recipient # return people end + def relation + end + def rights # Note: override by collection as needed get_text(@xpaths["rights"]) From 428ea4cfff83bf18206a03870a840dbb2dc48775 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 10 Nov 2021 14:51:00 -0600 Subject: [PATCH 056/163] add spatial field --- lib/datura/to_es/ead_to_es/fields.rb | 3 +++ lib/datura/to_es/ead_to_es_items/fields.rb | 3 +++ 2 files changed, 6 insertions(+) diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb index ca10636ed..54d03a359 100644 --- a/lib/datura/to_es/ead_to_es/fields.rb +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -183,6 +183,9 @@ def source get_text(@xpaths["source"]) end + def spatial + end + def subjects get_list(@xpaths["subjects"]) end diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index 34b613495..888a8d982 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -191,6 +191,9 @@ def source get_text(@xpaths["source"]) end + def spatial + end + def subjects # TODO default behavior? end From c5cb629d41633118959b4ce0af5aa2f00e7194aa Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 10 Nov 2021 15:14:05 -0600 Subject: [PATCH 057/163] fix a get_text method --- lib/datura/to_es/ead_to_es_items/fields.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index 888a8d982..6668bdeca 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -207,7 +207,7 @@ def text # handling separate fields in array # means no worrying about handling spacing between words text = [] - body = get_text(@xpaths["text"], false) + body = get_text(@xpaths["text"]) text << body # TODO: do we need to preserve tags like in text? if so, turn get_text to true # text << CommonXml.convert_tags_in_string(body) From 4ab0542f9f137efb7596f15567b4658c9f1edb09 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 10 Nov 2021 17:05:38 -0600 Subject: [PATCH 058/163] change post_es to match jessica's changes --- lib/datura/file_type.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 12a66b25f..30875d335 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -49,6 +49,7 @@ def parse_markup_lang_file CommonXml.create_xml_object(self.file_location) end + # expecting an instance of Datura::Elasticsearch::Index def post_es(es) error = nil begin From 193d69ec9d328ba093a91aec59f5f8449f47fdcc Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 11 Nov 2021 08:49:18 -0600 Subject: [PATCH 059/163] change CommonXML to Datura helpers --- lib/datura/to_es/ead_to_es/fields.rb | 6 +++--- lib/datura/to_es/ead_to_es_items/fields.rb | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb index 54d03a359..9db742050 100644 --- a/lib/datura/to_es/ead_to_es/fields.rb +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -33,7 +33,7 @@ def category def creator creators = get_list(@xpaths["creators"]) if creators - return creators.map { |creator| { "name" => CommonXml.normalize_space(creator) } } + return creators.map { |creator| { "name" => Datura::Helpers.normalize_space(creator) } } end end @@ -71,7 +71,7 @@ def data_type def date(before=true) datestr = get_text(@xpaths["date"]) - return CommonXml.date_standardize(datestr, before) + return Datura::Helpers.date_standardize(datestr, before) end def date_display @@ -225,7 +225,7 @@ def title def title_sort t = title - CommonXml.normalize_name(t) + Datura::Helpers.normalize_name(t) end def type diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index 6668bdeca..731421cf5 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -65,7 +65,7 @@ def data_type def date(before=true) datestr = get_text(@xpaths["date"]) - return CommonXml.date_standardize(datestr, before) + return Datura::Helpers.date_standardize(datestr, before) end def date_display @@ -212,7 +212,7 @@ def text # TODO: do we need to preserve tags like in text? if so, turn get_text to true # text << CommonXml.convert_tags_in_string(body) text += text_additional - return CommonXml.normalize_space(text.join(" ")) + return Datura::Helpers.normalize_space(text.join(" ")) end def text_additional @@ -230,7 +230,7 @@ def title def title_sort t = title - CommonXml.normalize_name(t) + Datura::Helpers.normalize_name(t) end def topics From 344381ccb5fdcb55ba9f5daee2b0324fe5f6d25a Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 11 Nov 2021 11:04:00 -0600 Subject: [PATCH 060/163] change xpaths to be less specific to Walt Whitman --- lib/datura/to_es/ead_to_es/xpaths.rb | 17 +------------- lib/datura/to_es/ead_to_es_items/xpaths.rb | 27 +--------------------- 2 files changed, 2 insertions(+), 42 deletions(-) diff --git a/lib/datura/to_es/ead_to_es/xpaths.rb b/lib/datura/to_es/ead_to_es/xpaths.rb index ddcc4d64f..b33e06c19 100644 --- a/lib/datura/to_es/ead_to_es/xpaths.rb +++ b/lib/datura/to_es/ead_to_es/xpaths.rb @@ -6,36 +6,21 @@ class EadToEs < XmlToEs def xpaths_list { "abstract" => "/ead/archdesc/did/abstract", - # "category" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='category'][1]/term", - # "contributors" => [ - # "/TEI/teiHeader/revisionDesc/change/name", - # "/TEI/teiHeader/fileDesc/titleStmt/editor" - # ], "creators" => ["/ead/archdesc/did/origination/persname", "/ead/eadheader/filedesc/titlestmt/creator"], "date" => "/ead/eadheader/filedesc/publicationstmt/date", "description" => "/ead/archdesc/scopecontent/p", - # a non normalized date taken directly from the source material ("Dec 9", "Winter 1889") - # "date_display" => "/TEI/teiHeader/fileDesc/sourceDesc/bibl/date", "formats" => "/ead/archdesc/did/physdesc/genreform", - # "image_id" => "/TEI/text//pb/@facs", - # "keywords" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='keywords']/term", - # note: language is global attribute xml:lang "identifier" => "/ead/archdesc/did/unitid", "language" => "/ead/eadheader/profiledesc/langusage/language", - # "languages" => "//body/div1/@lang", - # "person" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='people']/term", - # "places" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='places']/term", "publisher" => "/ead/eadheader/filedesc/publicationstmt/publisher", - # "recipient" => "/TEI/teiHeader/profileDesc/particDesc/person[@role='recipient']/persName", "repository_contact" => "/ead/archdesc/did/repository/address/*", "rights" => "/ead/archdesc/descgrp/accessrestrict/p", "rights_holder" => "/ead/archdesc/did/repository/corpname", "source" => "/ead/archdesc/descgrp/prefercite/p", "subjects" => "/ead/archdesc/controlaccess/*[not(name()='head')]", - # "subcategory" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='subcategory'][1]/term", "title" => "/ead/archdesc/did/unittitle", "text" => "/ead/eadheader/filedesc/titlestmt/*", - "items" => "//*[@level='item']/did/unitid[@type='WWA']" + "items" => "//*[@level='item']/did/unitid" }.merge(override_xpaths) end end diff --git a/lib/datura/to_es/ead_to_es_items/xpaths.rb b/lib/datura/to_es/ead_to_es_items/xpaths.rb index b39f2d802..bec9aba4d 100644 --- a/lib/datura/to_es/ead_to_es_items/xpaths.rb +++ b/lib/datura/to_es/ead_to_es_items/xpaths.rb @@ -6,40 +6,15 @@ class EadToEsItems < EadToEs def xpaths_list { "abstract" => "did/abstract", - # "category" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='category'][1]/term", - # "contributors" => [ - # "/TEI/teiHeader/revisionDesc/change/name", - # "/TEI/teiHeader/fileDesc/titleStmt/editor" - # # ], - # "creators" => ["/ead/archdesc/did/origination/persname", "/ead/eadheader/filedesc/titlestmt/creator"], "date" => "did/unitdate", "description" => "scopecontent/p", - # a non normalized date taken directly from the source material ("Dec 9", "Winter 1889") - # "date_display" => "/TEI/teiHeader/fileDesc/sourceDesc/bibl/date", "extent" => "did/physdesc/extent", "format" => "did/physdesc/physfacet", "image_url" => "did/dao/@href", - # "image_id" => "/TEI/text//pb/@facs", - # "keywords" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='keywords']/term", - # note: language is global attribute xml:lang - "identifier" => "did/unitid[@type='WWA']", - # "language" => "(//body/div1/@lang)[1]", - # "languages" => "//body/div1/@lang", - # "person" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='people']/term", - # "places" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='places']/term", - # "publisher" => "/TEI/teiHeader/fileDesc/sourceDesc/bibl[1]/publisher[1]", - # "recipient" => "/TEI/teiHeader/profileDesc/particDesc/person[@role='recipient']/persName", + "identifier" => "did/unitid", "repository_id" => "did/unitid[@type='repository']", - # "rights" => "/ead/archdesc/descgrp/accessrestrict/p", - # "rights_holder" => "/ead/archdesc/descgrp/accessrestrict/p", - # "source" => "/ead/archdesc/descgrp/prefercite/p", - # "subjects" => ["/ead/archdesc/controlaccess/[everything after head;persname, subject]"], - # "subcategory" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='subcategory'][1]/term", - # "text" => "//text", "title" => "did/unittitle", - # "topic" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='topic']/term", "type" => "did/physdesc/genreform", - # "works" => "/TEI/teiHeader/profileDesc/textClass/keywords[@n='works']/term", }.merge(override_xpaths) end end From df7e8966704e2b73cff4b45a32b70b224f9317fd Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 11 Nov 2021 16:36:14 -0600 Subject: [PATCH 061/163] refactor title fields and xpaths --- lib/datura/to_es/ead_to_es/fields.rb | 5 ++--- lib/datura/to_es/ead_to_es_items/fields.rb | 5 ++--- lib/datura/to_es/ead_to_es_items/xpaths.rb | 3 ++- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb index 9db742050..bf4a4e569 100644 --- a/lib/datura/to_es/ead_to_es/fields.rb +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -220,12 +220,11 @@ def text # end def title - get_text(@xpaths["title"]) + get_list(@xpaths["title"]) end def title_sort - t = title - Datura::Helpers.normalize_name(t) + Datura::Helpers.normalize_name(title) end def type diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index 731421cf5..9e0c48d87 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -225,12 +225,11 @@ def text_additional end def title - title = get_text(@xpaths["title"]) + get_text(@xpaths["title"])[0] end def title_sort - t = title - Datura::Helpers.normalize_name(t) + Datura::Helpers.normalize_name(title) end def topics diff --git a/lib/datura/to_es/ead_to_es_items/xpaths.rb b/lib/datura/to_es/ead_to_es_items/xpaths.rb index bec9aba4d..23b941e29 100644 --- a/lib/datura/to_es/ead_to_es_items/xpaths.rb +++ b/lib/datura/to_es/ead_to_es_items/xpaths.rb @@ -7,13 +7,14 @@ def xpaths_list { "abstract" => "did/abstract", "date" => "did/unitdate", + "date_display" => "did/unitdate", "description" => "scopecontent/p", "extent" => "did/physdesc/extent", "format" => "did/physdesc/physfacet", "image_url" => "did/dao/@href", "identifier" => "did/unitid", "repository_id" => "did/unitid[@type='repository']", - "title" => "did/unittitle", + "title" => "did/unittitle/title", "type" => "did/physdesc/genreform", }.merge(override_xpaths) end From bf65724320bf8835979f1071eae19cde6431d608 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 10 Dec 2021 11:21:37 -0600 Subject: [PATCH 062/163] add creator override for items so it is an array --- lib/datura/to_es/ead_to_es_items/fields.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index 9e0c48d87..bf1fc99d6 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -26,10 +26,10 @@ def category end # note this does not sort the creators - # def creator - # creators = get_list(@xpaths["creators"]) - # return creators.map { |creator| { "name" => CommonXml.normalize_space(creator) } } - # end + def creator + creators = get_list(@xpaths["creators"]) + return creators.map { |creator| { "name" => CommonXml.normalize_space(creator) } } + end # returns ; delineated string of alphabetized creators def creator_sort From c3679876ae77cb1f553a04029464f9e78289721e Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 21 Dec 2021 11:18:17 -0600 Subject: [PATCH 063/163] change creators to creator --- lib/datura/to_es/ead_to_es/fields.rb | 2 +- lib/datura/to_es/ead_to_es/xpaths.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb index bf4a4e569..f5599c97a 100644 --- a/lib/datura/to_es/ead_to_es/fields.rb +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -31,7 +31,7 @@ def category # note this does not sort the creators def creator - creators = get_list(@xpaths["creators"]) + creators = get_list(@xpaths["creator"]) if creators return creators.map { |creator| { "name" => Datura::Helpers.normalize_space(creator) } } end diff --git a/lib/datura/to_es/ead_to_es/xpaths.rb b/lib/datura/to_es/ead_to_es/xpaths.rb index b33e06c19..989d5122c 100644 --- a/lib/datura/to_es/ead_to_es/xpaths.rb +++ b/lib/datura/to_es/ead_to_es/xpaths.rb @@ -6,7 +6,7 @@ class EadToEs < XmlToEs def xpaths_list { "abstract" => "/ead/archdesc/did/abstract", - "creators" => ["/ead/archdesc/did/origination/persname", "/ead/eadheader/filedesc/titlestmt/creator"], + "creator" => ["/ead/archdesc/did/origination/persname", "/ead/eadheader/filedesc/titlestmt/creator"], "date" => "/ead/eadheader/filedesc/publicationstmt/date", "description" => "/ead/archdesc/scopecontent/p", "formats" => "/ead/archdesc/did/physdesc/genreform", From 480f95a5bb46fc04251e93525393e328534b25a6 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 25 May 2022 15:48:46 -0500 Subject: [PATCH 064/163] update gems in preparation for release --- Gemfile.lock | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ae964e7c8..82108be99 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,28 +13,26 @@ GEM colorize (0.8.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - http-accept (1.7.0) - http-cookie (1.0.4) + http-cookie (1.0.5) domain_name (~> 0.5) - mime-types (3.3.1) + mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0901) - mini_portile2 (2.6.1) - minitest (5.14.4) + mime-types-data (3.2022.0105) + mini_portile2 (2.8.0) + minitest (5.15.0) netrc (0.11.0) - nokogiri (1.12.5) - mini_portile2 (~> 2.6.1) + nokogiri (1.13.6) + mini_portile2 (~> 2.8.0) racc (~> 1.4) racc (1.6.0) rake (13.0.6) - rest-client (2.1.0) - http-accept (>= 1.7.0, < 2.0) + rest-client (2.0.2) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) unf (0.1.4) unf_ext - unf_ext (0.0.8) + unf_ext (0.0.8.1) PLATFORMS ruby From 51d85b2b933b93fa81e19d252fd7be6711411f6d Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 16 Jun 2022 12:22:31 -0500 Subject: [PATCH 065/163] add rdf schema --- lib/datura/to_es/es_request.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/datura/to_es/es_request.rb b/lib/datura/to_es/es_request.rb index 6aeeef056..923f56f98 100644 --- a/lib/datura/to_es/es_request.rb +++ b/lib/datura/to_es/es_request.rb @@ -30,6 +30,7 @@ def assemble_json assemble_people assemble_spatial assemble_references + assemble_rdf assemble_text assemble_collection_specific @@ -117,10 +118,16 @@ def assemble_spatial @json["spatial"] = spatial end + def assemble_rdf + @json["rdf"] = rdf + end + def assemble_text @json["annotations_text"] = annotations_text @json["text"] = text # @json["abstract"] end + + end From 45cb1a1065e2c7035416897faec1665d3ae71b56 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 20 Jun 2022 11:55:02 -0500 Subject: [PATCH 066/163] update schemas to include rdf fields --- lib/config/es_api_schemas/1.0.yml | 15 +++++++++++++++ lib/config/es_api_schemas/2.0.yml | 21 +++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/lib/config/es_api_schemas/1.0.yml b/lib/config/es_api_schemas/1.0.yml index 9d3a69650..4953b47be 100644 --- a/lib/config/es_api_schemas/1.0.yml +++ b/lib/config/es_api_schemas/1.0.yml @@ -88,6 +88,21 @@ mappings: type: keyword source: type: keyword + rdf: + type: nested + properties: + type: + type: keyword + subject: + type: keyword + predicate: + type: keyword + object: + type: keyword + source: + type: keyword + note: + type: keyword recipient: type: nested properties: diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index 841da72a4..06db363e3 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -150,6 +150,27 @@ mappings: source: type: keyword normalizer: keyword_normalized + rdf: + type: nested + properties: + type: + type: keyword + normalizer: keyword_normalized + subject: + type: keyword + normalizer: keyword_normalized + predicate: + type: keyword + normalizer: keyword_normalized + object: + type: keyword + normalizer: keyword_normalized + source: + type: keyword + normalizer: keyword_normalized + note: + type: keyword + normalizer: keyword_normalized recipient: type: nested properties: From 43c5c41d0e4380cb31c163360b5a6a0fa86a2162 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 20 Jun 2022 12:12:04 -0500 Subject: [PATCH 067/163] add rdf to default fields --- lib/datura/to_es/csv_to_es/fields.rb | 4 ++++ lib/datura/to_es/custom_to_es/fields.rb | 4 ++++ lib/datura/to_es/html_to_es/fields.rb | 4 ++++ lib/datura/to_es/tei_to_es/fields.rb | 4 ++++ lib/datura/to_es/vra_to_es/fields.rb | 4 ++++ lib/datura/to_es/webs_to_es/fields.rb | 4 ++++ 6 files changed, 24 insertions(+) diff --git a/lib/datura/to_es/csv_to_es/fields.rb b/lib/datura/to_es/csv_to_es/fields.rb index 0f4a2be61..1b649eb2a 100644 --- a/lib/datura/to_es/csv_to_es/fields.rb +++ b/lib/datura/to_es/csv_to_es/fields.rb @@ -139,6 +139,10 @@ def recipient end end + # nested field + def rdf + end + def relation @row["relation"] end diff --git a/lib/datura/to_es/custom_to_es/fields.rb b/lib/datura/to_es/custom_to_es/fields.rb index 0818f94e7..7a430d3be 100644 --- a/lib/datura/to_es/custom_to_es/fields.rb +++ b/lib/datura/to_es/custom_to_es/fields.rb @@ -85,6 +85,10 @@ def places def publisher end + # nested field + def rdf + end + # nested field def recipient end diff --git a/lib/datura/to_es/html_to_es/fields.rb b/lib/datura/to_es/html_to_es/fields.rb index c95d452e4..4b70303de 100644 --- a/lib/datura/to_es/html_to_es/fields.rb +++ b/lib/datura/to_es/html_to_es/fields.rb @@ -111,6 +111,10 @@ def publisher get_text(@xpaths["publisher"]) end + # nested field + def rdf + end + # nested field def recipient end diff --git a/lib/datura/to_es/tei_to_es/fields.rb b/lib/datura/to_es/tei_to_es/fields.rb index 861d42d70..f78b044f2 100644 --- a/lib/datura/to_es/tei_to_es/fields.rb +++ b/lib/datura/to_es/tei_to_es/fields.rb @@ -129,6 +129,10 @@ def publisher get_text(@xpaths["publisher"]) end + # nested field + def rdf + end + # nested field def recipient eles = @xml.xpath(@xpaths["recipient"]) diff --git a/lib/datura/to_es/vra_to_es/fields.rb b/lib/datura/to_es/vra_to_es/fields.rb index 75e374e80..233778287 100644 --- a/lib/datura/to_es/vra_to_es/fields.rb +++ b/lib/datura/to_es/vra_to_es/fields.rb @@ -138,6 +138,10 @@ def publisher get_text(@xpaths["publisher"]) end + # nested field + def rdf + end + # nested field def recipient eles = get_elements(@xpaths["recipient"]) diff --git a/lib/datura/to_es/webs_to_es/fields.rb b/lib/datura/to_es/webs_to_es/fields.rb index 815b78aa8..993813f63 100644 --- a/lib/datura/to_es/webs_to_es/fields.rb +++ b/lib/datura/to_es/webs_to_es/fields.rb @@ -111,6 +111,10 @@ def publisher get_text(@xpaths["publisher"]) end + # nested field + def rdf + end + # nested field def recipient end From 6198a7a425da73f55148abe09c0949f8fca133a6 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 21 Jun 2022 14:04:31 -0500 Subject: [PATCH 068/163] add spatial.title field --- lib/config/es_api_schemas/1.0.yml | 3 +++ lib/config/es_api_schemas/2.0.yml | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/lib/config/es_api_schemas/1.0.yml b/lib/config/es_api_schemas/1.0.yml index 4953b47be..da276c956 100644 --- a/lib/config/es_api_schemas/1.0.yml +++ b/lib/config/es_api_schemas/1.0.yml @@ -121,6 +121,9 @@ mappings: spatial: type: nested properties: + # display title for entire location + title: + type: keyword place_name: # TODO copy into text? type: keyword diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index 06db363e3..f67b8d519 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -195,6 +195,10 @@ mappings: spatial: type: nested properties: + # display title for entire location + title: + type: keyword + normalizer: keyword_normalized place_name: # TODO copy into text? type: keyword From c145ad8063cc65b68f2b6cf9c04b9ea88b87792e Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 18 Jul 2022 11:36:22 -0500 Subject: [PATCH 069/163] require byebug so it is in scope for posting etc. --- lib/datura/data_manager.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 96544e045..cdf80d9c3 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -3,7 +3,7 @@ require "yaml" require "byebug" require_relative "./requirer.rb" - +require "byebug" class Datura::DataManager attr_reader :log attr_reader :time @@ -46,6 +46,9 @@ def initialize prepare_xslt load_collection_classes set_up_logger + # set up posting URLs + @es_url = File.join(options["es_path"], options["es_index"]) + @solr_url = File.join(options["solr_path"], options["solr_core"], "update") end # NOTE: This step is what allows collection specific files to override ANY @@ -53,9 +56,13 @@ def initialize def load_collection_classes # load collection scripts at this point so they will override # any of the default ones (for example: TeiToEs) + puts !(defined?(byebug)) + path = File.join(@options["collection_dir"], "scripts", "overrides", "*.rb") Dir[path].each do |f| + puts "requiring" + f require f + puts defined?(byebug) end end @@ -66,6 +73,7 @@ def print_options end def run + byebug @time = [Time.now] # log starting information for user check_options From f089f42570072a4a969f4895ca516f6889df98bd Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 24 Aug 2022 13:39:45 -0500 Subject: [PATCH 070/163] remove inserted byebug --- lib/datura/data_manager.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index cdf80d9c3..7e75815d9 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -73,7 +73,6 @@ def print_options end def run - byebug @time = [Time.now] # log starting information for user check_options From d0bfc36c6aa1215719dd5c4383fee9065be0c0be Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 18 Jul 2022 11:36:22 -0500 Subject: [PATCH 071/163] require byebug so it is in scope for posting etc. --- lib/datura/data_manager.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 7e75815d9..d8c66936e 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -56,13 +56,10 @@ def initialize def load_collection_classes # load collection scripts at this point so they will override # any of the default ones (for example: TeiToEs) - puts !(defined?(byebug)) - path = File.join(@options["collection_dir"], "scripts", "overrides", "*.rb") Dir[path].each do |f| puts "requiring" + f require f - puts defined?(byebug) end end From 0bd3c5802e05a235759ebf495ad41aed6581b09f Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 8 Aug 2022 09:28:44 -0500 Subject: [PATCH 072/163] include full error message with backtrace --- lib/datura/file_type.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 30875d335..da92e7f85 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -55,7 +55,7 @@ def post_es(es) begin transformed = transform_es rescue => e - "Error transforming ES for #{self.filename(false)}: #{e}" + return { "error" => "Error transforming ES for #{self.filename(false)}: #{e.full_message}" } end if transformed && transformed.length > 0 transformed.each do |doc| From cd639b4ea8e3bcc4082aa3f7f6a96cb86abde34e Mon Sep 17 00:00:00 2001 From: Jessica Dussault Date: Thu, 9 Jan 2020 16:05:26 -0600 Subject: [PATCH 073/163] updates gems and fixes test suite had suffered from errors and from gem deprecation warnings --- Gemfile.lock | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 82108be99..2d67803de 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,7 +13,12 @@ GEM colorize (0.8.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) +<<<<<<< HEAD http-cookie (1.0.5) +======= + http-accept (1.7.0) + http-cookie (1.0.3) +>>>>>>> 4d02bee88 (updates gems and fixes test suite) domain_name (~> 0.5) mime-types (3.4.1) mime-types-data (~> 3.2015) @@ -26,8 +31,8 @@ GEM racc (~> 1.4) racc (1.6.0) rake (13.0.6) - rest-client (2.0.2) - http-cookie (>= 1.0.2, < 2.0) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) unf (0.1.4) From b40c22c90b4c45b630e8c9756ce35d957916c2cd Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 25 May 2022 15:48:46 -0500 Subject: [PATCH 074/163] update gems in preparation for release --- Gemfile.lock | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Gemfile.lock b/Gemfile.lock index 2d67803de..a20379abd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,12 +13,16 @@ GEM colorize (0.8.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) +<<<<<<< HEAD <<<<<<< HEAD http-cookie (1.0.5) ======= http-accept (1.7.0) http-cookie (1.0.3) >>>>>>> 4d02bee88 (updates gems and fixes test suite) +======= + http-cookie (1.0.5) +>>>>>>> deb1664b2 (update gems in preparation for release) domain_name (~> 0.5) mime-types (3.4.1) mime-types-data (~> 3.2015) From 7efc8fcaf16b5c57a12411d880ff72d10a92a9ac Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 9 Aug 2022 08:41:04 -0500 Subject: [PATCH 075/163] start adding new api fields --- lib/config/es_api_schemas/2.0.yml | 128 +++++++++++++++++++++++++----- 1 file changed, 106 insertions(+), 22 deletions(-) diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index f67b8d519..688ba5bc9 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -37,39 +37,45 @@ mappings: identifier: type: keyword normalizer: keyword_normalized - collection: + cdrhCollection: type: keyword normalizer: keyword_normalized - collection_desc: + cdrhCollectionDesc: type: keyword normalizer: keyword_normalized - uri: + cdrhUri: type: keyword normalizer: keyword_normalized - uri_data: + cdrhUriData: type: keyword normalizer: keyword_normalized - uri_html: + cdrhUriHtml: type: keyword normalizer: keyword_normalized - data_type: + cdrhDataType: type: keyword normalizer: keyword_normalized - image_location: + cdrhFigLocation: type: keyword normalizer: keyword_normalized - image_id: + # image_location: TODO how is this field handled? is it same as above? + # type: keyword + # normalizer: keyword_normalized + cdrhCoverImage: type: keyword normalizer: keyword_normalized # TODO copy to text? - title: + dcTitle: type: keyword normalizer: keyword_normalized title_sort: type: keyword normalizer: keyword_normalized # TODO copy to text? - alternative: + dctermsAlternative: + type: keyword + normalizer: keyword_normalized + cdrhDateUpdated: type: keyword normalizer: keyword_normalized creator_sort: @@ -85,15 +91,15 @@ mappings: id: type: keyword normalizer: keyword_normalized - subjects: + dctermsSubjects: type: keyword normalizer: keyword_normalized # TODO not sure yet if for display or search - abstract: + dctermsAbstract: type: keyword normalizer: keyword_normalized # TODO copy to text? - description: + dctermsDescription: type: keyword normalizer: keyword_normalized publisher: @@ -111,7 +117,7 @@ mappings: role: type: keyword normalizer: keyword_normalized - date: + date_sort: type: date format: "yyyy-MM-dd||epoch_millis" # ignore_malformed: true @@ -144,10 +150,10 @@ mappings: languages: type: keyword normalizer: keyword_normalized - relation: + dctermsRelation: type: keyword normalizer: keyword_normalized - source: + dctermsSource: type: keyword normalizer: keyword_normalized rdf: @@ -227,6 +233,21 @@ mappings: type: keyword normalizer: keyword_normalized postal_code: + type: keyword + normalizer: keyword_normalized + township: + type: keyword + normalizer: keyword_normalized + note: + type: keyword + normalizer: keyword_normalized + role: + type: keyword + normalizer: keyword_normalized + description: + type: keyword + normalizer: keyword_normalized + type: type: keyword normalizer: keyword_normalized person: @@ -245,27 +266,90 @@ mappings: annotations_text: type: text analyzer: english - category: + cdrhCategory1: type: keyword normalizer: keyword_normalized - subcategory: + cdrhCategory2: type: keyword normalizer: keyword_normalized - topics: + cdrhCategory3: type: keyword normalizer: keyword_normalized - keywords: + cdrhCategory4: type: keyword normalizer: keyword_normalized - people: + cdrhCategory5: type: keyword normalizer: keyword_normalized - places: + cdrhNotes: type: keyword normalizer: keyword_normalized + cdrhTopics: + type: keyword + normalizer: keyword_normalized + cdrhKeywords1: + type: keyword + normalizer: keyword_normalized + cdrhKeywords2: + type: keyword + normalizer: keyword_normalized + cdrhKeywords3: + type: keyword + normalizer: keyword_normalized + cdrhKeywords4: + type: keyword + normalizer: keyword_normalized + people: + type: keyword + normalizer: keyword_normalized + # places: #DEPRECATED + # type: keyword + # normalizer: keyword_normalized works: type: keyword normalizer: keyword_normalized + citation: + type: nested + properties: + role: + type: keyword + normalizer: keyword_normalized + date: + type: date + format: "yyyy-MM-dd||epoch_millis" + publisher: + type: keyword + normalizer: keyword_normalized + issue: + type: keyword + normalizer: keyword_normalized + page_begin: + type: keyword + normalizer: keyword_normalized + page_end: + type: keyword + normalizer: keyword_normalized + section: + type: keyword + normalizer: keyword_normalized + volume: + type: keyword + normalizer: keyword_normalized + place: + type: keyword + normalizer: keyword_normalized + title_a: + type: keyword + normalizer: keyword_normalized + title_m: + type: keyword + normalizer: keyword_normalized + title_j: + type: keyword + normalizer: keyword_normalized + date_created: + type: date + format: "yyyy-MM-dd||epoch_millis" text: type: text analyzer: english From 685a56333b8a31181f8b7106daafd1dc040a5d8a Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 11 Aug 2022 14:21:35 -0500 Subject: [PATCH 076/163] update schema to match spreadsheet with new field names --- lib/config/es_api_schemas/2.0.yml | 360 +++++++++++++++++------------- 1 file changed, 209 insertions(+), 151 deletions(-) diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index 688ba5bc9..e744f7bdc 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -37,72 +37,60 @@ mappings: identifier: type: keyword normalizer: keyword_normalized - cdrhCollection: + collection: type: keyword normalizer: keyword_normalized - cdrhCollectionDesc: + collection_desc: type: keyword normalizer: keyword_normalized - cdrhUri: + uri: type: keyword normalizer: keyword_normalized - cdrhUriData: + uri_data: type: keyword normalizer: keyword_normalized - cdrhUriHtml: + uri_html: type: keyword normalizer: keyword_normalized - cdrhDataType: + data_type: type: keyword normalizer: keyword_normalized - cdrhFigLocation: + fig_location: type: keyword normalizer: keyword_normalized - # image_location: TODO how is this field handled? is it same as above? - # type: keyword - # normalizer: keyword_normalized - cdrhCoverImage: + cover_image: type: keyword normalizer: keyword_normalized - # TODO copy to text? - dcTitle: + title: + # TODO copy to text type: keyword normalizer: keyword_normalized title_sort: type: keyword normalizer: keyword_normalized - # TODO copy to text? - dctermsAlternative: + alternative: + # TODO copy to text type: keyword normalizer: keyword_normalized - cdrhDateUpdated: + date_updated: + type: date + format: "yyyy-MM-dd||epoch_millis" + category1: type: keyword normalizer: keyword_normalized - creator_sort: + category2: type: keyword normalizer: keyword_normalized - creator: - type: nested - properties: - name: - # TODO copy into text? - type: keyword - normalizer: keyword_normalized - id: - type: keyword - normalizer: keyword_normalized - dctermsSubjects: + category3: type: keyword normalizer: keyword_normalized - # TODO not sure yet if for display or search - dctermsAbstract: + category4: type: keyword normalizer: keyword_normalized - # TODO copy to text? - dctermsDescription: + category5: type: keyword normalizer: keyword_normalized - publisher: + notes: type: keyword normalizer: keyword_normalized contributor: @@ -117,7 +105,58 @@ mappings: role: type: keyword normalizer: keyword_normalized - date_sort: + creator: + type: nested + properties: + name: + type: keyword + normalizer: keyword_normalized + id: + type: keyword + normalizer: keyword_normalized + citation: + type: nested + properties: + role: + type: keyword + normalizer: keyword_normalized + id: + type: keyword + normalizer: keyword_normalized + date: + type: date + format: "yyyy-MM-dd||epoch_millis" + publisher: + type: keyword + normalizer: keyword_normalized + issue: + type: keyword + normalizer: keyword_normalized + page_begin: + type: keyword + normalizer: keyword_normalized + page_end: + type: keyword + normalizer: keyword_normalized + section: + type: keyword + normalizer: keyword_normalized + volume: + type: keyword + normalizer: keyword_normalized + place: + type: keyword + normalizer: keyword_normalized + title_a: + type: keyword + normalizer: keyword_normalized + title_m: + type: keyword + normalizer: keyword_normalized + title_j: + type: keyword + normalizer: keyword_normalized + date: type: date format: "yyyy-MM-dd||epoch_millis" # ignore_malformed: true @@ -132,9 +171,6 @@ mappings: type: date format: "yyyy-MM-dd||epoch_millis" # ignore_malformed: true - type: - type: keyword - normalizer: keyword_normalized format: type: keyword normalizer: keyword_normalized @@ -147,37 +183,124 @@ mappings: language: type: keyword normalizer: keyword_normalized - languages: + rights_holder: + type: keyword + normalizer: keyword_normalized + rights: type: keyword normalizer: keyword_normalized - dctermsRelation: + rights_uri: type: keyword normalizer: keyword_normalized - dctermsSource: + subjects: type: keyword normalizer: keyword_normalized - rdf: + # TODO not sure yet if for display or search + abstract: + type: keyword + normalizer: keyword_normalized + # TODO copy to text? + description: + type: text + analyzer: english + type: + type: keyword + normalizer: keyword_normalized + topics: + type: keyword + normalizer: keyword_normalized + keywords1: + type: keyword + normalizer: keyword_normalized + keywords2: + type: keyword + normalizer: keyword_normalized + keywords3: + type: keyword + normalizer: keyword_normalized + keywords4: + type: keyword + normalizer: keyword_normalized + relation: + type: keyword + normalizer: keyword_normalized + source: + type: keyword + normalizer: keyword_normalized + has_part: + type: keyword + normalizer: keyword_normalized + is_part_of: + type: keyword + normalizer: keyword_normalized + # recipient: #DELETED + # type: nested + # properties: + # name: + # type: keyword + # normalizer: keyword_normalized + # id: + # type: keyword + # normalizer: keyword_normalized + # role: + # type: keyword + # normalizer: keyword_normalized + spatial: type: nested properties: + role: + type: keyword + normalizer: keyword_normalized + name: + # display title for entire location + type: keyword + normalizer: keyword_normalized + description: + type: text + analyzer: english type: type: keyword normalizer: keyword_normalized - subject: + short_name: + # TODO copy into text? type: keyword normalizer: keyword_normalized - predicate: + coordinates: + type: geo_point + id: type: keyword normalizer: keyword_normalized - object: + city: type: keyword normalizer: keyword_normalized - source: + township: + type: keyword + normalizer: keyword_normalized + county: + type: keyword + normalizer: keyword_normalized + country: + type: keyword + normalizer: keyword_normalized + region: + type: keyword + normalizer: keyword_normalized + state: + type: keyword + normalizer: keyword_normalized + street: type: keyword normalizer: keyword_normalized + postal_code: + type: keyword + normalizer: keyword_normalized note: - type: keyword + type: keyword normalizer: keyword_normalized - recipient: + places: #DEPRECATED + type: keyword + normalizer: keyword_normalized + person: type: nested properties: name: @@ -189,167 +312,102 @@ mappings: role: type: keyword normalizer: keyword_normalized - rights_holder: - type: keyword - normalizer: keyword_normalized - rights: - type: keyword - normalizer: keyword_normalized - rights_uri: - type: keyword - normalizer: keyword_normalized - spatial: - type: nested - properties: - # display title for entire location - title: + note: type: keyword normalizer: keyword_normalized - place_name: - # TODO copy into text? + order: type: keyword normalizer: keyword_normalized - coordinates: - type: geo_point - id: + birth_date: type: keyword normalizer: keyword_normalized - city: + death_date: type: keyword normalizer: keyword_normalized - county: + age_category: type: keyword normalizer: keyword_normalized - country: + name_last: type: keyword normalizer: keyword_normalized - region: + name_given: type: keyword normalizer: keyword_normalized - state: + name_alternate: type: keyword normalizer: keyword_normalized - street: + race: type: keyword normalizer: keyword_normalized - postal_code: - type: keyword - normalizer: keyword_normalized - township: + sex: type: keyword - normalizer: keyword_normalized - note: + normalizer: keyword_normalized + gender: type: keyword normalizer: keyword_normalized - role: + nationality: type: keyword normalizer: keyword_normalized - description: + trait1: type: keyword normalizer: keyword_normalized - type: + trait2: type: keyword normalizer: keyword_normalized - person: + event: type: nested properties: - name: - # TODO copy into text? + type: type: keyword normalizer: keyword_normalized - id: + agent: type: keyword normalizer: keyword_normalized - role: + factor: type: keyword normalizer: keyword_normalized - annotations_text: - type: text - analyzer: english - cdrhCategory1: - type: keyword - normalizer: keyword_normalized - cdrhCategory2: - type: keyword - normalizer: keyword_normalized - cdrhCategory3: - type: keyword - normalizer: keyword_normalized - cdrhCategory4: - type: keyword - normalizer: keyword_normalized - cdrhCategory5: - type: keyword - normalizer: keyword_normalized - cdrhNotes: - type: keyword - normalizer: keyword_normalized - cdrhTopics: - type: keyword - normalizer: keyword_normalized - cdrhKeywords1: - type: keyword - normalizer: keyword_normalized - cdrhKeywords2: - type: keyword - normalizer: keyword_normalized - cdrhKeywords3: - type: keyword - normalizer: keyword_normalized - cdrhKeywords4: - type: keyword - normalizer: keyword_normalized - people: - type: keyword - normalizer: keyword_normalized - # places: #DEPRECATED - # type: keyword - # normalizer: keyword_normalized - works: - type: keyword - normalizer: keyword_normalized - citation: - type: nested - properties: - role: + product: type: keyword normalizer: keyword_normalized - date: - type: date - format: "yyyy-MM-dd||epoch_millis" - publisher: + date_begin: type: keyword normalizer: keyword_normalized - issue: + date_end: type: keyword normalizer: keyword_normalized - page_begin: + trait1: type: keyword normalizer: keyword_normalized - page_end: + trait2: type: keyword normalizer: keyword_normalized - section: + notes: type: keyword normalizer: keyword_normalized - volume: + rdf: + type: nested + properties: + type: type: keyword normalizer: keyword_normalized - place: + subject: type: keyword normalizer: keyword_normalized - title_a: + predicate: type: keyword normalizer: keyword_normalized - title_m: + object: type: keyword normalizer: keyword_normalized - title_j: + source: type: keyword normalizer: keyword_normalized - date_created: - type: date - format: "yyyy-MM-dd||epoch_millis" + note: + type: keyword + normalizer: keyword_normalized + annotations_text: + type: text + analyzer: english text: type: text analyzer: english From 5b52ab1acbe8e36a0b23a2ed96277b0cb9e0b720 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 15 Aug 2022 15:29:25 -0500 Subject: [PATCH 077/163] assemble json based on api version --- lib/datura/to_es/es_request.rb | 154 +++++++++++++++++++++++++-------- 1 file changed, 120 insertions(+), 34 deletions(-) diff --git a/lib/datura/to_es/es_request.rb b/lib/datura/to_es/es_request.rb index 923f56f98..3b3ebe71e 100644 --- a/lib/datura/to_es/es_request.rb +++ b/lib/datura/to_es/es_request.rb @@ -19,29 +19,53 @@ def assemble_json # below not alphabetical to reflect their position # in the cdrh api schema - - assemble_identifiers - assemble_categories - assemble_locations - assemble_descriptions - assemble_other_metadata - assemble_dates - assemble_publishing - assemble_people - assemble_spatial - assemble_references - assemble_rdf - assemble_text + if @options["api_version"] == "1.0" + assemble_json_1 + elsif @options["api_version"] == "2.0" + assemble_json_2 + end assemble_collection_specific - + assemble_text @json end + def assemble_json_1 + #fields for API v 1.0 + assemble_identifiers_1 + assemble_categories_1 + assemble_locations_1 + assemble_descriptions_1 + assemble_other_metadata_1 + assemble_dates_1 + assemble_publishing_1 + assemble_people_1 + assemble_spatial_1 + assemble_references_1 + assemble_rdf_1 + assemble_text_1 + end + + def assemble_json_2 + #field for API v 2.0 + assemble_identifiers_2 + assemble_metadata_digital_2 + assemble_metadata_original_2 + assemble_metadata_interpretive_2 + assemble_relations_2 + assemble_additional_2 + assemble_text_2 + end + ############## # components # ############## + def assemble_collection_specific + # add your own per collection + # with format + # @json["fieldname"] = field_contents + end - def assemble_categories + def assemble_categories_1 @json["category"] = category @json["subcategory"] = subcategory @json["data_type"] = data_type @@ -50,20 +74,14 @@ def assemble_categories @json["subjects"] = subjects end - def assemble_collection_specific - # add your own per collection - # with format - # @json["fieldname"] = field_contents - end - - def assemble_dates + def assemble_dates_1 @json["date"] = date @json["date_not_after"] = date_not_after @json["date_not_before"] = date_not_before @json["date_display"] = date_display end - def assemble_descriptions + def assemble_descriptions_1 @json["alternative"] = alternative @json["description"] = description @json["title"] = title @@ -71,18 +89,18 @@ def assemble_descriptions @json["topics"] = topics end - def assemble_identifiers + def assemble_identifiers_1 @json["identifier"] = @id end - def assemble_locations + def assemble_locations_1 @json["uri"] = uri @json["uri_data"] = uri_data @json["uri_html"] = uri_html @json["image_id"] = image_id end - def assemble_other_metadata + def assemble_other_metadata_1 @json["format"] = format @json["language"] = language @json["languages"] = languages @@ -92,7 +110,7 @@ def assemble_other_metadata @json["medium"] = medium end - def assemble_people + def assemble_people_1 # container fields @json["person"] = person @json["contributor"] = contributor @@ -100,7 +118,7 @@ def assemble_people @json["recipient"] = recipient end - def assemble_publishing + def assemble_publishing_1 @json["publisher"] = publisher @json["rights"] = rights @json["rights_uri"] = rights_uri @@ -108,26 +126,94 @@ def assemble_publishing @json["source"] = source end - def assemble_references + def assemble_references_1 @json["keywords"] = keywords @json["places"] = places @json["works"] = works end - def assemble_spatial + def assemble_spatial_1 @json["spatial"] = spatial end - def assemble_rdf + def assemble_rdf_1 + @json["rdf"] = rdf + end + + def assemble_identifiers_2 + @json["identifier"] = @id # does this still work? + @json["collection"] = collection + @json["collection_desc"] = collection_desc + @json["uri"] = uri + @json["uri_data"] = uri_data + @json["uri_html"] = uri_html + @json["data_type"] = data_type + @json["fig_location"] = fig_location + @json["cover_image"] = cover_image + @json["title"] = title + @json["title_sort"] = title_sort + @json["alternative"] = alternative + @json["date_updated"] = date_updated + @json["category"] = category + @json["category2"] = category2 + @json["category3"] = category3 + @json["category4"] = category4 + @json["category5"] = category5 + @json["notes"] = notes + end + + def assemble_metadata_digital_2 + @json["contributor"] = contributor + end + + def assemble_metadata_original_2 + @json["creator"] = creator + @json["citation"] = citation + @json["date"] = date + @json["date_display"] = date_display + @json["date_not_before"] = date_not_before + @json["date_not_after"] = date_not_after + @json["format"] = format + @json["medium"] = medium + @json["extent"] = extent + @json["language"] = language + @json["rights_holder"] = rights_holder + @json["rights"] = rights + @json["rights_uri"] = rights_uri + end + + def assemble_metadata_interpretive_2 + @json["subjects"] = subjects + @json["abstract"] = abstract + @json["description"] = description + @json["type"] = type + @json["topics"] = topics + @json["keywords"] = keywords + @json["keywords2"] = keywords2 + @json["keywords3"] = keywords3 + @json["keywords4"] = keywords4 + end + + def assemble_relations_2 + @json["relation"] = relation + @json["source"] = source + @json["has_part"] = has_part + @json["is_part_of"] = is_part_of + @json["previous"] = previous + @json["next"] = next + end + + def assemble_additional_2 + @json["spatial"] = spatial + @json["places"] = places + @json["person"] = person + @json["event"] = event @json["rdf"] = rdf end def assemble_text @json["annotations_text"] = annotations_text @json["text"] = text - # @json["abstract"] end - - end From 3af4d4724ef5efc06deb6ef9495405821be69d25 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 15 Aug 2022 15:39:17 -0500 Subject: [PATCH 078/163] add overrides for 2.0 fields --- lib/datura/to_es/csv_to_es/fields.rb | 61 ++++++++++++++++++++-- lib/datura/to_es/html_to_es/fields.rb | 74 +++++++++++++++++++++++++-- lib/datura/to_es/tei_to_es/fields.rb | 70 +++++++++++++++++++++++++ lib/datura/to_es/vra_to_es/fields.rb | 70 +++++++++++++++++++++++++ lib/datura/to_es/webs_to_es/fields.rb | 71 +++++++++++++++++++++++++ 5 files changed, 338 insertions(+), 8 deletions(-) diff --git a/lib/datura/to_es/csv_to_es/fields.rb b/lib/datura/to_es/csv_to_es/fields.rb index 1b649eb2a..2f669fa86 100644 --- a/lib/datura/to_es/csv_to_es/fields.rb +++ b/lib/datura/to_es/csv_to_es/fields.rb @@ -5,6 +5,8 @@ class CsvToEs ########## # FIELDS # ########## + # beginning with fields from API 1.0, including those that are unchanged in 2.0 + def id @id end @@ -139,10 +141,6 @@ def recipient end end - # nested field - def rdf - end - def relation @row["relation"] end @@ -245,4 +243,59 @@ def works end end + # new/moved fields for API 2.0 + + def cover_image + @row["image_id"] + end + + def date_updated + end + + def category2 + @row["subcategory"] + end + + def category3 + end + + def category4 + end + + def category5 + end + + def notes + end + + def citation + end + + def keywords2 + end + + def keywords3 + end + + def keywords4 + end + + def has_part + end + + def is_part_of + end + + def previous_item + end + + def next_item + end + + def event + end + + def rdf + end + end diff --git a/lib/datura/to_es/html_to_es/fields.rb b/lib/datura/to_es/html_to_es/fields.rb index 4b70303de..f1b062824 100644 --- a/lib/datura/to_es/html_to_es/fields.rb +++ b/lib/datura/to_es/html_to_es/fields.rb @@ -111,10 +111,6 @@ def publisher get_text(@xpaths["publisher"]) end - # nested field - def rdf - end - # nested field def recipient end @@ -222,4 +218,74 @@ def works get_list(@xpaths["works"]) end + # new/moved fields for API 2.0 + + def cover_image + get_list(@xpaths["image_id"]).first + end + + def date_updated + get_list(@xpaths["date_updated"]) + end + + def category2 + get_list(@xpaths["subcategory"]) + end + + def category3 + get_text(@xpaths["category3"]) + end + + def category4 + get_text(@xpaths["category4"]) + end + + def category5 + get_text(@xpaths["category5"]) + end + + def notes + get_text(@xpaths["notes"]) + end + + def citation + # nested + end + + def keywords2 + get_text(@xpaths["keywords2"]) + end + + def keywords3 + get_text(@xpaths["keywords3"]) + end + + def keywords4 + get_text(@xpaths["keywords4"]) + end + + def has_part + # nested + end + + def is_part_of + # nested + end + + def previous_item + # nested + end + + def next_item + # nested + end + + def event + # nested + end + + def rdf + # nested + end + end diff --git a/lib/datura/to_es/tei_to_es/fields.rb b/lib/datura/to_es/tei_to_es/fields.rb index f78b044f2..bd351c004 100644 --- a/lib/datura/to_es/tei_to_es/fields.rb +++ b/lib/datura/to_es/tei_to_es/fields.rb @@ -255,6 +255,76 @@ def works get_list(@xpaths["works"]) end + # new/moved fields for API 2.0 + + def cover_image + get_list(@xpaths["image_id"]).first + end + + def date_updated + get_list(@xpaths["date_updated"]) + end + + def category2 + get_list(@xpaths["subcategory"]) + end + + def category3 + get_text(@xpaths["category3"]) + end + + def category4 + get_text(@xpaths["category4"]) + end + + def category5 + get_text(@xpaths["category5"]) + end + + def notes + get_text(@xpaths["notes"]) + end + + def citation + # nested + end + + def keywords2 + get_text(@xpaths["keywords2"]) + end + + def keywords3 + get_text(@xpaths["keywords3"]) + end + + def keywords4 + get_text(@xpaths["keywords4"]) + end + + def has_part + # nested + end + + def is_part_of + # nested + end + + def previous_item + # nested + end + + def next_item + # nested + end + + def event + # nested + end + + def rdf + # nested + end + protected # default behavior is simply to comma delineate fields diff --git a/lib/datura/to_es/vra_to_es/fields.rb b/lib/datura/to_es/vra_to_es/fields.rb index 233778287..c2cb61c8a 100644 --- a/lib/datura/to_es/vra_to_es/fields.rb +++ b/lib/datura/to_es/vra_to_es/fields.rb @@ -255,4 +255,74 @@ def uri_html def works get_list(@xpaths["works"]) end + + # new/moved fields for API 2.0 + + def cover_image + get_list(@xpaths["image_id"]).first + end + + def date_updated + get_list(@xpaths["date_updated"]) + end + + def category2 + get_list(@xpaths["subcategory"]) + end + + def category3 + get_text(@xpaths["category3"]) + end + + def category4 + get_text(@xpaths["category4"]) + end + + def category5 + get_text(@xpaths["category5"]) + end + + def notes + get_text(@xpaths["notes"]) + end + + def citation + # nested + end + + def keywords2 + get_text(@xpaths["keywords2"]) + end + + def keywords3 + get_text(@xpaths["keywords3"]) + end + + def keywords4 + get_text(@xpaths["keywords4"]) + end + + def has_part + # nested + end + + def is_part_of + # nested + end + + def previous_item + # nested + end + + def next_item + # nested + end + + def event + # nested + end + + def rdf + # nested + end end diff --git a/lib/datura/to_es/webs_to_es/fields.rb b/lib/datura/to_es/webs_to_es/fields.rb index 993813f63..5997b90d9 100644 --- a/lib/datura/to_es/webs_to_es/fields.rb +++ b/lib/datura/to_es/webs_to_es/fields.rb @@ -214,4 +214,75 @@ def uri_html def works get_list(@xpaths["works"]) end + + # new/moved fields for API 2.0 + + def cover_image + get_list(@xpaths["image_id"]).first + end + + def date_updated + get_list(@xpaths["date_updated"]) + end + + def category2 + get_list(@xpaths["subcategory"]) + end + + def category3 + get_text(@xpaths["category3"]) + end + + def category4 + get_text(@xpaths["category4"]) + end + + def category5 + get_text(@xpaths["category5"]) + end + + def notes + get_text(@xpaths["notes"]) + end + + def citation + # nested + end + + def keywords2 + get_text(@xpaths["keywords2"]) + end + + def keywords3 + get_text(@xpaths["keywords3"]) + end + + def keywords4 + get_text(@xpaths["keywords4"]) + end + + def has_part + # nested + end + + def is_part_of + # nested + end + + def previous_item + # nested + end + + def next_item + # nested + end + + def event + # nested + end + + def rdf + # nested + end + end From 8a3634a0d866551f8f4946841c564c73a6a5bfc1 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 15 Aug 2022 16:09:26 -0500 Subject: [PATCH 079/163] change next and previous fields --- lib/datura/to_es/es_request.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/datura/to_es/es_request.rb b/lib/datura/to_es/es_request.rb index 3b3ebe71e..ff28f6cba 100644 --- a/lib/datura/to_es/es_request.rb +++ b/lib/datura/to_es/es_request.rb @@ -139,7 +139,7 @@ def assemble_spatial_1 def assemble_rdf_1 @json["rdf"] = rdf end - + def assemble_identifiers_2 @json["identifier"] = @id # does this still work? @json["collection"] = collection @@ -199,8 +199,8 @@ def assemble_relations_2 @json["source"] = source @json["has_part"] = has_part @json["is_part_of"] = is_part_of - @json["previous"] = previous - @json["next"] = next + @json["previous_item"] = previous_item + @json["next_item"] = next_item end def assemble_additional_2 From 8e5f888bf30c2d3558b6b3464859d4f211f8e7c3 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 15 Aug 2022 16:13:49 -0500 Subject: [PATCH 080/163] add fig_location --- lib/datura/to_es/csv_to_es/fields.rb | 3 +++ lib/datura/to_es/html_to_es/fields.rb | 4 ++++ lib/datura/to_es/tei_to_es/fields.rb | 4 ++++ lib/datura/to_es/vra_to_es/fields.rb | 4 ++++ lib/datura/to_es/webs_to_es/fields.rb | 4 ++++ 5 files changed, 19 insertions(+) diff --git a/lib/datura/to_es/csv_to_es/fields.rb b/lib/datura/to_es/csv_to_es/fields.rb index 2f669fa86..b639954d8 100644 --- a/lib/datura/to_es/csv_to_es/fields.rb +++ b/lib/datura/to_es/csv_to_es/fields.rb @@ -252,6 +252,9 @@ def cover_image def date_updated end + def fig_location + end + def category2 @row["subcategory"] end diff --git a/lib/datura/to_es/html_to_es/fields.rb b/lib/datura/to_es/html_to_es/fields.rb index f1b062824..cb74c7ad3 100644 --- a/lib/datura/to_es/html_to_es/fields.rb +++ b/lib/datura/to_es/html_to_es/fields.rb @@ -228,6 +228,10 @@ def date_updated get_list(@xpaths["date_updated"]) end + def fig_location + get_list(@xpaths["fig_location"]) + end + def category2 get_list(@xpaths["subcategory"]) end diff --git a/lib/datura/to_es/tei_to_es/fields.rb b/lib/datura/to_es/tei_to_es/fields.rb index bd351c004..2cc23db01 100644 --- a/lib/datura/to_es/tei_to_es/fields.rb +++ b/lib/datura/to_es/tei_to_es/fields.rb @@ -265,6 +265,10 @@ def date_updated get_list(@xpaths["date_updated"]) end + def fig_location + get_list(@xpaths["fig_location"]) + end + def category2 get_list(@xpaths["subcategory"]) end diff --git a/lib/datura/to_es/vra_to_es/fields.rb b/lib/datura/to_es/vra_to_es/fields.rb index c2cb61c8a..8d3e5e65b 100644 --- a/lib/datura/to_es/vra_to_es/fields.rb +++ b/lib/datura/to_es/vra_to_es/fields.rb @@ -266,6 +266,10 @@ def date_updated get_list(@xpaths["date_updated"]) end + def fig_location + get_list(@xpaths["fig_location"]) + end + def category2 get_list(@xpaths["subcategory"]) end diff --git a/lib/datura/to_es/webs_to_es/fields.rb b/lib/datura/to_es/webs_to_es/fields.rb index 5997b90d9..4fb7a42b6 100644 --- a/lib/datura/to_es/webs_to_es/fields.rb +++ b/lib/datura/to_es/webs_to_es/fields.rb @@ -225,6 +225,10 @@ def date_updated get_list(@xpaths["date_updated"]) end + def fig_location + get_list(@xpaths["fig_location"]) + end + def category2 get_list(@xpaths["subcategory"]) end From 6397179e44df0f5569fce5557e8cd08ee2de2120 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 15 Aug 2022 16:20:11 -0500 Subject: [PATCH 081/163] add abstract --- lib/datura/to_es/csv_to_es/fields.rb | 3 +++ lib/datura/to_es/html_to_es/fields.rb | 4 ++++ lib/datura/to_es/tei_to_es/fields.rb | 4 ++++ lib/datura/to_es/vra_to_es/fields.rb | 4 ++++ lib/datura/to_es/webs_to_es/fields.rb | 4 ++++ 5 files changed, 19 insertions(+) diff --git a/lib/datura/to_es/csv_to_es/fields.rb b/lib/datura/to_es/csv_to_es/fields.rb index b639954d8..1a626f2ee 100644 --- a/lib/datura/to_es/csv_to_es/fields.rb +++ b/lib/datura/to_es/csv_to_es/fields.rb @@ -274,6 +274,9 @@ def notes def citation end + def abstract + end + def keywords2 end diff --git a/lib/datura/to_es/html_to_es/fields.rb b/lib/datura/to_es/html_to_es/fields.rb index cb74c7ad3..3998a6400 100644 --- a/lib/datura/to_es/html_to_es/fields.rb +++ b/lib/datura/to_es/html_to_es/fields.rb @@ -256,6 +256,10 @@ def citation # nested end + def abstract + get_text(@xpaths["abstract"]) + end + def keywords2 get_text(@xpaths["keywords2"]) end diff --git a/lib/datura/to_es/tei_to_es/fields.rb b/lib/datura/to_es/tei_to_es/fields.rb index 2cc23db01..2fe1cdeb9 100644 --- a/lib/datura/to_es/tei_to_es/fields.rb +++ b/lib/datura/to_es/tei_to_es/fields.rb @@ -293,6 +293,10 @@ def citation # nested end + def abstract + get_text(@xpaths["abstract"]) + end + def keywords2 get_text(@xpaths["keywords2"]) end diff --git a/lib/datura/to_es/vra_to_es/fields.rb b/lib/datura/to_es/vra_to_es/fields.rb index 8d3e5e65b..824ae685a 100644 --- a/lib/datura/to_es/vra_to_es/fields.rb +++ b/lib/datura/to_es/vra_to_es/fields.rb @@ -294,6 +294,10 @@ def citation # nested end + def abstract + get_text(@xpaths["abstract"]) + end + def keywords2 get_text(@xpaths["keywords2"]) end diff --git a/lib/datura/to_es/webs_to_es/fields.rb b/lib/datura/to_es/webs_to_es/fields.rb index 4fb7a42b6..44a49af96 100644 --- a/lib/datura/to_es/webs_to_es/fields.rb +++ b/lib/datura/to_es/webs_to_es/fields.rb @@ -253,6 +253,10 @@ def citation # nested end + def abstract + get_text(@xpaths["abstract"]) + end + def keywords2 get_text(@xpaths["keywords2"]) end From 2f80693dcdd0f157c2f3979b51af43352bf1730f Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 15 Aug 2022 16:36:02 -0500 Subject: [PATCH 082/163] remove split-out assemble_text methods --- lib/datura/to_es/es_request.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/datura/to_es/es_request.rb b/lib/datura/to_es/es_request.rb index ff28f6cba..245b31a88 100644 --- a/lib/datura/to_es/es_request.rb +++ b/lib/datura/to_es/es_request.rb @@ -42,7 +42,6 @@ def assemble_json_1 assemble_spatial_1 assemble_references_1 assemble_rdf_1 - assemble_text_1 end def assemble_json_2 @@ -53,7 +52,6 @@ def assemble_json_2 assemble_metadata_interpretive_2 assemble_relations_2 assemble_additional_2 - assemble_text_2 end ############## From df573e1972cbc0c50a15ee735bf4496d730892b8 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 24 Aug 2022 15:42:56 -0500 Subject: [PATCH 083/163] update gems and get rid of merge conflicts --- Gemfile.lock | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a20379abd..d5512b234 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,38 +13,29 @@ GEM colorize (0.8.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) -<<<<<<< HEAD -<<<<<<< HEAD - http-cookie (1.0.5) -======= http-accept (1.7.0) - http-cookie (1.0.3) ->>>>>>> 4d02bee88 (updates gems and fixes test suite) -======= http-cookie (1.0.5) ->>>>>>> deb1664b2 (update gems in preparation for release) domain_name (~> 0.5) mime-types (3.4.1) mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) - mini_portile2 (2.8.0) - minitest (5.15.0) + minitest (5.16.3) netrc (0.11.0) - nokogiri (1.13.6) - mini_portile2 (~> 2.8.0) + nokogiri (1.13.8-x86_64-darwin) racc (~> 1.4) racc (1.6.0) rake (13.0.6) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) unf (0.1.4) unf_ext - unf_ext (0.0.8.1) + unf_ext (0.0.8.2) PLATFORMS - ruby + x86_64-darwin-20 DEPENDENCIES bundler (>= 1.16.0, < 3.0) @@ -54,4 +45,4 @@ DEPENDENCIES rake (~> 13.0) BUNDLED WITH - 2.1.4 + 2.2.26 From ee079aecf7fd7656d4faac333b05935bb4765526 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 24 Aug 2022 15:56:30 -0500 Subject: [PATCH 084/163] add new fields --- lib/config/es_api_schemas/2.0.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index e744f7bdc..bc56e4ac3 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -192,6 +192,12 @@ mappings: rights_uri: type: keyword normalizer: keyword_normalized + container_box: + type: keyword + normalizer: keyword_normalized + container_field: + type: keyword + normalizer: keyword_normalized subjects: type: keyword normalizer: keyword_normalized @@ -336,6 +342,9 @@ mappings: name_alternate: type: keyword normalizer: keyword_normalized + name_previous: + type: keyword + normalizer: keyword_normalized race: type: keyword normalizer: keyword_normalized From c268f8cd282c8284a9760880da62f0b35b422511 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 24 Aug 2022 16:15:51 -0500 Subject: [PATCH 085/163] correct field name --- lib/config/es_api_schemas/2.0.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index bc56e4ac3..cdff6ffc7 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -195,7 +195,7 @@ mappings: container_box: type: keyword normalizer: keyword_normalized - container_field: + container_folder: type: keyword normalizer: keyword_normalized subjects: From 0eb14bca78b0a27b3e4e8e7421bbda6df42b343d Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 24 Aug 2022 16:16:38 -0500 Subject: [PATCH 086/163] add fields to ead overrides --- lib/datura/to_es/ead_to_es/fields.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb index f5599c97a..15cb7c9dc 100644 --- a/lib/datura/to_es/ead_to_es/fields.rb +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -50,6 +50,12 @@ def collection_desc @options["collection_desc"] || @options["collection"] end + def container_box + end + + def container_folder + end + def contributor # contribs = [] # @xpaths["contributors"].each do |xpath| From ccdede7ce5b0078c6418d21c71c108428a08c418 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 24 Aug 2022 16:23:39 -0500 Subject: [PATCH 087/163] populate new fields in json --- lib/datura/to_es/es_request.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/datura/to_es/es_request.rb b/lib/datura/to_es/es_request.rb index 245b31a88..6b7f42382 100644 --- a/lib/datura/to_es/es_request.rb +++ b/lib/datura/to_es/es_request.rb @@ -178,6 +178,8 @@ def assemble_metadata_original_2 @json["rights_holder"] = rights_holder @json["rights"] = rights @json["rights_uri"] = rights_uri + @json["container_box"] = container_box + @json["container_folder"] = container_folder end def assemble_metadata_interpretive_2 From dfa420ce64b6cf45a66843b03d9504aa2e7cd6b6 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 24 Aug 2022 16:32:35 -0500 Subject: [PATCH 088/163] resolve merge conflict --- lib/datura/elasticsearch/alias.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/datura/elasticsearch/alias.rb b/lib/datura/elasticsearch/alias.rb index adf1c5cf1..177ee14d1 100644 --- a/lib/datura/elasticsearch/alias.rb +++ b/lib/datura/elasticsearch/alias.rb @@ -6,11 +6,7 @@ module Datura::Elasticsearch::Alias def self.add -<<<<<<< HEAD - params = Datura::Parser.es_alias -======= params = Datura::Parser.es_alias_add ->>>>>>> 01ed9e56d (moves code out of bin elasticsearch files and into module) options = Datura::Options.new(params).all ali = options["alias"] From e85681213e81b9516e864b5fa1d9116eb8112bd4 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 25 Aug 2022 08:59:07 -0500 Subject: [PATCH 089/163] add new fields --- lib/datura/to_es/csv_to_es/fields.rb | 6 ++++++ lib/datura/to_es/html_to_es/fields.rb | 6 ++++++ lib/datura/to_es/tei_to_es/fields.rb | 6 ++++++ lib/datura/to_es/vra_to_es/fields.rb | 6 ++++++ lib/datura/to_es/webs_to_es/fields.rb | 6 ++++++ 5 files changed, 30 insertions(+) diff --git a/lib/datura/to_es/csv_to_es/fields.rb b/lib/datura/to_es/csv_to_es/fields.rb index 1a626f2ee..50c7efc81 100644 --- a/lib/datura/to_es/csv_to_es/fields.rb +++ b/lib/datura/to_es/csv_to_es/fields.rb @@ -39,6 +39,12 @@ def collection def collection_desc @options["collection_desc"] || @options["collection"] end + + def container_box + end + + def container_folder + end # nested field def contributor diff --git a/lib/datura/to_es/html_to_es/fields.rb b/lib/datura/to_es/html_to_es/fields.rb index 3998a6400..886c1bd7d 100644 --- a/lib/datura/to_es/html_to_es/fields.rb +++ b/lib/datura/to_es/html_to_es/fields.rb @@ -256,6 +256,12 @@ def citation # nested end + def container_box + end + + def container_folder + end + def abstract get_text(@xpaths["abstract"]) end diff --git a/lib/datura/to_es/tei_to_es/fields.rb b/lib/datura/to_es/tei_to_es/fields.rb index 2fe1cdeb9..4826635a3 100644 --- a/lib/datura/to_es/tei_to_es/fields.rb +++ b/lib/datura/to_es/tei_to_es/fields.rb @@ -293,6 +293,12 @@ def citation # nested end + def container_box + end + + def container_folder + end + def abstract get_text(@xpaths["abstract"]) end diff --git a/lib/datura/to_es/vra_to_es/fields.rb b/lib/datura/to_es/vra_to_es/fields.rb index 824ae685a..c65dda8db 100644 --- a/lib/datura/to_es/vra_to_es/fields.rb +++ b/lib/datura/to_es/vra_to_es/fields.rb @@ -294,6 +294,12 @@ def citation # nested end + def container_box + end + + def container_folder + end + def abstract get_text(@xpaths["abstract"]) end diff --git a/lib/datura/to_es/webs_to_es/fields.rb b/lib/datura/to_es/webs_to_es/fields.rb index 44a49af96..3163706be 100644 --- a/lib/datura/to_es/webs_to_es/fields.rb +++ b/lib/datura/to_es/webs_to_es/fields.rb @@ -245,6 +245,12 @@ def category5 get_text(@xpaths["category5"]) end + def container_box + end + + def container_folder + end + def notes get_text(@xpaths["notes"]) end From b252e6624a0d90f6702e08bc16de093048b82124 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 25 Aug 2022 13:33:03 -0500 Subject: [PATCH 090/163] update fields for related items, dates, order integers --- lib/config/es_api_schemas/2.0.yml | 79 +++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 15 deletions(-) diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index cdff6ffc7..d8ce637a3 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -75,7 +75,7 @@ mappings: date_updated: type: date format: "yyyy-MM-dd||epoch_millis" - category1: + category: type: keyword normalizer: keyword_normalized category2: @@ -234,11 +234,61 @@ mappings: type: keyword normalizer: keyword_normalized has_part: - type: keyword - normalizer: keyword_normalized + type: nested + properties: + role: keyword + type: keyword + normalizer: keyword_normalized + id: keyword + type: keyword + normalizer: keyword_normalized + title: keyword + type: keyword + normalizer: keyword_normalized + order: + type: integer is_part_of: - type: keyword - normalizer: keyword_normalized + type: nested + properties: + role: keyword + type: keyword + normalizer: keyword_normalized + id: keyword + type: keyword + normalizer: keyword_normalized + title: keyword + type: keyword + normalizer: keyword_normalized + order: + type: integer + previous_item: + type: nested + properties: + role: keyword + type: keyword + normalizer: keyword_normalized + id: keyword + type: keyword + normalizer: keyword_normalized + title: keyword + type: keyword + normalizer: keyword_normalized + order: + type: integer + next_item: + type: nested + properties: + role: keyword + type: keyword + normalizer: keyword_normalized + id: keyword + type: keyword + normalizer: keyword_normalized + title: keyword + type: keyword + normalizer: keyword_normalized + order: + type: integer # recipient: #DELETED # type: nested # properties: @@ -322,14 +372,13 @@ mappings: type: keyword normalizer: keyword_normalized order: - type: keyword - normalizer: keyword_normalized + type: integer birth_date: - type: keyword - normalizer: keyword_normalized + type: date + format: "yyyy-MM-dd||epoch_millis" death_date: - type: keyword - normalizer: keyword_normalized + type: date + format: "yyyy-MM-dd||epoch_millis" age_category: type: keyword normalizer: keyword_normalized @@ -379,11 +428,11 @@ mappings: type: keyword normalizer: keyword_normalized date_begin: - type: keyword - normalizer: keyword_normalized + type: date + format: "yyyy-MM-dd||epoch_millis" date_end: - type: keyword - normalizer: keyword_normalized + type: date + format: "yyyy-MM-dd||epoch_millis" trait1: type: keyword normalizer: keyword_normalized From 215da79a79b6f6e38b864d87de55c75c84280b56 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 25 Aug 2022 14:03:09 -0500 Subject: [PATCH 091/163] correct syntax errors --- lib/config/es_api_schemas/2.0.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index d8ce637a3..93a804979 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -236,13 +236,13 @@ mappings: has_part: type: nested properties: - role: keyword + role: type: keyword normalizer: keyword_normalized - id: keyword + id: type: keyword normalizer: keyword_normalized - title: keyword + title: type: keyword normalizer: keyword_normalized order: @@ -250,13 +250,13 @@ mappings: is_part_of: type: nested properties: - role: keyword + role: type: keyword normalizer: keyword_normalized - id: keyword + id: type: keyword normalizer: keyword_normalized - title: keyword + title: type: keyword normalizer: keyword_normalized order: @@ -264,10 +264,10 @@ mappings: previous_item: type: nested properties: - role: keyword + role: type: keyword normalizer: keyword_normalized - id: keyword + id: type: keyword normalizer: keyword_normalized title: keyword @@ -278,13 +278,13 @@ mappings: next_item: type: nested properties: - role: keyword + role: type: keyword normalizer: keyword_normalized - id: keyword + id: type: keyword normalizer: keyword_normalized - title: keyword + title: type: keyword normalizer: keyword_normalized order: From db217d9a5e4b48d1255dde95b59b6e9ee1468d4c Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 25 Aug 2022 14:06:45 -0500 Subject: [PATCH 092/163] correct another syntax error --- lib/config/es_api_schemas/2.0.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index 93a804979..95b9ebf16 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -270,7 +270,7 @@ mappings: id: type: keyword normalizer: keyword_normalized - title: keyword + title: type: keyword normalizer: keyword_normalized order: From 8065e3f7505895fe1011ba96407c95af584aae97 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 25 Aug 2022 15:51:20 -0500 Subject: [PATCH 093/163] change keywords1 to plain keywords --- lib/config/es_api_schemas/2.0.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index 95b9ebf16..766dda287 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -215,7 +215,7 @@ mappings: topics: type: keyword normalizer: keyword_normalized - keywords1: + keywords: type: keyword normalizer: keyword_normalized keywords2: From 05b556107a0290c000767ef89590646a338dacf1 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 30 Aug 2022 13:37:07 -0500 Subject: [PATCH 094/163] add more specific message to es validation --- lib/datura/elasticsearch/index.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index f774e7278..cb8da9b84 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -136,7 +136,7 @@ def valid_document?(doc) if nested.keys.all? { |k| valid_field?(k, field) } next else - # if one of the nested hashes fails, it + # if one of the nested hashes fails, it is invalid return false end end @@ -144,6 +144,7 @@ def valid_document?(doc) # all nested fields passed, so it is valid true else + puts "Field '#{field}' is invalid" false end end From fe9fb2b70a8ee161451dcc2d19f374edda9ae615 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 7 Sep 2022 11:33:50 -0500 Subject: [PATCH 095/163] remove extra byebug require --- lib/datura/data_manager.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index d8c66936e..0b0ebae72 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -3,7 +3,7 @@ require "yaml" require "byebug" require_relative "./requirer.rb" -require "byebug" + class Datura::DataManager attr_reader :log attr_reader :time @@ -78,7 +78,6 @@ def run msg = options_msg @log.info(msg) puts msg - pre_file_preparation @files = prepare_files pre_batch_processing From 59013d2f10b4dc40044d1be03fe4db2ed39a3395 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 7 Sep 2022 11:50:07 -0500 Subject: [PATCH 096/163] remove byebug, change error message --- lib/datura/file_type.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index da92e7f85..c702457ce 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -70,7 +70,6 @@ def post_es(es) begin RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, {:content_type => :json } ) rescue => e - # byebug error = "Error transforming or posting to ES for #{self.filename(false)}: #{e.response}" end else @@ -127,7 +126,7 @@ def transform_es # check if any xpaths hit before continuing results = file_xml.xpath(*subdoc_xpaths.keys) if results.length == 0 - raise "No possible xpaths found fo file #{self.filename}, check if XML is valid or customize 'subdoc_xpaths' method" + raise "No possible xpaths found for file #{self.filename}, check if XML is valid or customize 'subdoc_xpaths' method" end subdoc_xpaths.each do |xpath, classname| From 3889306169019b1af4861d77466b211bed5f138c Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 7 Sep 2022 13:00:27 -0500 Subject: [PATCH 097/163] update schema under citations --- lib/config/es_api_schemas/2.0.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index 766dda287..f400161b2 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -126,13 +126,16 @@ mappings: date: type: date format: "yyyy-MM-dd||epoch_millis" + title: + type: keyword + normalizer: keyword_normalized publisher: type: keyword normalizer: keyword_normalized issue: type: keyword normalizer: keyword_normalized - page_begin: + page_start: type: keyword normalizer: keyword_normalized page_end: From f0c19d832d1fffab0c759542fedf3e81982f37ec Mon Sep 17 00:00:00 2001 From: wkdewey Date: Fri, 16 Sep 2022 14:32:35 -0500 Subject: [PATCH 098/163] require fileutils to avoid errors in setup --- bin/setup | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/setup b/bin/setup index 1a25e109c..45258adf1 100755 --- a/bin/setup +++ b/bin/setup @@ -1,6 +1,7 @@ #!/usr/bin/env ruby require "colorize" +require 'fileutils' coll = Dir.getwd From c0b734da26390677a5efe92ebe52aba15bc2ddfb Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 20 Sep 2022 10:55:06 -0500 Subject: [PATCH 099/163] skip title_sort if title is nil --- lib/datura/file_type.rb | 1 - lib/datura/to_es/tei_to_es/fields.rb | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index c702457ce..0526a7daf 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -128,7 +128,6 @@ def transform_es if results.length == 0 raise "No possible xpaths found for file #{self.filename}, check if XML is valid or customize 'subdoc_xpaths' method" end - subdoc_xpaths.each do |xpath, classname| subdocs = file_xml.xpath(xpath) subdocs.each do |subdoc| diff --git a/lib/datura/to_es/tei_to_es/fields.rb b/lib/datura/to_es/tei_to_es/fields.rb index 4826635a3..0e1287428 100644 --- a/lib/datura/to_es/tei_to_es/fields.rb +++ b/lib/datura/to_es/tei_to_es/fields.rb @@ -213,7 +213,9 @@ def title end def title_sort - Datura::Helpers.normalize_name(title) + if title + Datura::Helpers.normalize_name(title) + end end def topics From 1ec95599a2b174e2d26cca333b19ad8c0a73c517 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 23 Sep 2022 12:01:22 -0500 Subject: [PATCH 100/163] return nil instead of empty string, addresses https://github.com/whitmanarchive/whitman-issues/issues/157 --- lib/datura/to_es/xml_to_es.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/datura/to_es/xml_to_es.rb b/lib/datura/to_es/xml_to_es.rb index 8cf50029a..9c0072af7 100644 --- a/lib/datura/to_es/xml_to_es.rb +++ b/lib/datura/to_es/xml_to_es.rb @@ -92,6 +92,9 @@ def get_elements(*xpaths, xml: nil) def get_list(xpaths, keep_tags: false, xml: nil, sort: false) xpath_array = Array(xpaths) list = get_xpaths(xpath_array, keep_tags: keep_tags, xml: xml) + if !list || list.empty? + return nil + end sort ? list.sort : list end @@ -103,6 +106,9 @@ def get_list(xpaths, keep_tags: false, xml: nil, sort: false) def get_text(xpaths, keep_tags: false, xml: nil, delimiter: ";", sort: false) # ensure all xpaths are an array before beginning list = get_list(xpaths, keep_tags: keep_tags, xml: xml, sort: sort) + if !list || list.empty? + return nil + end list.join("#{delimiter} ") end From 3eddf9af86beebf4216de55f70513f84d495d82d Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 26 Sep 2022 09:58:00 -0500 Subject: [PATCH 101/163] add more nil checks for results of xpath methods --- lib/datura/to_es/tei_to_es/fields.rb | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/lib/datura/to_es/tei_to_es/fields.rb b/lib/datura/to_es/tei_to_es/fields.rb index 0e1287428..7ef94adb6 100644 --- a/lib/datura/to_es/tei_to_es/fields.rb +++ b/lib/datura/to_es/tei_to_es/fields.rb @@ -49,8 +49,14 @@ def data_type end def date(before=true) - datestr = get_list(@xpaths["date"]).first - Datura::Helpers.date_standardize(datestr, before) + if get_list(@xpaths["date"]) + datestr = get_list(@xpaths["date"]).first + else + datestr = nil + end + if datestr && !datestr.empty? + Datura::Helpers.date_standardize(datestr, false) + end end def date_display @@ -84,12 +90,16 @@ def extent end def format - get_list(@xpaths["format"]).first + if get_list(@xpaths["image_id"]) + get_list(@xpaths["format"]).first + end end def image_id # Note: don't pull full path because will be pulled by IIIF - get_list(@xpaths["image_id"]).first + if get_list(@xpaths["image_id"]) + get_list(@xpaths["image_id"]).first + end end def keywords @@ -98,7 +108,9 @@ def keywords def language # uses the first language discovered in the document - get_list(@xpaths["language"]).first + if get_list(@xpaths["image_id"]) + get_list(@xpaths["language"]).first + end end def languages @@ -260,7 +272,9 @@ def works # new/moved fields for API 2.0 def cover_image - get_list(@xpaths["image_id"]).first + if get_list(@xpaths["image_id"]) + get_list(@xpaths["image_id"]).first + end end def date_updated @@ -357,5 +371,6 @@ def source_to_s(f) .reject! { |value| value.nil? || value.strip.empty? } .join(", ") end + end From a1413baa81848b470f868a7c6e5e76efa48175bc Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 26 Sep 2022 11:43:37 -0500 Subject: [PATCH 102/163] check the correct xpath fields --- lib/datura/to_es/tei_to_es/fields.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/to_es/tei_to_es/fields.rb b/lib/datura/to_es/tei_to_es/fields.rb index 7ef94adb6..100fff580 100644 --- a/lib/datura/to_es/tei_to_es/fields.rb +++ b/lib/datura/to_es/tei_to_es/fields.rb @@ -90,7 +90,7 @@ def extent end def format - if get_list(@xpaths["image_id"]) + if get_list(@xpaths["format"]) get_list(@xpaths["format"]).first end end @@ -108,7 +108,7 @@ def keywords def language # uses the first language discovered in the document - if get_list(@xpaths["image_id"]) + if get_list(@xpaths["language"]) get_list(@xpaths["language"]).first end end From b71d028c5d410380fd82b9905a1683d77b6f7c6f Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 26 Sep 2022 14:30:08 -0500 Subject: [PATCH 103/163] make sure input is in UTF-8 --- lib/datura/helpers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 831d148c3..ae2dc89f3 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -134,7 +134,7 @@ def self.normalize_name(abnormal) # removes leading / trailing whitespace, newlines, repeating whitespace, etc def self.normalize_space(abnormal) if abnormal - normal = abnormal.strip.gsub(/\s+/, " ") + normal = abnormal.encode!('UTF-8', 'UTF-8', :invalid => :replace).strip.gsub(/\s+/, " ") end normal || abnormal end From 520bbaa43e8c26057dd82e43f01e6b703013e145 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 26 Sep 2022 14:30:57 -0500 Subject: [PATCH 104/163] make changes for new api schema and revised xpath methods --- lib/datura/to_es/ead_to_es/fields.rb | 89 ++++++++++++++++++++++ lib/datura/to_es/ead_to_es_items/fields.rb | 8 +- 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb index 15cb7c9dc..5128f111e 100644 --- a/lib/datura/to_es/ead_to_es/fields.rb +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -92,6 +92,9 @@ def date_not_before date(true) end + def date_updated + end + def description get_text(@xpaths["description"]) end @@ -261,4 +264,90 @@ def uri_html def works # TODO need to create a list of items, maybe an array of ids end + + # new/moved fields for API 2.0 + + def cover_image + if get_list(@xpaths["image_id"]) + get_list(@xpaths["image_id"]).first + end + end + + def date_updated + get_list(@xpaths["date_updated"]) + end + + def fig_location + get_list(@xpaths["fig_location"]) + end + + def category2 + get_list(@xpaths["subcategory"]) + end + + def category3 + get_text(@xpaths["category3"]) + end + + def category4 + get_text(@xpaths["category4"]) + end + + def category5 + get_text(@xpaths["category5"]) + end + + def notes + get_text(@xpaths["notes"]) + end + + def citation + # nested + end + + def container_box + end + + def container_folder + end + + def abstract + get_text(@xpaths["abstract"]) + end + + def keywords2 + get_text(@xpaths["keywords2"]) + end + + def keywords3 + get_text(@xpaths["keywords3"]) + end + + def keywords4 + get_text(@xpaths["keywords4"]) + end + + def has_part + # nested + end + + def is_part_of + # nested + end + + def previous_item + # nested + end + + def next_item + # nested + end + + def event + # nested + end + + def rdf + # nested + end end diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index bf1fc99d6..6eaf9c718 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -28,7 +28,9 @@ def category # note this does not sort the creators def creator creators = get_list(@xpaths["creators"]) - return creators.map { |creator| { "name" => CommonXml.normalize_space(creator) } } + if creators + return creators.map { |creator| { "name" => CommonXml.normalize_space(creator) } } + end end # returns ; delineated string of alphabetized creators @@ -65,7 +67,9 @@ def data_type def date(before=true) datestr = get_text(@xpaths["date"]) - return Datura::Helpers.date_standardize(datestr, before) + if datestr + return Datura::Helpers.date_standardize(datestr, before) + end end def date_display From dd30d2f986efadd2e997b06705d538dc3bd5de2d Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 26 Sep 2022 14:31:30 -0500 Subject: [PATCH 105/163] add a nil check for creators --- lib/datura/to_es/tei_to_es/fields.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/datura/to_es/tei_to_es/fields.rb b/lib/datura/to_es/tei_to_es/fields.rb index 100fff580..757ea6f21 100644 --- a/lib/datura/to_es/tei_to_es/fields.rb +++ b/lib/datura/to_es/tei_to_es/fields.rb @@ -21,7 +21,9 @@ def category # nested field def creator creators = get_list(@xpaths["creator"]) - creators.map { |c| { "name" => Datura::Helpers.normalize_space(c) } } + if creators + creators.map { |c| { "name" => Datura::Helpers.normalize_space(c) } } + end end def collection From 0ad2053088274253a23f7ab81836e6e91b80932a Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 3 Oct 2022 09:53:28 -0500 Subject: [PATCH 106/163] Revert "make sure input is in UTF-8" This reverts commit b71d028c5d410380fd82b9905a1683d77b6f7c6f. --- lib/datura/helpers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index ae2dc89f3..831d148c3 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -134,7 +134,7 @@ def self.normalize_name(abnormal) # removes leading / trailing whitespace, newlines, repeating whitespace, etc def self.normalize_space(abnormal) if abnormal - normal = abnormal.encode!('UTF-8', 'UTF-8', :invalid => :replace).strip.gsub(/\s+/, " ") + normal = abnormal.strip.gsub(/\s+/, " ") end normal || abnormal end From dc727d696fba3164ee9ca2525758f42a1b4b01a6 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 17 Oct 2022 14:45:52 -0500 Subject: [PATCH 107/163] change error handling to avoid method that isn't present --- lib/datura/file_type.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 0526a7daf..d57a5b62a 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -70,7 +70,7 @@ def post_es(es) begin RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, {:content_type => :json } ) rescue => e - error = "Error transforming or posting to ES for #{self.filename(false)}: #{e.response}" + error = "Error transforming or posting to ES for #{self.filename(false)}: #{e}" end else error = "Document #{id} did not validate against the elasticsearch schema" From 2df4dde29e198eccab8fb67126ee1a872533d4f7 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 20 Oct 2022 14:55:27 -0500 Subject: [PATCH 108/163] make sure person is an array --- lib/datura/to_es/es_request.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/to_es/es_request.rb b/lib/datura/to_es/es_request.rb index 6b7f42382..693a37e07 100644 --- a/lib/datura/to_es/es_request.rb +++ b/lib/datura/to_es/es_request.rb @@ -206,7 +206,7 @@ def assemble_relations_2 def assemble_additional_2 @json["spatial"] = spatial @json["places"] = places - @json["person"] = person + @json["person"] = Array(person) @json["event"] = event @json["rdf"] = rdf end From 436deed9d0b1283fe9241890b83160a9895d0503 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 21 Oct 2022 10:58:19 -0500 Subject: [PATCH 109/163] make sure settings hash is what elasticsearch expects --- lib/config/es_api_schemas/2.0.yml | 65 ++++++++++++++++--------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index f400161b2..d46c63ca0 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -1,37 +1,38 @@ # compatible with Apium v2.0 settings: - analysis: - char_filter: - escapes: - type: mapping - mappings: - - " => " - - " => " - - " => " - - " => " - - " => " - - " => " - - "- => " - - "& => " - - ": => " - - "; => " - - ", => " - - ". => " - - "$ => " - - "@ => " - - "~ => " - - "\" => " - - "' => " - - "[ => " - - "] => " - normalizer: - keyword_normalized: - type: custom - char_filter: - - escapes - filter: - - asciifolding - - lowercase + settings: + analysis: + char_filter: + escapes: + type: mapping + mappings: + - " => " + - " => " + - " => " + - " => " + - " => " + - " => " + - "- => " + - "& => " + - ": => " + - "; => " + - ", => " + - ". => " + - "$ => " + - "@ => " + - "~ => " + - "\" => " + - "' => " + - "[ => " + - "] => " + normalizer: + keyword_normalized: + type: custom + char_filter: + - escapes + filter: + - asciifolding + - lowercase mappings: properties: identifier: From 8a35aa24e0b8ae7042bd18c0c19b4ba7c8059bb6 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 21 Oct 2022 11:16:25 -0500 Subject: [PATCH 110/163] change where mappings are posted for es upgrade --- lib/datura/elasticsearch/index.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index cb8da9b84..b90c4d0a0 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -21,7 +21,7 @@ def initialize(options = nil, schema_mapping: false) @index_url = File.join(@options["es_path"], @options["es_index"]) @pretty_url = "#{@index_url}?pretty=true" - @mapping_url = File.join(@index_url, "_mapping", "_doc?pretty=true") + @mapping_url = File.join(@index_url, "_mapping?pretty=true") # yaml settings (if exist) and mappings @requested_schema = YAML.load_file(@options["es_schema"]) @@ -33,7 +33,6 @@ def initialize(options = nil, schema_mapping: false) def create json = @requested_schema["settings"].to_json puts "Creating ES index for API version #{@options["api_version"]}: #{@pretty_url}" - if json && json != "null" RestClient.put(@pretty_url, json, { content_type: :json }) { |res, req, result| if result.code == "200" @@ -77,13 +76,13 @@ def get_schema_mapping # if mapping has not already been set, get the schema and manipulate if !defined?(@schema_mapping) @schema_mapping = { - "dyanmic" => nil, # /regex|regex/ + "dynamic" => nil, # /regex|regex/ "fields" => [], # [ fields ] "nested" => {} # { field: [ nested_fields ] } } schema = get_schema[@options["es_index"]] - doc = schema["mappings"]["_doc"] + doc = schema["mappings"] doc["properties"].each do |field, value| @schema_mapping["fields"] << field if value["type"] == "nested" From 2c7fa5c5a02d6b2e31310ae189a2090547f6e342 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 25 Oct 2022 12:41:36 -0500 Subject: [PATCH 111/163] add headers to ES requests for authorization --- lib/datura/elasticsearch/alias.rb | 6 +++--- lib/datura/elasticsearch/data.rb | 4 ++-- lib/datura/elasticsearch/index.rb | 18 +++++++++++------- lib/datura/file_type.rb | 5 ++++- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/lib/datura/elasticsearch/alias.rb b/lib/datura/elasticsearch/alias.rb index 177ee14d1..4d6a3a118 100644 --- a/lib/datura/elasticsearch/alias.rb +++ b/lib/datura/elasticsearch/alias.rb @@ -20,7 +20,7 @@ def self.add { add: { alias: ali, index: idx } } ] } - RestClient.post(base_url, data.to_json, { content_type: :json }) { |res, req, result| + RestClient.post(base_url, data.to_json, @auth_header.merge({ content_type: :json })) { |res, req, result| if result.code == "200" puts res puts "Successfully added alias #{ali}. Current alias list:" @@ -40,7 +40,7 @@ def self.delete url = File.join(options["es_path"], idx, "_alias", ali) - res = JSON.parse(RestClient.delete(url)) + res = JSON.parse(RestClient.delete(url, @auth_header)) puts JSON.pretty_generate(res) list end @@ -48,7 +48,7 @@ def self.delete def self.list options = Datura::Options.new({}).all - res = RestClient.get(File.join(options["es_path"], "_aliases")) + res = RestClient.get(File.join(options["es_path"], "_aliases"), ) JSON.pretty_generate(JSON.parse(res)) end diff --git a/lib/datura/elasticsearch/data.rb b/lib/datura/elasticsearch/data.rb index 5deedadb1..4af171fce 100644 --- a/lib/datura/elasticsearch/data.rb +++ b/lib/datura/elasticsearch/data.rb @@ -47,7 +47,7 @@ def self.clear_all(options) if confirm == "Yes I'm sure" url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") json = { "query" => { "match_all" => {} } } - RestClient.post(url, json.to_json, { content_type: :json }) { |res, req, result| + RestClient.post(url, json.to_json, @auth_header.merge({ content_type: :json })) { |res, req, result| if result.code == "200" puts res else @@ -66,7 +66,7 @@ def self.clear_index(options) if confirmation data = self.build_clear_data(options) - RestClient.post(url, data.to_json, { content_type: :json }) { |res, req, result| + RestClient.post(url, data.to_json, @auth_header.merge({ content_type: :json })) { |res, req, result| if result.code == "200" puts res else diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index b90c4d0a0..337f5cf04 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -1,6 +1,7 @@ require "json" require "rest-client" require "yaml" +require "base64" require_relative "./../elasticsearch.rb" @@ -25,6 +26,7 @@ def initialize(options = nil, schema_mapping: false) # yaml settings (if exist) and mappings @requested_schema = YAML.load_file(@options["es_schema"]) + @auth_header = Datura::Helpers.construct_auth_header(@options) # if requested, grab the mapping currently associated with this index # otherwise wait until after the requested schema is loaded get_schema_mapping if schema_mapping @@ -34,7 +36,7 @@ def create json = @requested_schema["settings"].to_json puts "Creating ES index for API version #{@options["api_version"]}: #{@pretty_url}" if json && json != "null" - RestClient.put(@pretty_url, json, { content_type: :json }) { |res, req, result| + RestClient.put(@pretty_url, json, @auth_header.merge({ content_type: :json })) { |res, req, result| if result.code == "200" puts res else @@ -42,7 +44,7 @@ def create end } else - RestClient.put(@pretty_url, nil) { |res, req, result| + RestClient.put(@pretty_url, nil, @auth_header) { |res, req, result| if result.code == "200" puts res else @@ -55,7 +57,7 @@ def create def delete puts "Deleting #{@options["es_index"]} via url #{@pretty_url}" - RestClient.delete(@pretty_url) { |res, req, result| + RestClient.delete(@pretty_url, @auth_header) { |res, req, result| if result.code != "200" raise "#{result.code} error deleting Elasticsearch index: #{res}" end @@ -63,7 +65,7 @@ def delete end def get_schema - RestClient.get(@mapping_url) { |res, req, result| + RestClient.get(@mapping_url, @auth_header) { |res, req, result| if result.code == "200" JSON.parse(res) else @@ -110,7 +112,7 @@ def set_schema json = @requested_schema["mappings"].to_json puts "Setting schema: #{@mapping_url}" - RestClient.put(@mapping_url, json, { content_type: :json }) { |res, req, result| + RestClient.put(@mapping_url, json, @auth_header.merge({ content_type: :json })) { |res, req, result| if result.code == "200" puts res else @@ -206,8 +208,9 @@ def self.clear_all(options) confirm = STDIN.gets.chomp if confirm == "Yes I'm sure" url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") + auth_header = Datura::Helpers.construct_auth_header(options) json = { "query" => { "match_all" => {} } } - RestClient.post(url, json.to_json, { content_type: :json }) { |res, req, result| + RestClient.post(url, json.to_json, auth_header.merge({ content_type: :json })) { |res, req, result| if result.code == "200" puts res else @@ -226,7 +229,8 @@ def self.clear_index(options) if confirmation data = self.build_clear_data(options) - RestClient.post(url, data.to_json, { content_type: :json }) { |res, req, result| + auth_header = Datura::Helpers.construct_auth_header(options) + RestClient.post(url, data.to_json, auth_header.merge({content_type: :json })) { |res, req, result| if result.code == "200" puts res else diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index d57a5b62a..aeb0f6f46 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -30,6 +30,7 @@ def initialize(location, options) @out_html = File.join(output, "html") @out_iiif = File.join(output, "iiif") @out_solr = File.join(output, "solr") + @auth_header = Datura::Helpers.construct_auth_header(options) Datura::Helpers.make_dirs(@out_es, @out_html, @out_iiif, @out_solr) # script locations set in child classes end @@ -68,7 +69,9 @@ def post_es(es) # NOTE: If you need to do partial updates rather than replacement of doc # you will need to add _update at the end of this URL begin - RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, {:content_type => :json } ) + puts @auth_header + byebug + RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, @auth_header.merge({:content_type => :json }) ) rescue => e error = "Error transforming or posting to ES for #{self.filename(false)}: #{e}" end From 25f0a2bc1e140cfb64980c181d36866d5e10f0a8 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 25 Oct 2022 12:42:20 -0500 Subject: [PATCH 112/163] add method to construct basic auth header from options --- lib/datura/helpers.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 831d148c3..6e64557e2 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -171,4 +171,10 @@ def self.should_update?(file, since_date=nil) end end + def self.construct_auth_header(options) + username = options["es_user"] + password = options["es_password"] + { "Authorization" => "Basic #{Base64::encode64("#{username}:#{password}")}" } + end + end From 0ec26656597648c9f2061c4bda03a9e947d71521 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 25 Oct 2022 14:52:59 -0500 Subject: [PATCH 113/163] remove debugging code --- lib/datura/file_type.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index aeb0f6f46..e17114837 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -69,8 +69,6 @@ def post_es(es) # NOTE: If you need to do partial updates rather than replacement of doc # you will need to add _update at the end of this URL begin - puts @auth_header - byebug RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, @auth_header.merge({:content_type => :json }) ) rescue => e error = "Error transforming or posting to ES for #{self.filename(false)}: #{e}" From 7b90a0954fca67286217033497b298335c515972 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 25 Oct 2022 14:56:24 -0500 Subject: [PATCH 114/163] update conditional logic for status code, dynamic_templates key --- lib/datura/elasticsearch/index.rb | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 337f5cf04..664d5c690 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -93,12 +93,14 @@ def get_schema_mapping end regex_pieces = [] - doc["dynamic_templates"].each do |template| - mapping = template.map { |k,v| v["match"] }.first - # dynamic fields are listed like *_k and will need - # to be converted to ^.*_k$, then combined into a mega-regex - es_match = mapping.sub("*", ".*") - regex_pieces << es_match + if doc["dynamic_templates"] + doc["dynamic_templates"].each do |template| + mapping = template.map { |k,v| v["match"] }.first + # dynamic fields are listed like *_k and will need + # to be converted to ^.*_k$, then combined into a mega-regex + es_match = mapping.sub("*", ".*") + regex_pieces << es_match + end end if !regex_pieces.empty? regex_joined = regex_pieces.join("|") @@ -231,7 +233,7 @@ def self.clear_index(options) data = self.build_clear_data(options) auth_header = Datura::Helpers.construct_auth_header(options) RestClient.post(url, data.to_json, auth_header.merge({content_type: :json })) { |res, req, result| - if result.code == "200" + if result.code == "200" || result.code == "201" puts res else raise "#{result.code} error when clearing index: #{res}" From f982173a634cc26bf4202bfb639902d8433739b3 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 26 Jan 2023 14:24:01 -0600 Subject: [PATCH 115/163] change endpoint for delete_by_query for ES8 compatibility --- lib/datura/elasticsearch/index.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 664d5c690..71582e7ac 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -209,7 +209,7 @@ def self.clear_all(options) puts "Type: 'Yes I'm sure'" confirm = STDIN.gets.chomp if confirm == "Yes I'm sure" - url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") + url = File.join(options["es_path"], options["es_index"], "_delete_by_query?pretty=true") auth_header = Datura::Helpers.construct_auth_header(options) json = { "query" => { "match_all" => {} } } RestClient.post(url, json.to_json, auth_header.merge({ content_type: :json })) { |res, req, result| @@ -226,7 +226,7 @@ def self.clear_all(options) end def self.clear_index(options) - url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") + url = File.join(options["es_path"], options["es_index"], "_delete_by_query?pretty=true") confirmation = self.confirm_clear(options, url) if confirmation From 602f6be3b5a2ca15bc2ba16515e2c15d3e059ce2 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 27 Oct 2022 11:32:57 -0500 Subject: [PATCH 116/163] upgrade to Ruby 3.0.4 --- .ruby-version | 2 +- Gemfile.lock | 9 +++++++-- datura.gemspec | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.ruby-version b/.ruby-version index 860487ca1..b0f2dcb32 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.7.1 +3.0.4 diff --git a/Gemfile.lock b/Gemfile.lock index d5512b234..fd39ff559 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,9 +19,13 @@ GEM mime-types (3.4.1) mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) + mini_portile2 (2.8.0) minitest (5.16.3) netrc (0.11.0) - nokogiri (1.13.8-x86_64-darwin) + nokogiri (1.13.9) + mini_portile2 (~> 2.8.0) + racc (~> 1.4) + nokogiri (1.13.9-x86_64-darwin) racc (~> 1.4) racc (1.6.0) rake (13.0.6) @@ -35,6 +39,7 @@ GEM unf_ext (0.0.8.2) PLATFORMS + ruby x86_64-darwin-20 DEPENDENCIES @@ -45,4 +50,4 @@ DEPENDENCIES rake (~> 13.0) BUNDLED WITH - 2.2.26 + 2.2.33 diff --git a/datura.gemspec b/datura.gemspec index 316a5e2a0..b673b8a1f 100644 --- a/datura.gemspec +++ b/datura.gemspec @@ -53,7 +53,7 @@ Gem::Specification.new do |spec| ] spec.require_paths = ["lib"] - spec.required_ruby_version = "~> 2.5" + spec.required_ruby_version = "~> 3.0" spec.add_runtime_dependency "colorize", "~> 0.8.1" spec.add_runtime_dependency "nokogiri", "~> 1.10" spec.add_runtime_dependency "rest-client", "~> 2.1" From c6e687de9263de01b9a1b7f45975260935cb8790 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 27 Oct 2022 11:40:51 -0500 Subject: [PATCH 117/163] make keyword arguments compatible with Ruby 3 --- lib/datura/file_types/file_csv.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/file_types/file_csv.rb b/lib/datura/file_types/file_csv.rb index cd8a4e381..92a1cbff6 100644 --- a/lib/datura/file_types/file_csv.rb +++ b/lib/datura/file_types/file_csv.rb @@ -33,7 +33,7 @@ def present?(item) # override to change encoding def read_csv(file_location, encoding="utf-8") - CSV.read(file_location, { + CSV.read(file_location, **{ encoding: encoding, headers: true, return_headers: true From dbe6aaa87cde4c99457b0c132ab922bb19c3101a Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 27 Oct 2022 12:55:37 -0500 Subject: [PATCH 118/163] go up to ruby 3.1.2 --- .ruby-version | 2 +- datura.gemspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ruby-version b/.ruby-version index b0f2dcb32..ef538c281 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.0.4 +3.1.2 diff --git a/datura.gemspec b/datura.gemspec index b673b8a1f..3d6f91b56 100644 --- a/datura.gemspec +++ b/datura.gemspec @@ -53,7 +53,7 @@ Gem::Specification.new do |spec| ] spec.require_paths = ["lib"] - spec.required_ruby_version = "~> 3.0" + spec.required_ruby_version = "~> 3.1" spec.add_runtime_dependency "colorize", "~> 0.8.1" spec.add_runtime_dependency "nokogiri", "~> 1.10" spec.add_runtime_dependency "rest-client", "~> 2.1" From 6b1e46841e941a70ef2401dd4312914dfae25e69 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 8 Nov 2022 15:42:16 -0600 Subject: [PATCH 119/163] add output if nested field is invalid --- lib/datura/elasticsearch/index.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index cb8da9b84..8124bad79 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -137,6 +137,7 @@ def valid_document?(doc) next else # if one of the nested hashes fails, it is invalid + puts "Nested field '#{field}' is invalid" return false end end From de02891ca9ab9a6fe28102faaa7dc7ab3b668846 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 8 Nov 2022 15:47:55 -0600 Subject: [PATCH 120/163] don't use array method on person to avoid errors --- lib/datura/to_es/es_request.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/to_es/es_request.rb b/lib/datura/to_es/es_request.rb index 693a37e07..6b7f42382 100644 --- a/lib/datura/to_es/es_request.rb +++ b/lib/datura/to_es/es_request.rb @@ -206,7 +206,7 @@ def assemble_relations_2 def assemble_additional_2 @json["spatial"] = spatial @json["places"] = places - @json["person"] = Array(person) + @json["person"] = person @json["event"] = event @json["rdf"] = rdf end From ba60f4c6ef93205b05733f22968ebc152ea8f5eb Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 9 Nov 2022 16:15:03 -0600 Subject: [PATCH 121/163] update changelog for new version --- CHANGELOG.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b90e11bf8..ceaf30818 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,21 +25,46 @@ Versioning](https://semver.org/spec/v2.0.0.html). ### Security --> -## [Unreleased](https://github.com/CDRH/datura/compare/v0.2.0-beta...dev) +## [1.0.0](https://github.com/CDRH/datura/compare/v0.2.0-beta...dev) ### Added - minor test for Datura::Helpers.date_standardize - documentation for web scraping - documentation for CsvToEs (transforming CSV files and posting to elasticsearch) +- documentation for adding new ingest formats to Datura +- byebug gem for debugging - instructions for installing Javascript Runtime files for Saxon +- API schema can either be 1.0 or 2.0 (which includes nested fields); 1.0 will be run by default unless 2.0 is specified. Add the following to `public.yml` or `private.yml` in the data repo: +``` +api_version: '2.0' +``` +- schema validation with API version 2.0, invalidly constructed documents will not post +- authentication with Elasticesarch 8.5; add the following to `public.yml` or `private.yml` in the data repo: +``` + es_user: username + es_password: ******** +``` +- field overrides for new fields in the new API schema +- Functionality to transform EAD files and post them to elasticsearch ### Changed +- update ruby to 3.1.2 - date_standardize now relies on strftime instead of manual zero padding for month, day - minor corrections to documentation - XPath: "text" is now ingested as an array and will be displayed delimitted by spaces +- refactored command line methods into elasticsearch library +- refactored and moved date_standardize and date_display helper methods +- Nokogiri methods `get_text` and `get_list` on TEI now return nil rather than empty strings or arrays if there are no matches ### Migration - check to make sure "text" xpath is doing desired behavior +- use Elasticsearch 8.5 or higher and add authentication as described above if security is enabled +- upgrade data repos to Ruby 3.1.2 +- add api version to config as described above +- make sure fields are consistent with the api schema, many have been renamed or changed in format +- add nil checks with get_text and get_list methods +- add EadToES overrides if ingesting EAD files +- if overriding the `read_csv` method in `lib/datura/file_type.rb`, the hash must be prefixed with ** (`**{}`). ## [v0.2.0-beta](https://github.com/CDRH/datura/compare/v0.1.6...v0.2.0-beta) - 2020-08-17 - Altering field and xpath behavior, adds get_elements From 0ddcc33686c5c28b144d3e1c56fce646902fbf86 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 9 Nov 2022 16:39:58 -0600 Subject: [PATCH 122/163] update reference to ruby version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5997622ee..55baa376f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Looking for information about how to post documents? Check out the ## Install / Set Up Data Repo -Check that Ruby is installed, preferably 2.7.x or up. If you are using RVM, see the RVM section below. +Check that Ruby is installed, preferably 3.1.2 or up. If you are using RVM, see the RVM section below. If your project already has a Gemfile, add the `gem "datura"` line. If not, create a new directory and add a file named `Gemfile` (no extension). From a614c6dd2b7409b98dd4e23a224a3f1e91c011fe Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 10 Nov 2022 12:00:34 -0600 Subject: [PATCH 123/163] make changes related to ES and API upgrade --- docs/1_setup/config.md | 5 +++++ docs/1_setup/prepare_index.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/1_setup/config.md b/docs/1_setup/config.md index fe4e7b29f..e58ccd5b5 100644 --- a/docs/1_setup/config.md +++ b/docs/1_setup/config.md @@ -9,7 +9,10 @@ default: collection: es_index es_path + es_user + es_password ``` +(The options es_user and es_password are needed if you are using a secured Elasticsearch index.) If there are any settings which must be different based on the local environment (your computer vs the server), place these in `config/private.yml`. @@ -118,6 +121,8 @@ Some stuff commonly in `private.yml`: - `threads: 5` (5 recommended for PC, 50 for powerful servers) - `es_path: http://localhost:9200` - `es_index: some_index` +- `es_user: elastic` (if you want to use security on your local elasticsearch instance) +- `es_password: ******` - `solr_path: http://localhost:8983/solr` - `solr_core: collection_name` diff --git a/docs/1_setup/prepare_index.md b/docs/1_setup/prepare_index.md index 944f9a719..fa79e7013 100644 --- a/docs/1_setup/prepare_index.md +++ b/docs/1_setup/prepare_index.md @@ -13,7 +13,7 @@ You will need to make sure that somewhere, the following are being set in your p ### Step 2: Prepare Elasticsearch Index -Make sure elasticsearch is installed and running in the location you wish to push to. If there is already an index you will be using, take note of its name and skip this step. If you want to add an index, run this command with a specified environment: +Make sure elasticsearch is installed and running in the location you wish to push to. If there is already an index you will be using, take note of its name and skip this step. (Note that each index must be dedicated to data on one version of the API schema) If you want to add an index, run this command with a specified environment: ``` admin_es_create_index -e development From fbc0897df0ae58e14e78ba88f1ac6a2bc390f957 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 10 Nov 2022 13:36:32 -0600 Subject: [PATCH 124/163] add links to more detailed documentation --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ceaf30818..8861fe0f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ Versioning](https://semver.org/spec/v2.0.0.html). ``` api_version: '2.0' ``` +See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/schema_v2.md) - schema validation with API version 2.0, invalidly constructed documents will not post - authentication with Elasticesarch 8.5; add the following to `public.yml` or `private.yml` in the data repo: ``` @@ -45,7 +46,7 @@ api_version: '2.0' es_password: ******** ``` - field overrides for new fields in the new API schema -- Functionality to transform EAD files and post them to elasticsearch +- functionality to transform EAD files and post them to elasticsearch ### Changed - update ruby to 3.1.2 @@ -58,7 +59,7 @@ api_version: '2.0' ### Migration - check to make sure "text" xpath is doing desired behavior -- use Elasticsearch 8.5 or higher and add authentication as described above if security is enabled +- use Elasticsearch 8.5 or higher and add authentication as described above if security is enabled. See [dev docs instructions](https://github.com/CDRH/cdrh_dev_docs/blob/update_elasticsearch_documentation/publishing/2_basic_requirements.md#downloading-elasticsearch). - upgrade data repos to Ruby 3.1.2 - add api version to config as described above - make sure fields are consistent with the api schema, many have been renamed or changed in format From 3cf22376135695eda8e064c48c319dc39e46cd0c Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 10 Nov 2022 15:02:19 -0600 Subject: [PATCH 125/163] add link to elasticsearch documentation --- docs/4_developers/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/4_developers/installation.md b/docs/4_developers/installation.md index 37eb521c2..0e0171bda 100644 --- a/docs/4_developers/installation.md +++ b/docs/4_developers/installation.md @@ -6,7 +6,7 @@ TODO ### Elasticsearch -TODO +See installation instructions [here](https://github.com/CDRH/cdrh_dev_docs/blob/update_elasticsearch_documentation/publishing/2_basic_requirements.md#downloading-elasticsearch). ### Apache Permissions From 3cadd26ee90874064cbdb73c93cf18fd73e2195e Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 18 Nov 2022 10:03:31 -0600 Subject: [PATCH 126/163] add conditional to creator for nil checking --- lib/datura/to_es/vra_to_es/fields.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/datura/to_es/vra_to_es/fields.rb b/lib/datura/to_es/vra_to_es/fields.rb index c65dda8db..e8ecb1fb2 100644 --- a/lib/datura/to_es/vra_to_es/fields.rb +++ b/lib/datura/to_es/vra_to_es/fields.rb @@ -20,8 +20,10 @@ def category # nested field def creator - creators = get_list(@xpaths["creators"]) - creators.map { |c| { "name" => Datura::Helpers.normalize_space(c) } } + creators = get_list(@xpaths["creator"]) + if creators + creators.map { |c| { "name" => Datura::Helpers.normalize_space(c) } } + end end def collection From 3498ebee701dd26652424f5abcdc24a48d97084b Mon Sep 17 00:00:00 2001 From: Karin Dalziel Date: Thu, 10 Nov 2022 09:29:26 -0600 Subject: [PATCH 127/163] Create schema_v2.md --- docs/schema_v2.md | 152 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 docs/schema_v2.md diff --git a/docs/schema_v2.md b/docs/schema_v2.md new file mode 100644 index 000000000..e5985a32a --- /dev/null +++ b/docs/schema_v2.md @@ -0,0 +1,152 @@ +## CDRH Schema, version 2 + +| NEW FIELD NAME | likely facet field? | Metadata Equivalent | ORIGINAL FIELD NAME | DESCRIPTION | FIELD TYPE | | EXAMPLE | +| ----------------------------------------------------------------------------------------- | ------------------- | --------------------------- | ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Resourse identification, website display | +| identifier | | | identifier | Unique identifier of the resource. | keyword | | oscys.case.0001.001 | +| collection | y | | collection | User friendly and URL valid name of project. Typically consists of directory under specified web domain. | keyword | | oscys, quillsandfeathers | +| collection\_desc | | | collection\_desc | Full CDRH name of the project. (e.g. “The William F. Cody Archive”) | keyword | | O Say Can You See: Early Washington, D.C., Law & Family | +| uri | | | uri | Full URI of resource. (Actual site not API site) | keyword | | [http://earlywashingtondc.org/doc/oscys.case.0001.001](http://earlywashingtondc.org/doc/oscys.case.0001.001) | +| uri\_data | | | uri\_data | Full URL to XML of data, when available | keyword | | [http://earlywashingtondc.org/files/oscys/tei/oscys.case.0001.001.xml](http://earlywashingtondc.org/files/oscys/tei/oscys.case.0001.001.xml) | +| uri\_html | | | uri\_html | Full URL to HTML snippit of data | keyword | | [http://earlywashingtondc.org/files/oscys/html-generated/oscys.case.0001.001.txt](http://earlywashingtondc.org/files/oscys/html-generated/oscys.case.0001.001.txt) | +| data\_type | | | data\_type | Format the data was originally stored in at CDRH. | keyword | | tei
| +| fig\_location | | | fig\_location | URI to location of figure. | keyword | | [http://earlywashingtondc.org/figures/](http://earlywashingtondc.org/figures/) | +| cover\_image | | | image\_id | Unambiguous reference to the image when the image id does not match the file id. | keyword | | oscys.case.0001.001.001.jpg | +| title | | dcterms:title | title | Name given to the resource. | keyword, copied into text | text? | The Once and Future King | +| title\_sort | | | title\_sort | Name given to the resource lowercased with articles removed | keyword | | once and future king | +| alternative | | dcterms:alternative | alternative | Alternative name for the resource. | keyword, copied into text | text? | Petition for Habeas Corpus | +| date\_updated | m | | NEW | | date | | | +| category | y | | category | Category on web page where resource occurs. Category fields are meant to be hierarchical and exclusive, for other types of organization look to subjects, keywords, etc

Each site will have a controlled vocabulary of its own | keyword | | works | +| category2 | y | | subcategory | | keyword | | works | novels | +| category3 | y | | NEW | 3rd level category | keyword | | works | novels | historical fiction | +| category4 | y | | NEW | 4th level category | keyword | | works | novels | historical fiction | civil war | +| category5 | y | | NEW | 5th level category | keyword | | etc | +| notes | | | NEW | | keyword | | | +| Metadata: Digital Item | +| contributor | | dcterms:contributor | contributor | CONTAINER FIELD
"Entity responsible for making contributions
to the resource." | | | | +| contributor.name | | | [contributor.name](http://contributor.name) | Entity responsible for making contributions
to the resource. | keyword | | \[Allison, Dee Ann\]
\[Walter, Katherine\] | +| [contributor.id](http://contributor.id/) | | | [contributor.id](http://contributor.id) | ID of the contributor | keyword | | \[https://orcid.org/0000-0002-4671-061X\]
(leave blank for no id) | +| contributor.role | | | contributor.role | | keyword | | \[researcher\]
\[Principal Investigator\]
\[encoder\] | +| Metadata: Original Item | +| creator | | dcterms:creator | creator | CONTAINER FIELD
An entity primarily responsible for making the resource.
Examples of a Creator include a person, an organization, or a service. | | | Use person field with role instead | +| creator.name | y | | [creator.name](http://creator.name) | Creator field name | keyword | copied into text | Use person field with role instead | +| creator.id | y | | [creator.id](http://creator.id) | Creator field ID (if available) | keyword | | Use person field with role instead | +| citation | | | | | | | | +| citation.role | | | NEW | | keyword | | | +| [citation.id](http://citation.id/) | | bibo:identifier | NEW | an identifier of the original item | keyword | | | +| citation.title | | dcterms:title | NEW | Used to describe the title of a bibliographic resource | keyword | text? | | +| citation.publisher | y | bibo:producer | publisher | Entity responsible for making the resource available. | keyword | | University of Nebraska Press, Lincoln & London, 1992 | +| citation.date | | dcterms:date | NEW | Date the resource was orginally created. | date | | 1900-01-01 | +| citation.issue | | bibo:issue | NEW | An issue number | keyword | | | +| citation.page\_start | | bibo:pageStart | NEW | Starting page number within a continuous page range. | keyword (some pages are roman numerals) | | 4 | +| citation.page\_end | | bibo:pageEnd | NEW | Ending page number within a continuous page range. | keyword | | 5 (if applicable) | +| citation.section | | bibo:section | NEW | A section number | keyword | | | +| citation.volume | | bibo:volume | NEW | A volume number | keyword | | | +| citation.place | | juso:name | NEW | This property indicates the name of the spatial thing. | keyword | | | +| citation.title\_a | | tei title level a | NEW | typically an article | keyword | text? | | +| citation.title\_m | | tei title level m | NEW | typically a monograph | keyword | text? | | +| citation.title\_j | y | tei title level j | NEW | typically a journal name | keyword | text? | | +| date | y | dcterms:date | | the date that will be used to sort and run date queries on item | date | | | +| date\_display | | | date\_display | Date in whatever display format is used on the site | keyword | text? | January, 1900 | +| date\_not\_before | | | date\_not\_before | Inclusive beginning date of resource. | date | | 1900-01-01 | +| date\_not\_after | | | date\_not\_after | Inclusive ending date of resource. | date | | 1900-01-31 | +| format | y | dcterms:format | format | File format, physical medium, or dimensions of the resource. | keyword | copied into text? | Film: 16mm Safety Film | +| medium | y | dcterms:medium | medium | Material or physical carrier of the resource. | keyword | copied into text? | Film | +| extent | | dcterms:extent | extent | Size or duration of the resource. | keyword | | 4:03 | +| language | y | dcterms:language | language | Primary / original language of the resource | keyword | | en | +| rights\_holder | y | dcterms:rightsHolder | rights\_holder | A person or organization owning or managing rights over the resource. | keyword | copied into text? | Huntington Library | +| rights | | dcterms:rights | rights | Information about the rights held in and over the resource. | keyword | copied into text? | All Rights Reserved. Contact Rights Holder for Permissions Information.
or
Covered by a CC-By License https://creativecommons.org/licenses/by/2.0/ | +| rights\_uri | | | rights\_uri | URI to rights holder information. | keyword | | [http://www.huntington.org/](http://www.huntington.org/) | +| container\_box | | ead container type = box | NEW | box an item is kept in, as in an archive | keyword | | | +| container\_folder | | ead container type = folder | NEW | folder an item is kept in, as in an archive | keyword | | | +| Metadata: Interpretive | +| subjects | y | dcterms:subject | subjects | Topic of the content of the resource. | keyword | copied into text? | \[Horror in art\]
\[Poisonous spiders--Venom\] | +| abstract | | dcterms:abstract | abstract | Abstract of the resource. | keyword? (for display or searching?)
| text? | The poem is not one of DGR's great sonnets, and it pales before the majestic painting it was written to accompany. Nevertheless, it is quite an interesting and important text. | +| description | | dcterms:description | description | Short description of the resource. | text | text? | A Poem by Dante Gabriel Rossetti | +| type | y | dcterms:type | type | Nature or genre of the resource. | keyword | copied into text? | Video | +| topics | y | | topics | Topics of content of resource. | keyword | copied into text? | | +| keywords | y | | keywords | Keywords used for resource. | keyword | copied into text? | | +| keywords2 | y | | NEW | Another set of keywords, used in sites to create another way to browse | keyword | copied into text? | decade | +| keywords3 | y | | NEW | Another set of keywords, used in sites to create another way to browse | keyword | copied into text? | | +| keywords4 | y | | NEW | Another set of keywords, used in sites to create another way to browse | keyword | copied into text? | | +| Relation to other items | +| relation | | dcterms:relation | relation | A related resource that is substantially the same as the described resource, but in another format. | keyword | | oscys.case.0001.001-B | +| source | | dcterms:source | source | A related resource from which the described resource is derived | keyword | | oscys.case.0001.001-A | +| has\_part | | dcterms:hasPart | NEW | parts of the resource, for example items pasted into a scrapbook | | | | +| has\_part.role | | | | | | | | +| has\_part.id | | | | | keyword | | cdrh.0001 | +| has\_part.title | | | | | keyword | | Resource title | +| has\_part.order | | | | | whole number | | 1 | +| is\_part\_of | | dcterms:isPartOf | NEW | the containing resource, for example the scrapbook the individual items are in | | | | +| is\_part\_of.role | | | | | | | | +| is\_part\_of.id | | | | | keyword | | cdrh.0001 | +| is\_part\_of.title | | | | | keyword | | Resource title | +| is\_part\_of.order | | | | | whole number | | 1 | +| previous\_item | | | NEW | previous item in a series. role can be used to create multiple nexts - for instance, previous letter in a mailing sequence, pervious letter by date | | | | +| previous\_item.role | | | | | | | | +| [previous\_item.id](http://previous_item.id/) | | | | | keyword | | cdrh.0001 | +| previous\_item.title | | | | | keyword | | Resource title | +| previous\_item.order | | | | | whole number | | 1 | +| next\_item | | | NEW | next item in a series. role can be used to create multiple nexts - for instance, next letter in a mailing sequence, next letter by date | | | | +| next\_item.role | | | | | | | | +| [next\_item.id](http://next_item.id/) | | | | | keyword | | cdrh.0001 | +| next\_item.title | | | | | keyword | | Resource title | +| next\_item.order | | | | | whole number | | 1 | +| Additional Data types | +| spatial | | | spatial | CONTAINER FIELD | | | | +| spatial.role | | | | | keyword | | | +| spatial.name | y | juso:name | spatial.title | Title / display name of location | keyword | copied into text? | Display name for this location, typically built from other fields, but potentially not. | +| spatial.description | | | spatial.description | Description | text | text? | | +| spatial.type | y | | spatial.type | | keyword | copied into text? | "origin" or "destination" used to distinguish multiple spatial records for one item (for example, for an item of correspondence) | +| spatial.short\_name | y | juso:short\_name | spatial.place\_name | Specific name of location in question, such as the army camp name, business, event title, etc | keyword, copied into text | | Camp Hollowell, Kimball Recital Hall, The Coffeehouse, Lancaster County Fairgrounds | +| spatial.coordinates | y | juso:geometry | spatial.coordinates | | geopoint | | \[-96.6669600, 40.8000000\] | +| spatial.id | | | [spatial.id](http://coverage.spatial.id/) | | keyword | | ????
| +| spatial.city | y | juso:city | spatial.city | | keyword | copied into text? | | +| spatial.township | | juso:Township | NEW | | | copied into text? | | +| spatial.county | | juso:county | spatial.county | | keyword | copied into text? | | +| spatial.country | y | juso:country | spatial.country | | keyword | copied into text? | | +| spatial.region | y | juso:within | NEW? | | keyword | copied into text? | | +| spatial.state | | juso:state | spatial.state | | keyword | copied into text? | | +| spatial.street | | juso:street | spatial.street | | keyword | | | +| spatial.postal\_code | | juso:postal\_code | spatial.postal\_code | | keyword | | | +| spatial.note | | | | | | | | +| deprecate and replace with place with role of "placename" and only place\_name filled out | | | places | Place names mentioned in the resource. | keyword | | | +| person | | foaf:Person | person | any people other than contributors associated with resource | | | | +| person.name | y | foaf:name | [person.name](http://person.name) | Name as we wish it to appear | keyword | copied into text? | \[Cody, William F.\] | +| [person.id](http://person.id/) | y | | [person.id](http://person.id) | Optional, if exists, may be from VIAF or similar. | keyword | | \[http://viaf.org/viaf/100252467\] | +| person.role | y | | person.role | Role of person. Common examples are recipient and sender, less common examples are attorney and defendant | keyword | copied into text? | \[sender\]
\[recipient\]
\[creator\]
\[editor\] | +| person.note | | | NEW | | keyword | | | +| person.order | | | NEW | | keyword | | | +| person.birth\_date | | foaf:birthday | NEW | | date | | \[1899-03-04\] | +| person.death\_date | | | NEW | | date | | | +| person.age\_category | y | | NEW | used when resources are categorizing the age of the participant at the time of the event. For instance, a minor in a court case | keyword | | \[minor\]
\[adult\] | +| person.name\_last | | foaf:lastName | NEW | | keyword | | | +| person.name\_given | | foaf:givenName | NEW | | keyword | | | +| person.name\_alternate | | | NEW | | keyword | | | +| person.name\_previous | | | | | | | | +| person.race | y | | NEW | | keyword | | | +| person.sex | y | | NEW | | keyword | | | +| person.gender | y | foaf:gender | NEW | | keyword | | | +| person.nationality | y | | NEW | | keyword | | | +| person.trait1 | y | | NEW | | keyword | | | +| person.trait2 | y | | NEW | | keyword | | | +| event | | | | | | | | +| event.type | y | | NEW | | keyword | | | +| event.agent | y | event:agent | NEW | Relates an event to an active agent (a person, a computer, ... :-) ) | keyword | | | +| event.factor | | event:factor | NEW | Relates an event to a passive factor (a tool, an instrument, an abstract cause...) | keyword | | points of law cited in a case | +| event.product | y | event:product | NEW | | keyword | | case outcome | +| event.date\_begin | | event\_date\_begin | NEW | | date | | | +| event.date\_end | | event\_date\_end | NEW | | date | | | +| event.trait1 | | | NEW | | keyword | | can be used for case keywords, i.e. civil, criminal | +| event.trait2 | | | NEW | | keyword | | | +| event.notes | | | NEW | | keyword | | | +| RDF | | | | The RDF field can be used to record any other data that needs to be associated with the record, for instance relationships | | | | +| rdf.type | | | NEW | | keyword | | \[relationship\] | +| rdf.subject | y | ref:subject | NEW | | keyword | | \[Smith, John\] | +| rdf.predicate | y | rdf:predicate | NEW | | keyword | | \[is married to\] | +| rdf.object | y | rdf:object | NEW | | keyword | | \[Smith, Mary\] | +| rdf.source | | | NEW | | keyword | | item.0001 | +| rdf.note | | | NEW | | keyword | | | +| Text search | +| annotations\_text | | | annotations\_text | Place for annotations text, so we can search annotations separately from the main text | text | | | +| text | | | text | Combined text of all the above fields for key word searching. | text
| | | From 5b0f54f1d81af819513740441ec8e7d1a9ee2986 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 25 Jan 2023 12:44:34 -0600 Subject: [PATCH 128/163] make sure webs_to_es fields can handle nil values --- lib/datura/to_es/webs_to_es/fields.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/datura/to_es/webs_to_es/fields.rb b/lib/datura/to_es/webs_to_es/fields.rb index 3163706be..1ef633126 100644 --- a/lib/datura/to_es/webs_to_es/fields.rb +++ b/lib/datura/to_es/webs_to_es/fields.rb @@ -39,7 +39,11 @@ def data_type end def date(before=true) - datestr = get_list(@xpaths["date"]).first + if get_list(@xpaths["date"]) + datestr = get_list(@xpaths["date"]).first + else + datestr = nil + end if datestr Datura::Helpers.date_standardize(datestr, true) end @@ -80,7 +84,9 @@ def format end def image_id - get_list(@xpaths["image_id"]).first + if get_list(@xpaths["image_id"]) + get_list(@xpaths["image_id"]).first + end end def keywords @@ -218,7 +224,9 @@ def works # new/moved fields for API 2.0 def cover_image - get_list(@xpaths["image_id"]).first + if get_list(@xpaths["image_id"]) + get_list(@xpaths["image_id"]).first + end end def date_updated From 594e57f232c07cafc3fbe7a55010633e69a9fbad Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 4 May 2023 12:31:24 -0500 Subject: [PATCH 129/163] add new fields to 2.0 schema implements #219 --- lib/config/es_api_schemas/2.0.yml | 63 ++++++++++++++++++++++++++- lib/datura/to_es/csv_to_es/fields.rb | 9 ++++ lib/datura/to_es/ead_to_es/fields.rb | 12 +++++ lib/datura/to_es/es_request.rb | 5 ++- lib/datura/to_es/html_to_es/fields.rb | 12 +++++ lib/datura/to_es/tei_to_es/fields.rb | 12 +++++ lib/datura/to_es/vra_to_es/fields.rb | 12 +++++ lib/datura/to_es/webs_to_es/fields.rb | 11 +++++ 8 files changed, 132 insertions(+), 4 deletions(-) diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index d46c63ca0..2616f6c49 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -231,12 +231,38 @@ mappings: keywords4: type: keyword normalizer: keyword_normalized - relation: + keywords5: type: keyword normalizer: keyword_normalized - source: + relation: type: keyword normalizer: keyword_normalized + has_source: + type: nested + properties: + type: keyword + normalizer: keyword_normalized + id: + type: keyword + normalizer: keyword_normalized + title: + type: keyword + normalizer: keyword_normalized + order: + type: integer + has_relation: + type: nested + properties: + type: keyword + normalizer: keyword_normalized + id: + type: keyword + normalizer: keyword_normalized + title: + type: keyword + normalizer: keyword_normalized + order: + type: integer has_part: type: nested properties: @@ -357,6 +383,21 @@ mappings: note: type: keyword normalizer: keyword_normalized + trait1: + type: keyword + normalizer: keyword_normalized + trait2: + type: keyword + normalizer: keyword_normalized + trait3: + type: keyword + normalizer: keyword_normalized + trait4: + type: keyword + normalizer: keyword_normalized + trait5: + type: keyword + normalize: keyword_normalized places: #DEPRECATED type: keyword normalizer: keyword_normalized @@ -416,6 +457,15 @@ mappings: trait2: type: keyword normalizer: keyword_normalized + trait3: + type: keyword + normalizer: keyword_normalized + trait4: + type: keyword + normalizer: keyword_normalized + trait5: + type: keyword + normalizer: keyword_normalized event: type: nested properties: @@ -443,6 +493,15 @@ mappings: trait2: type: keyword normalizer: keyword_normalized + trait3: + type: keyword + normalizer: keyword_normalized + trait4: + type: keyword + normalizer: keyword_normalized + trait5: + type: keyword + normalizer: keyword_normalized notes: type: keyword normalizer: keyword_normalized diff --git a/lib/datura/to_es/csv_to_es/fields.rb b/lib/datura/to_es/csv_to_es/fields.rb index 50c7efc81..9adc59692 100644 --- a/lib/datura/to_es/csv_to_es/fields.rb +++ b/lib/datura/to_es/csv_to_es/fields.rb @@ -292,6 +292,9 @@ def keywords3 def keywords4 end + def keywords5 + end + def has_part end @@ -310,4 +313,10 @@ def event def rdf end + def has_source + end + + def has_relation + end + end diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb index 5128f111e..dfca070f7 100644 --- a/lib/datura/to_es/ead_to_es/fields.rb +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -327,6 +327,10 @@ def keywords4 get_text(@xpaths["keywords4"]) end + def keywords5 + get_text(@xpaths["keywords5"]) + end + def has_part # nested end @@ -350,4 +354,12 @@ def event def rdf # nested end + + def has_source + # nested + end + + def has_relation + # nested + end end diff --git a/lib/datura/to_es/es_request.rb b/lib/datura/to_es/es_request.rb index 6b7f42382..ee120bbf7 100644 --- a/lib/datura/to_es/es_request.rb +++ b/lib/datura/to_es/es_request.rb @@ -192,11 +192,12 @@ def assemble_metadata_interpretive_2 @json["keywords2"] = keywords2 @json["keywords3"] = keywords3 @json["keywords4"] = keywords4 + @json["keywords5"] = keywords5 end def assemble_relations_2 - @json["relation"] = relation - @json["source"] = source + @json["has_relation"] = has_relation + @json["has_source"] = has_source @json["has_part"] = has_part @json["is_part_of"] = is_part_of @json["previous_item"] = previous_item diff --git a/lib/datura/to_es/html_to_es/fields.rb b/lib/datura/to_es/html_to_es/fields.rb index 886c1bd7d..725ac4c4d 100644 --- a/lib/datura/to_es/html_to_es/fields.rb +++ b/lib/datura/to_es/html_to_es/fields.rb @@ -278,6 +278,10 @@ def keywords4 get_text(@xpaths["keywords4"]) end + def keywords4 + get_text(@xpaths["keywords5"]) + end + def has_part # nested end @@ -302,4 +306,12 @@ def rdf # nested end + def has_source + # nested + end + + def has_relation + # nested + end + end diff --git a/lib/datura/to_es/tei_to_es/fields.rb b/lib/datura/to_es/tei_to_es/fields.rb index 757ea6f21..9b435cb72 100644 --- a/lib/datura/to_es/tei_to_es/fields.rb +++ b/lib/datura/to_es/tei_to_es/fields.rb @@ -333,6 +333,10 @@ def keywords4 get_text(@xpaths["keywords4"]) end + def keywords5 + get_text(@xpaths["keywords5"]) + end + def has_part # nested end @@ -357,6 +361,14 @@ def rdf # nested end + def has_source + # nested + end + + def has_relation + # nested + end + protected # default behavior is simply to comma delineate fields diff --git a/lib/datura/to_es/vra_to_es/fields.rb b/lib/datura/to_es/vra_to_es/fields.rb index e8ecb1fb2..0f62b94d0 100644 --- a/lib/datura/to_es/vra_to_es/fields.rb +++ b/lib/datura/to_es/vra_to_es/fields.rb @@ -318,6 +318,10 @@ def keywords4 get_text(@xpaths["keywords4"]) end + def keywords5 + get_text(@xpaths["keywords5"]) + end + def has_part # nested end @@ -341,4 +345,12 @@ def event def rdf # nested end + + def has_source + # nested + end + + def has_relation + # nested + end end diff --git a/lib/datura/to_es/webs_to_es/fields.rb b/lib/datura/to_es/webs_to_es/fields.rb index 1ef633126..90e91d70d 100644 --- a/lib/datura/to_es/webs_to_es/fields.rb +++ b/lib/datura/to_es/webs_to_es/fields.rb @@ -283,6 +283,10 @@ def keywords4 get_text(@xpaths["keywords4"]) end + def keywords5 + get_text(@xpaths["keywords5"]) + end + def has_part # nested end @@ -307,4 +311,11 @@ def rdf # nested end + def has_source + # nested + end + + def has_relation + # nested + end end From e928e7e4b010da769b0baed66ffd019a95050e42 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 4 May 2023 14:02:06 -0500 Subject: [PATCH 130/163] fix errors --- lib/config/es_api_schemas/2.0.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/config/es_api_schemas/2.0.yml b/lib/config/es_api_schemas/2.0.yml index 2616f6c49..303b22ec6 100644 --- a/lib/config/es_api_schemas/2.0.yml +++ b/lib/config/es_api_schemas/2.0.yml @@ -240,7 +240,8 @@ mappings: has_source: type: nested properties: - type: keyword + role: + type: keyword normalizer: keyword_normalized id: type: keyword @@ -253,7 +254,8 @@ mappings: has_relation: type: nested properties: - type: keyword + role: + type: keyword normalizer: keyword_normalized id: type: keyword @@ -397,7 +399,7 @@ mappings: normalizer: keyword_normalized trait5: type: keyword - normalize: keyword_normalized + normalizer: keyword_normalized places: #DEPRECATED type: keyword normalizer: keyword_normalized From aa0770b34d9d6684ca722b436cdf2ba4e022d2ff Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 2 Mar 2023 11:12:29 -0600 Subject: [PATCH 131/163] create methods and overrides to transform pdf files to elasticsearch --- lib/datura/file_types/file_pdf.rb | 96 ++++++++ lib/datura/to_es/pdf_to_es.rb | 53 +++++ lib/datura/to_es/pdf_to_es/fields.rb | 313 ++++++++++++++++++++++++++ lib/datura/to_es/pdf_to_es/request.rb | 8 + 4 files changed, 470 insertions(+) create mode 100644 lib/datura/file_types/file_pdf.rb create mode 100644 lib/datura/to_es/pdf_to_es.rb create mode 100644 lib/datura/to_es/pdf_to_es/fields.rb create mode 100644 lib/datura/to_es/pdf_to_es/request.rb diff --git a/lib/datura/file_types/file_pdf.rb b/lib/datura/file_types/file_pdf.rb new file mode 100644 index 000000000..43b8e9ef7 --- /dev/null +++ b/lib/datura/file_types/file_pdf.rb @@ -0,0 +1,96 @@ +require "pdf-reader" +require_relative "../file_type.rb" + +class FilePdf < FileType + def initialize(file_location, options) + super(file_location, options) + #convert to pdf reading + @pdf = read_pdf(file_location) + end + + def build_html_from_pdf + # #can this be converted? not sure + # @csv.each_with_index do |row, index| + # next if row.header_row? + # # Note: if overriding this function, it's recommended to use + # # a more specific identifier for each row of the CSV + # # but since this is a generic version, simply using the current iteration number + # # using XML instead of HTML for simplicity's sake + # builder = Nokogiri::XML::Builder.new do |xml| + # xml.div(class: "main_content") { + # xml.ul { + # @csv.headers.each do |header| + # xml.li("#{header}: #{row[header]}") + # end + # } + # } + # end + # write_html_to_file(builder, index) + # end + end + + def present?(item) + !item.nil? && !item.empty? + end + + # override to change encoding + def read_pdf(file_location) + #convert to pdf + PDF::Reader.new(file_location) + end + + # override as necessary per project + def pdf_to_es(pdf) + PdfToEs.new(pdf, options, self.filename(false)).json + end + + + def transform_es + puts "transforming #{self.filename}" + es_doc = [] + es_doc << pdf_to_es(@pdf) + if @options["output"] + filepath = "#{@out_es}/#{self.filename(false)}.json" + File.open(filepath, "w") { |f| f.write(pretty_json(es_doc)) } + end + es_doc + end + + def transform_iiif + raise "PDF to IIIF is not yet generalized, please override on a per project basis" + end + + def transform_html + puts "transforming #{self.filename} to HTML subdocuments" + build_html_from_csv + # transform_html method is expected to send back a hash + # but already wrote to filesystem so just sending back empty + {} + end + + # I am not sure that this is going to be the best way to set this up + # but until we have more examples of CSVs that need to be ingested + # it will have to do! (transmississippi only collection so far) + def transform_solr + puts "transforming #{self.filename}" + solr_doc = Nokogiri::XML("") + doc = Nokogiri::XML::Node.new("doc", solr_doc) + # row_to_solr should return an XML::Node object with children + doc = pdf_to_solr(doc, pdf) + solr_doc.at_css("add").add_child(doc) + + # Uncomment to debug + # puts solr_doc.root.to_xml + if @options["output"] + filepath = "#{@out_solr}/#{self.filename(false)}.xml" + File.open(filepath, "w") { |f| f.write(solr_doc.root.to_xml) } + end + { "doc" => solr_doc.root.to_xml } + end + + def write_html_to_file(builder, index) + filepath = "#{@out_html}/#{index}.html" + puts "writing to #{filepath}" if @options["verbose"] + File.open(filepath, "w") { |f| f.write(builder.to_xml) } + end +end diff --git a/lib/datura/to_es/pdf_to_es.rb b/lib/datura/to_es/pdf_to_es.rb new file mode 100644 index 000000000..587524466 --- /dev/null +++ b/lib/datura/to_es/pdf_to_es.rb @@ -0,0 +1,53 @@ +require_relative "../helpers.rb" +require_relative "pdf_to_es/fields.rb" +require_relative "pdf_to_es/request.rb" + +######################################### +# NOTE: DO NOT EDIT THIS FILE!!!!!!!!! # +######################################### +# (unless you are a CDRH dev and then you may do so very cautiously) +# this file provides defaults for ALL of the collections included +# in the API and changing it could alter dozens of sites unexpectedly! +# PLEASE RUN LOADS OF TESTS AFTER A CHANGE BEFORE PUSHING TO PRODUCTION + +# WHAT IS THIS FILE? +# This file sets up default behavior for transforming PDF +# documents to Elasticsearch JSON documents + +class PdfToEs + + attr_reader :json, :pdf + # variables + # id, row, pdf + + def initialize(pdf, options={}, filename=nil) + @pdf = pdf + @options = options + @filename = filename + @id = get_id + + create_json + end + + # getter for @json response object + def create_json + @json = {} + # if anything needs to be done before processing + # do it here (ex: reading in annotations into memory) + preprocessing + assemble_json + postprocessing + end + + def get_id + filename.delete_suffix(".pdf") + end + + def preprocessing + # copy this in your csv_to_es collection file to customize + end + + def postprocessing + # copy this in your csv_to_es collection file to customize + end +end diff --git a/lib/datura/to_es/pdf_to_es/fields.rb b/lib/datura/to_es/pdf_to_es/fields.rb new file mode 100644 index 000000000..58b895baf --- /dev/null +++ b/lib/datura/to_es/pdf_to_es/fields.rb @@ -0,0 +1,313 @@ +class PdfToEs + # Note to add custom fields, use "assemble_collection_specific" from request.rb + # and be sure to either use the _d, _i, _k, or _t to use the correct field type + + ########## + # FIELDS # + ########## + # beginning with fields from API 1.0, including those that are unchanged in 2.0 + + def id + get_id + end + + def alternative + # @row["alternative"] + end + + def annotations_text + # @row["annotations_text"] + end + + def category + # @row["category"] + end + + # nested field + def creator + # if @row["creator"] + # @row["creator"].split("; ").map do |p| + # { "name" => p } + # end + # end + end + + def collection + @options["collection"] + end + + def collection_desc + @options["collection_desc"] || @options["collection"] + end + + def container_box + end + + def container_folder + end + + # nested field + def contributor + # if @row["contributor"] + # @row["contributor"].split("; ").map do |p| + # { "name" => p } + # end + # end + end + + def data_type + "pdf" + end + + def date(before=true) + # Datura::Helpers.date_standardize(@row["date"], before) + end + + def date_display + # Datura::Helpers.date_display(date) + end + + def date_not_after + # if @row["date_not_after"] && !@row["date_not_after"].empty? + # Datura::Helpers.date_standardize(@row["date_not_after"], false) + # else + # date(false) + # end + end + + def date_not_before + # if @row["date_not_before"] && !@row["date_not_before"].empty? + # Datura::Helpers.date_standardize(@row["date_not_before"], true) + # else + # date(true) + # end + end + + def description + # @row["description"] + end + + def extent + # @row["extent"] + end + + def format + # @row["format"] + end + + def image_id + # @row["image_id"] + end + + def keywords + # if @row["keywords"] + # @row["keywords"].split("; ") + # end + end + + def language + # @row["language"] + end + + def languages + # if @row["languages"] + # @row["languages"].split("; ") + # end + end + + def medium + # @row["medium"] + end + + # nested field + def person + # if @row["person"] + # @row["person"].split("; ").map do |p| + # { "name" => p } + # end + # end + end + + def places + # if @row["places"] + # @row["places"].split("; ") + # end + end + + def publisher + # @row["publisher"] + end + + # nested field + def recipient + # if @row["recipient"] + # @row["recipient"].split("; ").map do |p| + # { "name" => p } + # end + # end + end + + def relation + # @row["relation"] + end + + def rights + # @row["rights"] + end + + def rights_holder + # @row["rights_holder"] + end + + def rights_uri + # @row["rights_uri"] + end + + def source + # @row["source"] + end + + # nested field + def spatial + end + + def subjects + + end + + def subcategory + @row["subcategory"] + end + + # text is generally going to be pulled from + def text + text_all = [] + reader.pages.each do |page| + text_all << page.text + end + text_all += text_additional + text_all = text_all.compact + Datura::Helpers.normalize_space(text_all.join(" ")) + end + + # override and add by collection as needed + def text_additional + [ title ] + end + + def title + @filename.delete_suffix(".pdf") + end + + def title_sort + Datura::Helpers.normalize_name(title) if title + end + + def topics + # if @row["topics"] + # @row["topics"].split("; ") + # end + end + + def type + # @row["type"] + end + + def uri + File.join( + @options["site_url"], + "item", + @id + ) + end + + def uri_data + File.join( + @options["data_base"], + "data", + @options["collection"], + "source/pdf", + "#{@filename}.pdf" + ) + end + + def uri_html + File.join( + @options["data_base"], + "data", + @options["collection"], + "output", + @options["environment"], + "html", + "#{@id}.html" + ) + end + + def works + # if @row["works"] + # @row["works"].split("; ") + # end + end + + # new/moved fields for API 2.0 + + def cover_image + # @row["image_id"] + end + + def date_updated + end + + def fig_location + end + + def category2 + # @row["subcategory"] + end + + def category3 + end + + def category4 + end + + def category5 + end + + def notes + end + + def citation + end + + def abstract + end + + def keywords2 + end + + def keywords3 + end + + def keywords4 + end + + def has_part + end + + def is_part_of + end + + def previous_item + end + + def next_item + end + + def event + end + + def rdf + end + +end diff --git a/lib/datura/to_es/pdf_to_es/request.rb b/lib/datura/to_es/pdf_to_es/request.rb new file mode 100644 index 000000000..dd20f8a62 --- /dev/null +++ b/lib/datura/to_es/pdf_to_es/request.rb @@ -0,0 +1,8 @@ +class PdfToEs + include EsRequest + + # please refer to generic es_request.rb file + # and override the JSON being sent to elasticsearch here, if needed + # project specific overrides should go in the COLLECTION's overrides! + +end From 16a421bcd789d7b7a23b396b23cbc3c4fd98a244 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 2 Mar 2023 11:18:18 -0600 Subject: [PATCH 132/163] add pdf option for command line options --- lib/datura/data_manager.rb | 3 ++- lib/datura/parser_options/post.rb | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 0b0ebae72..24f4898e9 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -24,7 +24,8 @@ def self.format_to_class "html" => FileHtml, "tei" => FileTei, "vra" => FileVra, - "webs" => FileWebs + "webs" => FileWebs, + "pdf" => FilePdf } classes.default = FileCustom classes diff --git a/lib/datura/parser_options/post.rb b/lib/datura/parser_options/post.rb index daa9b7408..c3dd1218d 100644 --- a/lib/datura/parser_options/post.rb +++ b/lib/datura/parser_options/post.rb @@ -22,12 +22,12 @@ def self.post_params # default to no restricted format options["format"] = nil - opts.on( '-f', '--format [input]', 'Supported formats (csv, html, tei, vra, webs)') do |input| + opts.on( '-f', '--format [input]', 'Supported formats (csv, html, pdf, tei, vra, webs)') do |input| if %w[authority annotations].include?(input) puts "'authority' and 'annotations' are invalid formats".red puts "Please select a supported format or rename your custom format" exit - elsif !%w[csv html tei vra webs].include?(input) + elsif !%w[csv html pdf tei vra webs].include?(input) puts "Caution: Requested custom format #{input}.".red puts "See FileCustom class for implementation instructions" end From d3f87be77ef3048eb4e563192589c5cccff31378 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Thu, 2 Mar 2023 11:47:27 -0600 Subject: [PATCH 133/163] fix variable names --- lib/datura/to_es/pdf_to_es.rb | 2 +- lib/datura/to_es/pdf_to_es/fields.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/to_es/pdf_to_es.rb b/lib/datura/to_es/pdf_to_es.rb index 587524466..852981b38 100644 --- a/lib/datura/to_es/pdf_to_es.rb +++ b/lib/datura/to_es/pdf_to_es.rb @@ -40,7 +40,7 @@ def create_json end def get_id - filename.delete_suffix(".pdf") + @filename.delete_suffix(".pdf") end def preprocessing diff --git a/lib/datura/to_es/pdf_to_es/fields.rb b/lib/datura/to_es/pdf_to_es/fields.rb index 58b895baf..8b0c4f277 100644 --- a/lib/datura/to_es/pdf_to_es/fields.rb +++ b/lib/datura/to_es/pdf_to_es/fields.rb @@ -182,7 +182,7 @@ def subcategory # text is generally going to be pulled from def text text_all = [] - reader.pages.each do |page| + @pdf.pages.each do |page| text_all << page.text end text_all += text_additional From ab921b7aef9a3ed673e1391baee01194cff9734f Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 6 Mar 2023 09:48:18 -0600 Subject: [PATCH 134/163] adjust and clarify documentation for adding new formats --- docs/4_developers/new_formats.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/4_developers/new_formats.md b/docs/4_developers/new_formats.md index a731bb9a7..9ebdb6640 100644 --- a/docs/4_developers/new_formats.md +++ b/docs/4_developers/new_formats.md @@ -36,7 +36,7 @@ In the `config/public.yml` file you need to add a link to the xsl scripts for th ``` ## Datura overrides and new files -You will need to create a `file_format.rb` (i.e. `file_ead.rb`) file in `lib/datura/file_types`. Copy from a similar file type (file_tei.rb is a good model for XML=based formats) and make any necessary changes for the file format. In particular the `subdoc_xpaths` should be modified to get the correct XPath for the files you want to transform: +You will need to create a `file_format.rb` (i.e. `file_ead.rb`) file in `lib/datura/file_types`. Copy from a similar file type (file_tei.rb is a good model for XML-based formats) and make any necessary changes for the file format. Make sure to change the class name to reflect the new file format. In particular, in the case of an XML-based format, the `subdoc_xpaths` should be modified to get the correct XPath for the files you want to transform: ``` def subdoc_xpaths # match subdocs against classes @@ -46,7 +46,7 @@ def subdoc_xpaths end ``` -In the `/lib/datura/to_es` folder you also need to make a format_to_es.rb file, i.e. `ead_to_es.rb` and also a folder with fields.rb, request.rb, and xpaths.rb overrides +In the `/lib/datura/to_es` folder you also need to make a format_to_es.rb file, i.e. `ead_to_es.rb` and also a folder with fields.rb, request.rb, and (for XML-based formats) xpaths.rb overrides Be sure to require all the necessary files at the top (and create them in the proper folder). ``` require_relative "xml_to_es.rb" @@ -59,11 +59,11 @@ require_relative "ead_to_es/xpaths.rb" class EadToEs < XmlToEs end ``` -The new files you have added must to be required in `lib/datura/requirer.rb`. Add the following to make sure they get picked up: +The new files you have added must to be required in `lib/datura/requirer.rb`. This should happen automatically, but if not add the following to make sure they get picked up: ``` require_relative "to_es/ead_to_es.rb" ``` -All code in these files should be within the same class, inheriting from XmlToEs. +All code in these files should be within the same class. If the format is based on XML, it should inherit from XmlToEs. ``` class EadToEs < XmlToEs end From d0ba3f0cdf7c8c6eb1182b45fbef00300e163a8c Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 13 Mar 2023 12:35:46 -0500 Subject: [PATCH 135/163] fix comments --- lib/datura/to_es/pdf_to_es.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/to_es/pdf_to_es.rb b/lib/datura/to_es/pdf_to_es.rb index 852981b38..80e86b2c4 100644 --- a/lib/datura/to_es/pdf_to_es.rb +++ b/lib/datura/to_es/pdf_to_es.rb @@ -44,10 +44,10 @@ def get_id end def preprocessing - # copy this in your csv_to_es collection file to customize + # copy this in your pdf_to_es collection file to customize end def postprocessing - # copy this in your csv_to_es collection file to customize + # copy this in your pdf_to_es collection file to customize end end From 13ba8f7a6df40823a815c68b696e00dc24d3e8e2 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 20 Mar 2023 10:59:03 -0500 Subject: [PATCH 136/163] fix html transformation method --- lib/datura/file_types/file_pdf.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/file_types/file_pdf.rb b/lib/datura/file_types/file_pdf.rb index 43b8e9ef7..4dcbc2ad8 100644 --- a/lib/datura/file_types/file_pdf.rb +++ b/lib/datura/file_types/file_pdf.rb @@ -62,7 +62,7 @@ def transform_iiif def transform_html puts "transforming #{self.filename} to HTML subdocuments" - build_html_from_csv + build_html_from_pdf # transform_html method is expected to send back a hash # but already wrote to filesystem so just sending back empty {} From 68842a98304e7b3fa82a1319e4f4fd42fe93d24c Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 20 Mar 2023 11:43:16 -0500 Subject: [PATCH 137/163] clarify that this is not yet implement --- lib/datura/file_types/file_pdf.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/datura/file_types/file_pdf.rb b/lib/datura/file_types/file_pdf.rb index 4dcbc2ad8..426fe12fb 100644 --- a/lib/datura/file_types/file_pdf.rb +++ b/lib/datura/file_types/file_pdf.rb @@ -61,11 +61,11 @@ def transform_iiif end def transform_html - puts "transforming #{self.filename} to HTML subdocuments" - build_html_from_pdf - # transform_html method is expected to send back a hash - # but already wrote to filesystem so just sending back empty - {} + puts "transforming #{self.filename} to HTML subdocuments (not implemented yet)" + # build_html_from_pdf + # # transform_html method is expected to send back a hash + # # but already wrote to filesystem so just sending back empty + # {} end # I am not sure that this is going to be the best way to set this up From 1f612a7efb8a9bb84334646e2d52652d17f6a233 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 13 Jun 2023 15:57:49 -0500 Subject: [PATCH 138/163] add keywords5 to pdftoes fields --- lib/datura/to_es/pdf_to_es/fields.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/datura/to_es/pdf_to_es/fields.rb b/lib/datura/to_es/pdf_to_es/fields.rb index 8b0c4f277..f1a52aed2 100644 --- a/lib/datura/to_es/pdf_to_es/fields.rb +++ b/lib/datura/to_es/pdf_to_es/fields.rb @@ -292,6 +292,9 @@ def keywords3 def keywords4 end + def keywords5 + end + def has_part end From 43a58217a796c02c81757a4aa910c2868c146b99 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 13 Jun 2023 16:13:54 -0500 Subject: [PATCH 139/163] add has_relation to pdftoes fields --- lib/datura/to_es/pdf_to_es/fields.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/datura/to_es/pdf_to_es/fields.rb b/lib/datura/to_es/pdf_to_es/fields.rb index f1a52aed2..f85665238 100644 --- a/lib/datura/to_es/pdf_to_es/fields.rb +++ b/lib/datura/to_es/pdf_to_es/fields.rb @@ -313,4 +313,7 @@ def event def rdf end + def has_relation + end + end From 9fe353c60b1e951c4cf9afb401a29d32bc04442a Mon Sep 17 00:00:00 2001 From: William Dewey Date: Tue, 13 Jun 2023 16:22:30 -0500 Subject: [PATCH 140/163] add has_source to pdftoes fields --- lib/datura/to_es/pdf_to_es/fields.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/datura/to_es/pdf_to_es/fields.rb b/lib/datura/to_es/pdf_to_es/fields.rb index f85665238..25800d8cf 100644 --- a/lib/datura/to_es/pdf_to_es/fields.rb +++ b/lib/datura/to_es/pdf_to_es/fields.rb @@ -316,4 +316,7 @@ def rdf def has_relation end + def has_source + end + end From caed7ba2f1ad90a49743682f9f564ffe3c232a02 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 23 Aug 2023 10:35:25 -0500 Subject: [PATCH 141/163] add xpath for notes related to https://github.com/whitmanarchive/whitman-issues/issues/34 --- lib/datura/to_es/tei_to_es/xpaths.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/datura/to_es/tei_to_es/xpaths.rb b/lib/datura/to_es/tei_to_es/xpaths.rb index e9b9140b6..f5f78f694 100644 --- a/lib/datura/to_es/tei_to_es/xpaths.rb +++ b/lib/datura/to_es/tei_to_es/xpaths.rb @@ -72,6 +72,8 @@ def xpaths_list # "medium" => "", + "notes" => "//note[@type='project']", + # NOTE: if you would like to associate a role you may need the parent element # such as correspAction[@type='deliveredTo'], etc "person" => [ From 91dcfc3026af3c6fa450f17993a42bf2a55dade1 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 23 Aug 2023 13:33:25 -0500 Subject: [PATCH 142/163] add note xpath to text fields related to https://github.com/whitmanarchive/whitman-issues/issues/34 --- lib/datura/to_es/tei_to_es/xpaths.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/to_es/tei_to_es/xpaths.rb b/lib/datura/to_es/tei_to_es/xpaths.rb index f5f78f694..8f4f2605e 100644 --- a/lib/datura/to_es/tei_to_es/xpaths.rb +++ b/lib/datura/to_es/tei_to_es/xpaths.rb @@ -127,7 +127,7 @@ def xpaths_list # NOTE this xpath will often catch notes, back, etc which a project may wish to # exclude if they are using the annotations_text field for editorial comments - "text" => "//text//text()", + "text" => ["//text//text()", "//note[@type='project']"], "title" => "/TEI/teiHeader/fileDesc/titleStmt/title[1]", From 867cad7eb91f65cf0515e31010c720686a447806 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Fri, 25 Aug 2023 10:11:01 -0500 Subject: [PATCH 143/163] truncate pdf text so it doesn't exceed ES limit fixes https://github.com/whitmanarchive/whitman-issues/issues/273 --- lib/datura/to_es/pdf_to_es/fields.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/to_es/pdf_to_es/fields.rb b/lib/datura/to_es/pdf_to_es/fields.rb index 25800d8cf..b8ce90f52 100644 --- a/lib/datura/to_es/pdf_to_es/fields.rb +++ b/lib/datura/to_es/pdf_to_es/fields.rb @@ -187,7 +187,7 @@ def text end text_all += text_additional text_all = text_all.compact - Datura::Helpers.normalize_space(text_all.join(" ")) + Datura::Helpers.normalize_space(text_all.join(" "))[0..999999] end # override and add by collection as needed From 4dfd67bac55acd3dac8c8528339200df30fb2b44 Mon Sep 17 00:00:00 2001 From: Greg Tunink Date: Wed, 30 Aug 2023 16:30:14 -0500 Subject: [PATCH 144/163] Update solr_clear_index to use renamed method in parser code Parser code and es bin files were updated together across a couple commits: - https://github.com/CDRH/datura/commit/b01397a9718ed3bfd3e4b53eac3fe811aed591f2 - https://github.com/CDRH/datura/commit/419a8fd092e32413ebd8cdc88226f2251e922d2c This file should have been updated as well --- bin/solr_clear_index | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/solr_clear_index b/bin/solr_clear_index index 248a709b1..4b0cc078b 100755 --- a/bin/solr_clear_index +++ b/bin/solr_clear_index @@ -2,7 +2,7 @@ require "datura" -params = Datura::Parser.clear_index_params +params = Datura::Parser.clear_index options = Datura::Options.new(params).all url = File.join(options["solr_path"], options["solr_core"], "update") From b680e70cfbc1ba83a89bccd6d501d87ed53490ca Mon Sep 17 00:00:00 2001 From: Greg Tunink Date: Thu, 21 Dec 2023 17:31:31 -0600 Subject: [PATCH 145/163] Output schema in human readable format --- bin/es_get_schema | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/es_get_schema b/bin/es_get_schema index 664988d32..24f173c46 100755 --- a/bin/es_get_schema +++ b/bin/es_get_schema @@ -4,7 +4,7 @@ require "datura" begin es = Datura::Elasticsearch::Index.new - puts es.get_schema + puts JSON.pretty_generate(es.get_schema) rescue => e puts e end From 4a1c478a8e771a891bb80849b4b24ee382ab6702 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 15 Apr 2024 16:08:58 -0500 Subject: [PATCH 146/163] make sure uri_html is blank so html is not loaded in rails adding this fix to base datura fixes https://github.com/whitmanarchive/whitman-issues/issues/598 --- lib/datura/to_es/pdf_to_es/fields.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/datura/to_es/pdf_to_es/fields.rb b/lib/datura/to_es/pdf_to_es/fields.rb index b8ce90f52..8a5269fd4 100644 --- a/lib/datura/to_es/pdf_to_es/fields.rb +++ b/lib/datura/to_es/pdf_to_es/fields.rb @@ -319,4 +319,7 @@ def has_relation def has_source end + def uri_html + end + end From e0bb62e7c340322e78edac0d0942e7f96178821c Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 15 Apr 2024 16:30:35 -0500 Subject: [PATCH 147/163] don't hardcode collection name for ead_to_es_items related to https://github.com/whitmanarchive/whitman-issues/issues/666 --- lib/datura/to_es/ead_to_es_items/fields.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index 6eaf9c718..449b706e8 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -39,7 +39,7 @@ def creator_sort end def collection - "whitman-finding_aid_manuscripts" + "#{@options["collection"]}_items" end def collection_desc From 0bb4c079722effdb644ee193bc1ad01ca7bac8f4 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 5 Jun 2024 16:10:21 -0500 Subject: [PATCH 148/163] fix nested fields in vra that were making the schema not validate --- lib/datura/to_es/vra_to_es/fields.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/datura/to_es/vra_to_es/fields.rb b/lib/datura/to_es/vra_to_es/fields.rb index 0f62b94d0..3c3f6187c 100644 --- a/lib/datura/to_es/vra_to_es/fields.rb +++ b/lib/datura/to_es/vra_to_es/fields.rb @@ -116,16 +116,16 @@ def person # subject element if get_text("@type", xml: p) == "personalName" { - id: nil, - name: get_text(".", xml: p), - role: nil + "id" => nil, + "name" => get_text(".", xml: p), + "role" => nil } # agent element else { - id: nil, - name: get_text("name", xml: p), - role: get_text("role", xml: p) + "id" => nil, + "name" => get_text("name", xml: p), + "role" => get_text("role", xml: p) } end end From 0d393796eb76d5f6e2bb1ce722b43f1f84943f2f Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 22 Jul 2024 14:16:06 -0500 Subject: [PATCH 149/163] add keyword5 for default htmltoes overrides --- lib/datura/to_es/html_to_es/fields.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/to_es/html_to_es/fields.rb b/lib/datura/to_es/html_to_es/fields.rb index 725ac4c4d..261296ca7 100644 --- a/lib/datura/to_es/html_to_es/fields.rb +++ b/lib/datura/to_es/html_to_es/fields.rb @@ -278,7 +278,7 @@ def keywords4 get_text(@xpaths["keywords4"]) end - def keywords4 + def keywords5 get_text(@xpaths["keywords5"]) end From 7ae15837344821392d40f1d953c7a61fb4a74d57 Mon Sep 17 00:00:00 2001 From: wkdewey Date: Wed, 7 Aug 2024 17:06:56 -0500 Subject: [PATCH 150/163] check for nil values when variables depende on nokogiri methods --- lib/datura/to_es/ead_to_es/fields.rb | 8 ++++++-- lib/datura/to_es/ead_to_es_items/fields.rb | 14 +++++++++----- lib/datura/to_es/html_to_es/fields.rb | 4 +++- lib/datura/to_es/tei_to_es/fields.rb | 4 +++- .../to_es/tei_to_es/tei_to_es_personography.rb | 4 +++- lib/datura/to_es/vra_to_es/fields.rb | 8 ++++++-- .../to_es/vra_to_es/vra_to_es_personography.rb | 4 +++- lib/datura/to_es/webs_to_es/fields.rb | 4 +++- 8 files changed, 36 insertions(+), 14 deletions(-) diff --git a/lib/datura/to_es/ead_to_es/fields.rb b/lib/datura/to_es/ead_to_es/fields.rb index dfca070f7..95b89976e 100644 --- a/lib/datura/to_es/ead_to_es/fields.rb +++ b/lib/datura/to_es/ead_to_es/fields.rb @@ -77,7 +77,9 @@ def data_type def date(before=true) datestr = get_text(@xpaths["date"]) - return Datura::Helpers.date_standardize(datestr, before) + if datestr + return Datura::Helpers.date_standardize(datestr, before) + end end def date_display @@ -210,7 +212,9 @@ def text text = [] @xpaths.keys.each do |xpath| body = get_text(@xpaths[xpath]) - text << body + if body + text << body + end end text # TODO: do we need to preserve tags like in text? if so, turn get_text to true diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index 449b706e8..dda00bed1 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -104,9 +104,11 @@ def format def get_id # doc = id doc = get_text(@xpaths["identifier"]) - if doc == "" + if !doc title = get_text(@xpaths["file"]) - return "#{@filename}_#{title}" + if title + return "#{@filename}_#{title}" + end end return "#{@filename}_#{doc}" end @@ -203,8 +205,8 @@ def subjects end def subcategory - subcategory = get_text(@xpaths["subcategory"]) - subcategory.length > 0 ? subcategory : "none" + # subcategory = get_text(@xpaths["subcategory"]) + # subcategory.length > 0 ? subcategory : "none" end def text @@ -212,7 +214,9 @@ def text # means no worrying about handling spacing between words text = [] body = get_text(@xpaths["text"]) - text << body + if body + text << body + end # TODO: do we need to preserve tags like in text? if so, turn get_text to true # text << CommonXml.convert_tags_in_string(body) text += text_additional diff --git a/lib/datura/to_es/html_to_es/fields.rb b/lib/datura/to_es/html_to_es/fields.rb index 261296ca7..925809916 100644 --- a/lib/datura/to_es/html_to_es/fields.rb +++ b/lib/datura/to_es/html_to_es/fields.rb @@ -152,7 +152,9 @@ def text # means no worrying about handling spacing between words text = [] body = get_text(@xpaths["text"]) - text << body + if body + text << body + end text += text_additional Datura::Helpers.normalize_space(text.join(" ")) end diff --git a/lib/datura/to_es/tei_to_es/fields.rb b/lib/datura/to_es/tei_to_es/fields.rb index 9b435cb72..9fe9b7f96 100644 --- a/lib/datura/to_es/tei_to_es/fields.rb +++ b/lib/datura/to_es/tei_to_es/fields.rb @@ -205,7 +205,9 @@ def text # means no worrying about handling spacing between words text_all = [] body = get_text(@xpaths["text"], keep_tags: false, delimiter: '') - text_all << body + if body + text_all << body + end # TODO: do we need to preserve tags like in text? if so, turn get_text to true # text_all << CommonXml.convert_tags_in_string(body) text_all += text_additional diff --git a/lib/datura/to_es/tei_to_es/tei_to_es_personography.rb b/lib/datura/to_es/tei_to_es/tei_to_es_personography.rb index 7e4ff79be..4d3d43de2 100644 --- a/lib/datura/to_es/tei_to_es/tei_to_es_personography.rb +++ b/lib/datura/to_es/tei_to_es/tei_to_es_personography.rb @@ -16,7 +16,9 @@ def category def creator creators = get_list(@xpaths["creators"], false, @parent_xml) - creators.map { |c| { "name" => c } } + if creators + creators.map { |c| { "name" => c } } + end end def creators diff --git a/lib/datura/to_es/vra_to_es/fields.rb b/lib/datura/to_es/vra_to_es/fields.rb index 3c3f6187c..581d18f9c 100644 --- a/lib/datura/to_es/vra_to_es/fields.rb +++ b/lib/datura/to_es/vra_to_es/fields.rb @@ -52,7 +52,9 @@ def data_type def date(before=true) datestr = get_list(@xpaths["date"]).first - Datura::Helpers.date_standardize(datestr, before) + if datestr + Datura::Helpers.date_standardize(datestr, before) + end end def date_display @@ -191,7 +193,9 @@ def text # handling separate fields in array # means no worrying about handling spacing between words text_all = [] - text_all << get_text(@xpaths["text"]) + if get_text(@xpaths["text"]) + text_all << get_text(@xpaths["text"]) + end # TODO: do we need to preserve tags like in text? if so, turn get_text to true # text_all << CommonXml.convert_tags_in_string(body) text_all += text_additional diff --git a/lib/datura/to_es/vra_to_es/vra_to_es_personography.rb b/lib/datura/to_es/vra_to_es/vra_to_es_personography.rb index 8d8f904b1..cab0c5591 100644 --- a/lib/datura/to_es/vra_to_es/vra_to_es_personography.rb +++ b/lib/datura/to_es/vra_to_es/vra_to_es_personography.rb @@ -13,7 +13,9 @@ def category def creator creators = get_list(@xpaths["creators"], xml: @parent_xml) - creators.map { |c| { "name" => c } } + if creators + creators.map { |c| { "name" => c } } + end end def creator_sort diff --git a/lib/datura/to_es/webs_to_es/fields.rb b/lib/datura/to_es/webs_to_es/fields.rb index 90e91d70d..dbccc9f82 100644 --- a/lib/datura/to_es/webs_to_es/fields.rb +++ b/lib/datura/to_es/webs_to_es/fields.rb @@ -162,7 +162,9 @@ def text # means no worrying about handling spacing between words text = [] body = get_text(@xpaths["text"]) - text << body + if body + text << body + end text += text_additional Datura::Helpers.normalize_space(text.join(" ")) end From 5088c141bb71de257f6093548539a256aa32f559 Mon Sep 17 00:00:00 2001 From: wkdewey Date: Thu, 8 Aug 2024 13:54:16 -0500 Subject: [PATCH 151/163] add character limit and truncate text fields --- lib/config/public.yml | 24 +++++++++++----------- lib/datura/to_es/csv_to_es/fields.rb | 2 +- lib/datura/to_es/ead_to_es_items/fields.rb | 2 +- lib/datura/to_es/html_to_es/fields.rb | 2 +- lib/datura/to_es/pdf_to_es/fields.rb | 2 +- lib/datura/to_es/tei_to_es/fields.rb | 2 +- lib/datura/to_es/vra_to_es/fields.rb | 2 +- lib/datura/to_es/webs_to_es/fields.rb | 2 +- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/config/public.yml b/lib/config/public.yml index 11b42716c..7fefc6c85 100644 --- a/lib/config/public.yml +++ b/lib/config/public.yml @@ -9,22 +9,20 @@ # the collection specific configuration files: # (config/public.yml and config/private.yml) - ################### # Defaults # ################### default: - # SCRIPT POWER # recommend this be increased in private.yml # on more powerful systems to improve runtime threads: 5 # LOGGING - log_old_number: 1 # number of log files before beginning to erase - log_size: 32768000 # size of log file in bytes - log_level: Logger::INFO # available levels: UNKNOWN, FATAL, ERROR, WARN, INFO, DEBUG + log_old_number: 1 # number of log files before beginning to erase + log_size: 32768000 # size of log file in bytes + log_level: Logger::INFO # available levels: UNKNOWN, FATAL, ERROR, WARN, INFO, DEBUG # ELASTICSEARCH SCHEMA CONFIGURATION # if es_schema_override is false, datura is base directory @@ -40,14 +38,17 @@ default: api_version: "1.0" # NOTE: es_schema option is set later as combination of above # es_schema_override, es_schema_path, and api_version + # ES currently has a limited character size for keyword fields of 1000000 + # exceeding this limit (generally in text field) will cause errors when searching + text_limit: 900000 # RESOURCE LOCATIONS - data_base: https://cdrhmedia.unl.edu # xml, csv, html snippets, etc - media_base: https://cdrhmedia.unl.edu # images, audio, video - es_index: override_to_set_index # elasticsearch index name - es_path: http://localhost:9200 # elasticsearch path (recommend override) - solr_core: override_to_set_core # solr core name - solr_path: http://localhost:8983/solr # solr path (recommend override) + data_base: https://cdrhmedia.unl.edu # xml, csv, html snippets, etc + media_base: https://cdrhmedia.unl.edu # images, audio, video + es_index: override_to_set_index # elasticsearch index name + es_path: http://localhost:9200 # elasticsearch path (recommend override) + solr_core: override_to_set_core # solr core name + solr_path: http://localhost:8983/solr # solr path (recommend override) # OUTPUT LOCATION # default is [environment]/output/[file_type] @@ -92,7 +93,6 @@ default: development: data_base: https://cdrhdev1.unl.edu/media - ################## # Production # ################## diff --git a/lib/datura/to_es/csv_to_es/fields.rb b/lib/datura/to_es/csv_to_es/fields.rb index 9adc59692..616470280 100644 --- a/lib/datura/to_es/csv_to_es/fields.rb +++ b/lib/datura/to_es/csv_to_es/fields.rb @@ -187,7 +187,7 @@ def text text_all += text_additional text_all = text_all.compact - Datura::Helpers.normalize_space(text_all.join(" ")) + Datura::Helpers.normalize_space(text_all.join(" "))[0..@options["text_limit"]] end # override and add by collection as needed diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index dda00bed1..8a297ca91 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -220,7 +220,7 @@ def text # TODO: do we need to preserve tags like in text? if so, turn get_text to true # text << CommonXml.convert_tags_in_string(body) text += text_additional - return Datura::Helpers.normalize_space(text.join(" ")) + return Datura::Helpers.normalize_space(text.join(" "))[0..@options["text_limit"]] end def text_additional diff --git a/lib/datura/to_es/html_to_es/fields.rb b/lib/datura/to_es/html_to_es/fields.rb index 925809916..babe8adfc 100644 --- a/lib/datura/to_es/html_to_es/fields.rb +++ b/lib/datura/to_es/html_to_es/fields.rb @@ -156,7 +156,7 @@ def text text << body end text += text_additional - Datura::Helpers.normalize_space(text.join(" ")) + Datura::Helpers.normalize_space(text.join(" "))[0..@options["text_limit"]] end def text_additional diff --git a/lib/datura/to_es/pdf_to_es/fields.rb b/lib/datura/to_es/pdf_to_es/fields.rb index 8a5269fd4..1c3bfe4f2 100644 --- a/lib/datura/to_es/pdf_to_es/fields.rb +++ b/lib/datura/to_es/pdf_to_es/fields.rb @@ -187,7 +187,7 @@ def text end text_all += text_additional text_all = text_all.compact - Datura::Helpers.normalize_space(text_all.join(" "))[0..999999] + Datura::Helpers.normalize_space(text_all.join(" "))[0..@options["text_limit"]] end # override and add by collection as needed diff --git a/lib/datura/to_es/tei_to_es/fields.rb b/lib/datura/to_es/tei_to_es/fields.rb index 9fe9b7f96..960fd1929 100644 --- a/lib/datura/to_es/tei_to_es/fields.rb +++ b/lib/datura/to_es/tei_to_es/fields.rb @@ -211,7 +211,7 @@ def text # TODO: do we need to preserve tags like in text? if so, turn get_text to true # text_all << CommonXml.convert_tags_in_string(body) text_all += text_additional - Datura::Helpers.normalize_space(text_all.join(" ")) + Datura::Helpers.normalize_space(text_all.join(" "))[0..@options["text_limit"]] end def text_additional diff --git a/lib/datura/to_es/vra_to_es/fields.rb b/lib/datura/to_es/vra_to_es/fields.rb index 581d18f9c..bdd3c4fd3 100644 --- a/lib/datura/to_es/vra_to_es/fields.rb +++ b/lib/datura/to_es/vra_to_es/fields.rb @@ -199,7 +199,7 @@ def text # TODO: do we need to preserve tags like in text? if so, turn get_text to true # text_all << CommonXml.convert_tags_in_string(body) text_all += text_additional - Datura::Helpers.normalize_space(text_all.join(" ")) + Datura::Helpers.normalize_space(text_all.join(" "))[0..@options["text_limit"]] end def text_additional diff --git a/lib/datura/to_es/webs_to_es/fields.rb b/lib/datura/to_es/webs_to_es/fields.rb index dbccc9f82..8721e4462 100644 --- a/lib/datura/to_es/webs_to_es/fields.rb +++ b/lib/datura/to_es/webs_to_es/fields.rb @@ -166,7 +166,7 @@ def text text << body end text += text_additional - Datura::Helpers.normalize_space(text.join(" ")) + Datura::Helpers.normalize_space(text.join(" "))[0..@options["text_limit"]] end def text_additional From 10e45d0ee91bf1e63fd7178e5c7eb04fc332e4be Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 21 Aug 2024 16:11:06 -0500 Subject: [PATCH 152/163] update changelog --- CHANGELOG.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8861fe0f5..c760f99b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,12 +34,12 @@ Versioning](https://semver.org/spec/v2.0.0.html). - documentation for adding new ingest formats to Datura - byebug gem for debugging - instructions for installing Javascript Runtime files for Saxon -- API schema can either be 1.0 or 2.0 (which includes nested fields); 1.0 will be run by default unless 2.0 is specified. Add the following to `public.yml` or `private.yml` in the data repo: +- API schema can either be the original 1.0 or the newly updated 2.0 (which includes new fields including nested fields); 1.0 will be run by default unless 2.0 is specified. Add the following to `public.yml` or `private.yml` in the data repo: ``` api_version: '2.0' ``` See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/schema_v2.md) -- schema validation with API version 2.0, invalidly constructed documents will not post +- schema validation with API version 2.0: invalidly constructed documents will not post - authentication with Elasticesarch 8.5; add the following to `public.yml` or `private.yml` in the data repo: ``` es_user: username @@ -47,24 +47,32 @@ See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/sc ``` - field overrides for new fields in the new API schema - functionality to transform EAD files and post them to elasticsearch +- functionality to transform PDF files (including text and metadata) and post them to elasticsearch +- limiting `text` field to a specific limit: `text_field` in `public.yml` or `private.yml` +- configuration options related to Elasticsearch, including `text_limit` and `es_schema_override` and `es_schema_path` to change the location of the Elasticsearch schema +- more detailed errors including a stack trace ### Changed - update ruby to 3.1.2 - date_standardize now relies on strftime instead of manual zero padding for month, day - minor corrections to documentation - XPath: "text" is now ingested as an array and will be displayed delimitted by spaces +- "text" field now includes "notes" XPath +- refactored posting script (`Datura.run`) - refactored command line methods into elasticsearch library - refactored and moved date_standardize and date_display helper methods -- Nokogiri methods `get_text` and `get_list` on TEI now return nil rather than empty strings or arrays if there are no matches +- Nokogiri methods `get_text` and `get_list` on TEI now return nil rather than empty strings or arrays if there are no matches. fields have been changed to check for these nil values ### Migration - check to make sure "text" xpath is doing desired behavior - use Elasticsearch 8.5 or higher and add authentication as described above if security is enabled. See [dev docs instructions](https://github.com/CDRH/cdrh_dev_docs/blob/update_elasticsearch_documentation/publishing/2_basic_requirements.md#downloading-elasticsearch). - upgrade data repos to Ruby 3.1.2 +- - add api version to config as described above - make sure fields are consistent with the api schema, many have been renamed or changed in format -- add nil checks with get_text and get_list methods +- add nil checks with get_text and get_list methods as needed - add EadToES overrides if ingesting EAD files +- add `byebug` and `pdf-reader` to Gemfile in repos based on Datura - if overriding the `read_csv` method in `lib/datura/file_type.rb`, the hash must be prefixed with ** (`**{}`). ## [v0.2.0-beta](https://github.com/CDRH/datura/compare/v0.1.6...v0.2.0-beta) - 2020-08-17 - Altering field and xpath behavior, adds get_elements From 4fcb329cfaabb69198497755c403d6f47667b7d1 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 21 Aug 2024 16:18:26 -0500 Subject: [PATCH 153/163] add ead to list of possible formats --- lib/datura/parser_options/post.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/parser_options/post.rb b/lib/datura/parser_options/post.rb index c3dd1218d..b6e48880f 100644 --- a/lib/datura/parser_options/post.rb +++ b/lib/datura/parser_options/post.rb @@ -22,12 +22,12 @@ def self.post_params # default to no restricted format options["format"] = nil - opts.on( '-f', '--format [input]', 'Supported formats (csv, html, pdf, tei, vra, webs)') do |input| + opts.on( '-f', '--format [input]', 'Supported formats (csv, html, ead, pdf, tei, vra, webs)') do |input| if %w[authority annotations].include?(input) puts "'authority' and 'annotations' are invalid formats".red puts "Please select a supported format or rename your custom format" exit - elsif !%w[csv html pdf tei vra webs].include?(input) + elsif !%w[csv ead html pdf tei vra webs].include?(input) puts "Caution: Requested custom format #{input}.".red puts "See FileCustom class for implementation instructions" end From 22b2d3271e04d27d759f80d413a9d0d5b0419387 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 28 Aug 2024 09:40:46 -0500 Subject: [PATCH 154/163] fix incorrect field label and remove redundancy --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c760f99b7..81b874be9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,8 +48,8 @@ See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/sc - field overrides for new fields in the new API schema - functionality to transform EAD files and post them to elasticsearch - functionality to transform PDF files (including text and metadata) and post them to elasticsearch -- limiting `text` field to a specific limit: `text_field` in `public.yml` or `private.yml` -- configuration options related to Elasticsearch, including `text_limit` and `es_schema_override` and `es_schema_path` to change the location of the Elasticsearch schema +- limiting `text` field to a specific limit: `text_limit` in `public.yml` or `private.yml` +- configuration options related to Elasticsearch, including `es_schema_override` and `es_schema_path` to change the location of the Elasticsearch schema - more detailed errors including a stack trace ### Changed From 40393d8d4de8243a4876ed751d4b9d6c90844144 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Wed, 28 Aug 2024 09:41:07 -0500 Subject: [PATCH 155/163] revert accidental commenting of subcategory override --- lib/datura/to_es/ead_to_es_items/fields.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/to_es/ead_to_es_items/fields.rb b/lib/datura/to_es/ead_to_es_items/fields.rb index 8a297ca91..7764e083b 100644 --- a/lib/datura/to_es/ead_to_es_items/fields.rb +++ b/lib/datura/to_es/ead_to_es_items/fields.rb @@ -205,8 +205,8 @@ def subjects end def subcategory - # subcategory = get_text(@xpaths["subcategory"]) - # subcategory.length > 0 ? subcategory : "none" + subcategory = get_text(@xpaths["subcategory"]) + subcategory.length > 0 ? subcategory : "none" end def text From dd21a6deffdf5656628016049bbf6609ede3830d Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 16 Sep 2024 14:16:52 -0500 Subject: [PATCH 156/163] change version back to unreleased --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81b874be9..753e49cb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ Versioning](https://semver.org/spec/v2.0.0.html). ### Security --> -## [1.0.0](https://github.com/CDRH/datura/compare/v0.2.0-beta...dev) +## [Unreleased](https://github.com/CDRH/datura/compare/v0.2.0-beta...dev) ### Added - minor test for Datura::Helpers.date_standardize From 8ba660cc3a59b05d18a35993b16f0312931a7e3c Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 16 Sep 2024 14:22:34 -0500 Subject: [PATCH 157/163] replace dead link and link to private repo --- CHANGELOG.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 753e49cb2..06a9c1bdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,13 +38,9 @@ Versioning](https://semver.org/spec/v2.0.0.html). ``` api_version: '2.0' ``` -See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/schema_v2.md) +See new schema (2.0) documentation [here](https://github.com/CDRH/datura/blob/main/docs/schema_v2.md) - schema validation with API version 2.0: invalidly constructed documents will not post -- authentication with Elasticesarch 8.5; add the following to `public.yml` or `private.yml` in the data repo: -``` - es_user: username - es_password: ******** -``` +- authentication with Elasticesarch 8.5 - field overrides for new fields in the new API schema - functionality to transform EAD files and post them to elasticsearch - functionality to transform PDF files (including text and metadata) and post them to elasticsearch @@ -65,7 +61,11 @@ See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/sc ### Migration - check to make sure "text" xpath is doing desired behavior -- use Elasticsearch 8.5 or higher and add authentication as described above if security is enabled. See [dev docs instructions](https://github.com/CDRH/cdrh_dev_docs/blob/update_elasticsearch_documentation/publishing/2_basic_requirements.md#downloading-elasticsearch). +- use Elasticsearch 8.5 or higher and add authentication as described above if security is enabled. Add the following to `public.yml` or `private.yml` in the data repo: +``` + es_user: username + es_password: ******** +``` - upgrade data repos to Ruby 3.1.2 - - add api version to config as described above From 20799c51923e135570cf504aebec4269d79bc4f0 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 16 Sep 2024 14:29:15 -0500 Subject: [PATCH 158/163] add pdf-reader to gemspec --- datura.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/datura.gemspec b/datura.gemspec index 3d6f91b56..01d38881b 100644 --- a/datura.gemspec +++ b/datura.gemspec @@ -57,6 +57,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "colorize", "~> 0.8.1" spec.add_runtime_dependency "nokogiri", "~> 1.10" spec.add_runtime_dependency "rest-client", "~> 2.1" + spec.add_runtime_dependency "pdf-reader", "~> 2.12" spec.add_development_dependency "bundler", ">= 1.16.0", "< 3.0" spec.add_development_dependency "minitest", "~> 5.0" spec.add_development_dependency "rake", "~> 13.0" From 544b59d9e7f8d5b967fae6df2ee9bffaa8e97356 Mon Sep 17 00:00:00 2001 From: wkdewey Date: Wed, 18 Sep 2024 14:20:56 -0500 Subject: [PATCH 159/163] change link from private docs to ES site --- docs/4_developers/installation.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/4_developers/installation.md b/docs/4_developers/installation.md index 0e0171bda..1c486b6db 100644 --- a/docs/4_developers/installation.md +++ b/docs/4_developers/installation.md @@ -6,11 +6,12 @@ TODO ### Elasticsearch -See installation instructions [here](https://github.com/CDRH/cdrh_dev_docs/blob/update_elasticsearch_documentation/publishing/2_basic_requirements.md#downloading-elasticsearch). +Download Elasticsearch 8 [here](https://www.elastic.co/downloads/elasticsearch). ### Apache Permissions -Assuming that you place this collection in your web tree, you will need to take care to protect any sensitive information you place it in that you do not want to be accessible through the browser (copywritten material, drafts, passwords, etc). To protect the configuration files that contain information about your solr instance, you should put these lines in your main apache configuration file. If you have an older version of Apache, you may need to specify `Order deny,allow` and `Deny from all` instead of using `Require all denied`. +Assuming that you place this collection in your web tree, you will need to take care to protect any sensitive information you place it in that you do not want to be accessible through the browser (copywritten material, drafts, passwords, etc). To protect the configuration files that contain information about your solr instance, you should put these lines in your main apache configuration file. If you have an older version of Apache, you may need to specify `Order deny,allow` and `Deny from all` instead of using `Require all denied`. + ``` # # Prevent private.yml files that might be in the webtree from being viewable @@ -19,4 +20,5 @@ Assuming that you place this collection in your web tree, you will need to take Require all denied ``` + Otherwise, you should not need to do anything with apache assuming that you already had it set up with a webtree. From 18e1aad40f1ddec3b58c22b9babb9af46f873260 Mon Sep 17 00:00:00 2001 From: wkdewey Date: Wed, 18 Sep 2024 15:43:47 -0500 Subject: [PATCH 160/163] remove file with redundant methods --- lib/datura/elasticsearch/data.rb | 94 -------------------------------- 1 file changed, 94 deletions(-) delete mode 100644 lib/datura/elasticsearch/data.rb diff --git a/lib/datura/elasticsearch/data.rb b/lib/datura/elasticsearch/data.rb deleted file mode 100644 index 4af171fce..000000000 --- a/lib/datura/elasticsearch/data.rb +++ /dev/null @@ -1,94 +0,0 @@ -require "json" -require "rest-client" - -require_relative "./../elasticsearch.rb" - -module Datura::Elasticsearch::Data - - def self.clear - # run the parameters through the option parser - params = Datura::Parser.clear_index_params - options = Datura::Options.new(params).all - if options["collection"] == "all" - self.clear_all(options) - else - self.clear_index(options) - end - end - - private - - def self.build_clear_data(options) - if options["regex"] - field = options["field"] || "identifier" - { - "query" => { - "bool" => { - "must" => [ - { "regexp" => { field => options["regex"] } }, - { "term" => { "collection" => options["collection"] } } - ] - } - } - } - else - { - "query" => { "term" => { "collection" => options["collection"] } } - } - end - end - - def self.clear_all(options) - puts "Please verify that you want to clear EVERY ENTRY from the ENTIRE INDEX\n\n" - puts "== FIELD / REGEX FILTERS NOT AVAILABLE FOR THIS OPTION, YOU'LL WIPE EVERYTHING ==\n\n" - puts "Running this on something other than your computer's localhost? DON'T." - puts "Type: 'Yes I'm sure'" - confirm = STDIN.gets.chomp - if confirm == "Yes I'm sure" - url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") - json = { "query" => { "match_all" => {} } } - RestClient.post(url, json.to_json, @auth_header.merge({ content_type: :json })) { |res, req, result| - if result.code == "200" - puts res - else - raise "#{result.code} error when clearing entire index: #{res}" - end - } - else - puts "You typed '#{confirm}'. This is incorrect, exiting program" - exit - end - end - - def self.clear_index(options) - url = File.join(options["es_path"], options["es_index"], "_doc", "_delete_by_query?pretty=true") - confirmation = self.confirm_clear(options, url) - - if confirmation - data = self.build_clear_data(options) - RestClient.post(url, data.to_json, @auth_header.merge({ content_type: :json })) { |res, req, result| - if result.code == "200" - puts res - else - raise "#{result.code} error when clearing index: #{res}" - end - } - else - puts "come back anytime!" - exit - end - end - - def self.confirm_clear(options, url) - # verify that the user is really sure about the index they're about to wipe - puts "Are you sure that you want to remove entries from" - puts " #{options["collection"]}'s #{options['environment']} environment?" - puts "url: #{url}" - puts "y/N" - answer = STDIN.gets.chomp - # boolean - !!(answer =~ /[yY]/) - end - - -end From 2c9b4b49a39d9214cab7e3b8b3d24eeb7ed6a491 Mon Sep 17 00:00:00 2001 From: wkdewey Date: Wed, 18 Sep 2024 15:52:40 -0500 Subject: [PATCH 161/163] remove uri_html definition which is blanked later --- lib/datura/to_es/pdf_to_es/fields.rb | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib/datura/to_es/pdf_to_es/fields.rb b/lib/datura/to_es/pdf_to_es/fields.rb index 1c3bfe4f2..ef0848938 100644 --- a/lib/datura/to_es/pdf_to_es/fields.rb +++ b/lib/datura/to_es/pdf_to_es/fields.rb @@ -231,18 +231,6 @@ def uri_data ) end - def uri_html - File.join( - @options["data_base"], - "data", - @options["collection"], - "output", - @options["environment"], - "html", - "#{@id}.html" - ) - end - def works # if @row["works"] # @row["works"].split("; ") From 19e1bfe562dce2dc270b0a9572bac58b76daa190 Mon Sep 17 00:00:00 2001 From: wkdewey Date: Thu, 19 Sep 2024 11:27:01 -0500 Subject: [PATCH 162/163] move byebug into gemspec, don't restrict to dev only --- Gemfile | 1 - datura.gemspec | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index afaf20dc0..88e0e14c5 100644 --- a/Gemfile +++ b/Gemfile @@ -2,6 +2,5 @@ source "https://rubygems.org" git_source(:github) {|repo_name| "https://github.com/#{repo_name}" } -gem "byebug" # Specify your gem's dependencies in datura.gemspec gemspec diff --git a/datura.gemspec b/datura.gemspec index 01d38881b..eaeccddd6 100644 --- a/datura.gemspec +++ b/datura.gemspec @@ -58,8 +58,8 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "nokogiri", "~> 1.10" spec.add_runtime_dependency "rest-client", "~> 2.1" spec.add_runtime_dependency "pdf-reader", "~> 2.12" + spec.add_runtime_dependency "byebug", "~> 11.0" spec.add_development_dependency "bundler", ">= 1.16.0", "< 3.0" spec.add_development_dependency "minitest", "~> 5.0" spec.add_development_dependency "rake", "~> 13.0" - spec.add_development_dependency "byebug", "~> 11.0" end From d0bac716c97b708518505ebfa64a7339d2aa2e59 Mon Sep 17 00:00:00 2001 From: William Dewey Date: Mon, 23 Sep 2024 11:20:55 -0500 Subject: [PATCH 163/163] remove reference to info that was moved --- CHANGELOG.md | 2 +- Gemfile.lock | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06a9c1bdc..874b4fe28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,7 @@ See new schema (2.0) documentation [here](https://github.com/CDRH/datura/blob/ma ### Migration - check to make sure "text" xpath is doing desired behavior -- use Elasticsearch 8.5 or higher and add authentication as described above if security is enabled. Add the following to `public.yml` or `private.yml` in the data repo: +- use Elasticsearch 8.5 or higher and add authentication if security is enabled. Add the following to `public.yml` or `private.yml` in the data repo: ``` es_user: username es_password: ******** diff --git a/Gemfile.lock b/Gemfile.lock index fd39ff559..3ad9b3074 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,17 +2,23 @@ PATH remote: . specs: datura (0.2.0.pre.beta) + byebug (~> 11.0) colorize (~> 0.8.1) nokogiri (~> 1.10) + pdf-reader (~> 2.12) rest-client (~> 2.1) GEM remote: https://rubygems.org/ specs: + Ascii85 (1.1.1) + afm (0.2.2) + bigdecimal (3.1.8) byebug (11.1.3) colorize (0.8.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) + hashery (2.1.2) http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) @@ -27,6 +33,12 @@ GEM racc (~> 1.4) nokogiri (1.13.9-x86_64-darwin) racc (~> 1.4) + pdf-reader (2.12.0) + Ascii85 (~> 1.0) + afm (~> 0.2.1) + hashery (~> 2.0) + ruby-rc4 + ttfunk racc (1.6.0) rake (13.0.6) rest-client (2.1.0) @@ -34,6 +46,9 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) + ruby-rc4 (0.1.5) + ttfunk (1.8.0) + bigdecimal (~> 3.1) unf (0.1.4) unf_ext unf_ext (0.0.8.2) @@ -44,7 +59,6 @@ PLATFORMS DEPENDENCIES bundler (>= 1.16.0, < 3.0) - byebug datura! minitest (~> 5.0) rake (~> 13.0)