From a4e4fbd85c4c96b30b6b6e82cfc9ffc6a1647fdf Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Tue, 14 May 2024 21:16:07 -0400 Subject: [PATCH 1/9] Add default_single_authority_plain_last_versions value for hku --- lib/cspace_config_untangler.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cspace_config_untangler.rb b/lib/cspace_config_untangler.rb index 35cbd4ad..24116cd9 100644 --- a/lib/cspace_config_untangler.rb +++ b/lib/cspace_config_untangler.rb @@ -61,7 +61,7 @@ module CspaceConfigUntangler "fcart" => "6-0-0", "herbarium" => "2-0-9", "lhmc" => "6-0-0", - "hku" => "", + "hku" => "1-0-0", "materials" => "3-2-0", "ohc" => "1-0-19", "omca" => "1-0-0-rc6", From a56c866fe63b39c62f0235bdd3ed500f5ecb170e Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Wed, 15 May 2024 10:45:07 -0400 Subject: [PATCH 2/9] Move commands from debug to forms CLI --- doc/ui_config_notes.adoc | 6 +- lib/cspace_config_untangler/cli/debug_cli.rb | 144 ------------------- lib/cspace_config_untangler/cli/forms_cli.rb | 141 ++++++++++++++++++ 3 files changed, 144 insertions(+), 147 deletions(-) diff --git a/doc/ui_config_notes.adoc b/doc/ui_config_notes.adoc index 2694476a..6c88b5ef 100644 --- a/doc/ui_config_notes.adoc +++ b/doc/ui_config_notes.adoc @@ -72,7 +72,7 @@ The key signatures and counts shown above can be generated by running: [source,shell] ---- -ccu debug form_props_sig -p all +ccu forms props_key_sigs -p all ---- The same command can be used to view examples for a given key signature, @@ -80,10 +80,10 @@ as shown in the examples below. [source,shell] ---- -ccu debug form_props_sig -p all -k _owner key props ref -e true +ccu forms props_key_sigs -p all -k _owner key props ref -e true ---- -Do `+ccu debug help form_props_sig+` for details on command usage. +Do `+ccu forms help props_key_sigs+` for details on command usage. ==== _owner, key, props, ref diff --git a/lib/cspace_config_untangler/cli/debug_cli.rb b/lib/cspace_config_untangler/cli/debug_cli.rb index 109594cd..1df8fa6a 100644 --- a/lib/cspace_config_untangler/cli/debug_cli.rb +++ b/lib/cspace_config_untangler/cli/debug_cli.rb @@ -7,150 +7,6 @@ module Cli class DebugCli < Thor include CCU::Cli::Helpers - desc "form_subpaths", - "Get all form props subpath values used" - option :rectype, - desc: "Comma separated list (no spaces) of record types to include. "\ - "Defaults to all.", - default: "all", - aliases: "-r" - option :mode, - desc: "Whether to output just list of subpath values, or to "\ - "include occurrence counts", - enum: %w[value count code], - default: "value", - type: :string, - aliases: "-m" - def form_subpaths - props = get_profiles.map do |profile| - p = CCU::Profile.new(profile:) - p.rectypes - .map(&:forms) - .flatten - .map(&:values) - .flatten - .reject(&:disabled?) - .map { |form| form.send(:iterator).send(:allprops) } - .flatten - end.flatten - .select { |prop| prop.config.key?("subpath") } - - case options[:mode] - when "value" - props.map { |prop| prop.config["subpath"] } - .uniq - .map { |val| val.is_a?(Array) ? val.inspect : val } - .sort - .each { |val| puts val } - when "count" - props.group_by { |prop| prop.config["subpath"] } - .map { |subpath, arr| [subpath, arr.length] } - .sort_by { |subarr| subarr[1] } - .reverse - .to_h - .each { |val, ct| puts "#{ct}\t#{val}" } - when "code" - result = props.map { |prop| prop.config["subpath"] } - .uniq - .sort_by { |val| val.is_a?(Array) ? val.inspect : val } - puts result.inspect - end - end - - desc "form_props_types", - "Get all values of props/type used" - option :rectype, - desc: "Comma separated list (no spaces) of record types to include. "\ - "Defaults to all.", - default: "all", - aliases: "-r" - def form_props_types - props = get_profiles.map do |profile| - p = CCU::Profile.new(profile:) - p.rectypes - .map(&:forms) - .flatten - .map(&:values) - .flatten - .reject(&:disabled?) - .map { |form| form.send(:iterator).send(:allprops) } - .flatten - end.flatten - .select { |prop| prop.config.key?("type") } - - result = props.group_by { |prop| prop.config["type"] } - .map { |subpath, arr| [subpath, arr.length] } - .sort_by { |subarr| subarr[1] } - .reverse - .to_h - - result.each { |val, ct| puts "#{ct}\t#{val}" } - end - - desc "form_props_sigs", - "Get all used props key signature patterns" - option :keys, desc: "Key signature to find", type: :array, - required: false, aliases: "-k" - option :selector, desc: "CCU::Forms::Props method used to "\ - "select specific props.", aliases: "-s", type: :string, required: false - option :rejector, desc: "CCU::Forms::Props method used to "\ - "reject specific props.", aliases: "-r", type: :string, required: false - option :count, desc: "Whether to print count of props following pattern", - aliases: "-c", type: :boolean, default: true, required: false - option :example, desc: "Whether to print examples", aliases: "-e", - type: :boolean, default: false, required: false - option :maxex, desc: "Maximum number of examples to print", aliases: "-m", - type: :numeric, default: 10, required: false - option :rectype, - desc: "Comma separated list (no spaces) of record types to include. "\ - "Defaults to all.", - default: "all", - aliases: "-r" - def form_props_sigs - all = get_profiles.map do |profile| - p = CCU::Profile.new(profile:) - p.rectypes - .map(&:forms) - .flatten - .map(&:values) - .flatten - .reject(&:disabled?) - .map { |form| form.send(:iterator).send(:allprops) } - .flatten - end.flatten - - by_sig = if options[:keys] - all.select { |p| p.keys == options[:keys].sort } - else - all - end - selected = if options[:selector] - meth = options[:selector].to_sym - by_sig.select { |p| p.send(meth) } - else - by_sig - end - rejected = if options[:rejector] - selected.reject { |p| p.send(options[:rejector].to_sym) } - else - selected - end - grouped = rejected.group_by { |prop| prop.keys } - - grouped.sort_by { |keys, props| props.length } - .reverse_each do |keys, props| - puts "#{props.length}\t#{keys.inspect}" if options[:count] - if options[:example] - props.first(options[:maxex]).each do |prop| - puts prop - puts prop.config - puts "\n" - end - puts "\n\n" - end - end - end - desc "check_xpath_depth", "Reports fields with unusual xpath depth (i.e. not 0, 1, 2, 3, or 4)" option :rectype, diff --git a/lib/cspace_config_untangler/cli/forms_cli.rb b/lib/cspace_config_untangler/cli/forms_cli.rb index b15cd754..1e3390a2 100644 --- a/lib/cspace_config_untangler/cli/forms_cli.rb +++ b/lib/cspace_config_untangler/cli/forms_cli.rb @@ -23,6 +23,147 @@ def disabled .select { |form| form.disabled? } .each { |form| puts form.id } end + + desc "subpaths", "Get all form props subpath values used" + option :rectype, + desc: "Comma separated list (no spaces) of record types to include. "\ + "Defaults to all.", + default: "all", + aliases: "-r" + option :mode, + desc: "Whether to output just list of subpath values, or to "\ + "include occurrence counts", + enum: %w[value count code], + default: "value", + type: :string, + aliases: "-m" + def subpaths + props = get_profiles.map do |profile| + p = CCU::Profile.new(profile:) + p.rectypes + .map(&:forms) + .flatten + .map(&:values) + .flatten + .reject(&:disabled?) + .map { |form| form.send(:iterator).send(:allprops) } + .flatten + end.flatten + .select { |prop| prop.config.key?("subpath") } + + case options[:mode] + when "value" + props.map { |prop| prop.config["subpath"] } + .uniq + .map { |val| val.is_a?(Array) ? val.inspect : val } + .sort + .each { |val| puts val } + when "count" + props.group_by { |prop| prop.config["subpath"] } + .map { |subpath, arr| [subpath, arr.length] } + .sort_by { |subarr| subarr[1] } + .reverse + .to_h + .each { |val, ct| puts "#{ct}\t#{val}" } + when "code" + result = props.map { |prop| prop.config["subpath"] } + .uniq + .sort_by { |val| val.is_a?(Array) ? val.inspect : val } + puts result.inspect + end + end + + desc "props_types", "Get all values of props/type used" + option :rectype, + desc: "Comma separated list (no spaces) of record types to include. "\ + "Defaults to all.", + default: "all", + aliases: "-r" + def props_types + props = get_profiles.map do |profile| + p = CCU::Profile.new(profile:) + p.rectypes + .map(&:forms) + .flatten + .map(&:values) + .flatten + .reject(&:disabled?) + .map { |form| form.send(:iterator).send(:allprops) } + .flatten + end.flatten + .select { |prop| prop.config.key?("type") } + + result = props.group_by { |prop| prop.config["type"] } + .map { |subpath, arr| [subpath, arr.length] } + .sort_by { |subarr| subarr[1] } + .reverse + .to_h + + result.each { |val, ct| puts "#{ct}\t#{val}" } + end + + desc "props_key_sigs", "Get all used props key signature patterns" + option :keys, desc: "Key signature to find", type: :array, + required: false, aliases: "-k" + option :selector, desc: "CCU::Forms::Props method used to "\ + "select specific props.", aliases: "-s", type: :string, required: false + option :rejector, desc: "CCU::Forms::Props method used to "\ + "reject specific props.", aliases: "-r", type: :string, required: false + option :count, desc: "Whether to print count of props following pattern", + aliases: "-c", type: :boolean, default: true, required: false + option :example, desc: "Whether to print examples", aliases: "-e", + type: :boolean, default: false, required: false + option :maxex, desc: "Maximum number of examples to print", aliases: "-m", + type: :numeric, default: 10, required: false + option :rectype, + desc: "Comma separated list (no spaces) of record types to include. "\ + "Defaults to all.", + default: "all", + aliases: "-r" + def props_key_sigs + all = get_profiles.map do |profile| + p = CCU::Profile.new(profile:) + p.rectypes + .map(&:forms) + .flatten + .map(&:values) + .flatten + .reject(&:disabled?) + .map { |form| form.send(:iterator).send(:allprops) } + .flatten + end.flatten + + by_sig = if options[:keys] + all.select { |p| p.keys == options[:keys].sort } + else + all + end + selected = if options[:selector] + meth = options[:selector].to_sym + by_sig.select { |p| p.send(meth) } + else + by_sig + end + rejected = if options[:rejector] + selected.reject { |p| p.send(options[:rejector].to_sym) } + else + selected + end + grouped = rejected.group_by { |prop| prop.keys } + + grouped.sort_by { |keys, props| props.length } + .reverse_each do |keys, props| + puts "#{props.length}\t#{keys.inspect}" if options[:count] + if options[:example] + props.first(options[:maxex]).each do |prop| + puts prop + puts prop.config + puts "\n" + end + puts "\n\n" + end + end + end end end end From a0a896abbab44d76266b0873842518f0ae129046 Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Wed, 15 May 2024 10:55:02 -0400 Subject: [PATCH 3/9] Update CHANGELOG for v3.0.0 --- CHANGELOG.adoc | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 34b88714..3f04cfa3 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -39,6 +39,32 @@ These changes are merged into the `main` branch, but have not been released. Aft == Releases +=== 3.0.0 - 2024-05-14 +==== Breaking +* Fields controlled by a single authority vocabulary no longer have simplified header/column names. `default_single_authority_plain_last_versions` has been added in `lib/cspace_config_untangler.rb` and records the last UI config version of known/directly supported by Lyrasis profiles that will have simplified header/column names. This may need to be updated if you are using the Untangler for other profiles. +** The main thing to be concerned with here is that the CSV templates you use are generated with the same header/column names as the mappers you use in the CSV Importer or other tooling + +==== Added + +* Community-supported profile configs, mappers, mapper manifest, CSV templates, and reference documents for 8.0 release +* Handling of new field definition override pattern in form definition (show only one field from repeatable field group, and make it non-repeatable, materials_4-0-0 objectCount field) +* CLI commands +** forms subpaths +** forms props_type +** forms props_key_sigs + +==== Changed + +* Upgrade-related warnings now generated by a class that emits each warning only once per run +* Log more info/oddness + +==== Dev/Internal +* Implement almost-standard for code formatting/style/linting +* Add Github workflow to run tests and lint on PRs +* Remove code obviated by fixes in CollectionSpace 8.0 +* Major refactor of code that extracts field data from forms +* Update dependencies; remove unused dependencies + === 2.0.1 - 2023-10-03 ==== Bugfixes From 46b922a64f977cb8044e9d8c91ef34ca29dfef51 Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Wed, 15 May 2024 10:56:02 -0400 Subject: [PATCH 4/9] Bump version to 3.0.0 --- Gemfile.lock | 4 +--- lib/cspace_config_untangler/version.rb | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 88c13be2..4455582d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -20,7 +20,7 @@ GIT PATH remote: . specs: - cspace_config_untangler (2.0.2) + cspace_config_untangler (3.0.0) dry-configurable facets nokogiri @@ -46,8 +46,6 @@ GEM method_source (1.0.0) nokogiri (1.16.5-x86_64-darwin) racc (~> 1.4) - nokogiri (1.16.5-x86_64-linux) - racc (~> 1.4) parallel (1.24.0) parser (3.3.1.0) ast (~> 2.4.1) diff --git a/lib/cspace_config_untangler/version.rb b/lib/cspace_config_untangler/version.rb index bff2d57d..31fbede1 100644 --- a/lib/cspace_config_untangler/version.rb +++ b/lib/cspace_config_untangler/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module CspaceConfigUntangler - VERSION = "2.0.2" + VERSION = "3.0.0" end From c7513e832490d41e40de2f2a485ed63a4411d70b Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Wed, 15 May 2024 11:17:01 -0400 Subject: [PATCH 5/9] Add ruby to Gemfile.lock platforms --- Gemfile.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Gemfile.lock b/Gemfile.lock index 4455582d..228b3ca9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -116,6 +116,7 @@ GEM zeitwerk (2.6.14) PLATFORMS + ruby x86_64-darwin-20 x86_64-linux From 72f0cf1a7755dcb859c75d3cd12ad45d3374156a Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Wed, 15 May 2024 11:22:09 -0400 Subject: [PATCH 6/9] Keep ruby as only platform in Gemfile.lock --- Gemfile.lock | 2 -- 1 file changed, 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 228b3ca9..c53c3e52 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -117,8 +117,6 @@ GEM PLATFORMS ruby - x86_64-darwin-20 - x86_64-linux DEPENDENCIES almost_standard! From 6fbbb9294f76a44605dff29b6f4e0937c25d032c Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Wed, 15 May 2024 11:26:04 -0400 Subject: [PATCH 7/9] Do not run CI in deployment mode --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b62ec21..49e29b95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,9 @@ jobs: bundler-cache: true ruby-version: ${{ matrix.ruby-version }} + - name: Do not run in deployment mode + run: bundle config unset deployment + - name: Install dependencies run: bundle install From fc09bfcdce42a22633bc9d9476514a7a2b751f0d Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Wed, 15 May 2024 11:50:03 -0400 Subject: [PATCH 8/9] Tweak workflow to force new cache; drop redundant bundle install --- .github/workflows/ci.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49e29b95..c0122cff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,22 +7,17 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - ruby-version: ['3.1'] + ruby: ['3.1'] steps: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Ruby ${{ matrix.ruby-version }} + - name: Set up Ruby ${{ matrix.ruby }} uses: ruby/setup-ruby@v1 with: + ruby-version: ${{ matrix.ruby }} bundler-cache: true - ruby-version: ${{ matrix.ruby-version }} - - - name: Do not run in deployment mode - run: bundle config unset deployment - - - name: Install dependencies - run: bundle install + cache-version: 2 - name: Lint run: | From 4578077f8ea0938355b71c4079cab82f5401b552 Mon Sep 17 00:00:00 2001 From: Kristina Spurgin Date: Wed, 15 May 2024 11:55:56 -0400 Subject: [PATCH 9/9] CI: remove bundler cache from ruby setup --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0122cff..cbe6df04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,12 @@ jobs: uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby }} - bundler-cache: true - cache-version: 2 + + - name: Configure bundle + run: bundle config unset deployment + + - name: Install dependencies + run: bundle install - name: Lint run: |