traject 3.7.0 → 3.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +1 -1
  3. data/CHANGES.md +10 -0
  4. data/Gemfile +8 -0
  5. data/lib/traject/solr_json_writer.rb +30 -0
  6. data/lib/traject/version.rb +1 -1
  7. data/test/solr_json_writer_test.rb +33 -9
  8. data/traject.gemspec +1 -1
  9. metadata +7 -13
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7ffc677e0ebb13e01b852a1d59ddfdd3cd9906142520e0c296f69ebb0eeb7429
4
- data.tar.gz: 61b0e966f6ecd4d27e757e4cfc1057c72ac6deca5ad119c78ff883c246744814
3
+ metadata.gz: 9e9868f3b83385402413a2fc6c8865dc6ab3dd3776c25c0f1f1b88bf20024005
4
+ data.tar.gz: f86c298c93905948ca9425983e65811b2091ba39d6105895744df183a7c43695
5
5
  SHA512:
6
- metadata.gz: 8240b450b27df011c2ff998c24c612f44bdd21a2fde5fbab996ffe509f3fc45cec7a8a8947e385a35d06f5bd8ed19732287e8f2d3dab682cc6295f7320f8dfab
7
- data.tar.gz: f5dbcb44edb8d37a4e74cd1255aa1b05b638913337577c92d4fa276150c023b9e29823cc2b20ef050b33879ec63d76194d0202697d7c858a7aacd3d08241dcce
6
+ metadata.gz: 7d1e1122020632ac10d4da030915e0f710c8dd9bb6e9780089129c1ec7febb76f7b0e23ac4828d1e4860429164fe6052c8db768d478ae6102f4819b8cd512f4d
7
+ data.tar.gz: '08eead90c2ddfebe141aa4bea2f280878c27af35509a9f5b220d2899fad820f9954dbc30392aa3cfd00b79e77b8f5266c69ac7101fe381d5e777e47b2e12aca7'
data/.github/workflows/ruby.yml CHANGED
@@ -12,7 +12,7 @@ jobs:
12
12
  strategy:
13
13
  fail-fast: false
14
14
  matrix:
15
- ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0', 'jruby-9.1', 'jruby-9.2' ]
15
+ ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0', '3.1', 'jruby-9.1', 'jruby-9.2' ]
16
16
  name: Ruby ${{ matrix.ruby }}
17
17
  steps:
18
18
  - uses: actions/checkout@v2
data/CHANGES.md CHANGED
@@ -6,6 +6,16 @@
6
6
 
7
7
  *
8
8
 
9
+ ## 3.8.0
10
+
11
+ SolrJsonWriter: HTTPClient should use OS certs instead of packaged ones
12
+
13
+ HTTPClient, for whatever reason, prefers its own packaged certs, which are now years out of date
14
+ and don't work with Let's Encrypt.
15
+
16
+ This changes the code to prefer the OS certs, which can be overridden by setting
17
+ `solr_json_writer.use_packaged_certs` to `true` or `"true"`.
18
+
9
19
  ## 3.7.0
10
20
 
11
21
  * Add two new transformation macros, `Traject::Macros::Transformation.delete_if` and `Traject::Macros::Transformations.select`.
data/Gemfile CHANGED
@@ -11,3 +11,11 @@ group :debug do
11
11
  gem "ruby-debug", :platform => "jruby"
12
12
  gem "byebug", :platform => "mri"
13
13
  end
14
+
15
+ # ruby-marc stopped supporting ruby 2.3 and 2.4 in newer 1.x versions,
16
+ # while we would still like to support those old versions. When running
17
+ # CI, run with older ruby-marc that still supports them.
18
+ ruby_version_parts = RUBY_VERSION.split(".")
19
+ if ruby_version_parts[0] == "2" && ruby_version_parts[1].to_i < 5
20
+ gem "marc", "< 1.2.0"
21
+ end
data/lib/traject/solr_json_writer.rb CHANGED
@@ -86,6 +86,9 @@ require 'concurrent' # for atomic_fixnum
86
86
  # * solr_json_writer.http_client Mainly intended for testing, set your own HTTPClient
87
87
  # or mock object to be used for HTTP.
88
88
  #
89
+ # * solr_json_writer.use_packaged_certs: unlikely to be needed, set to true for legacy
90
+ # behavior, to use packaged HTTPClient gem ssl certs. https://github.com/nahi/httpclient/issues/445
91
+ #
89
92
  class Traject::SolrJsonWriter
90
93
  include Traject::QualifiedConstGet
91
94
 
@@ -118,6 +121,15 @@ class Traject::SolrJsonWriter
118
121
  @settings["solr_json_writer.http_client"]
119
122
  else
120
123
  client = HTTPClient.new
124
+
125
+ # By default we'll use teh host OS SSL certs, but you can use
126
+ # setting solr_json_writer.use_packaged_certs to true or "true"
127
+ # to go back to previous behavior if you have a perverse reason to.
128
+ # https://github.com/nahi/httpclient/issues/445
129
+ unless @settings["solr_json_writer.use_packaged_certs"].to_s == "true"
130
+ client.ssl_config.set_default_paths
131
+ end
132
+
121
133
  if @settings["solr_writer.http_timeout"]
122
134
  client.connect_timeout = client.receive_timeout = client.send_timeout = @settings["solr_writer.http_timeout"]
123
135
  end
@@ -431,9 +443,27 @@ class Traject::SolrJsonWriter
431
443
  attr_reader :response
432
444
 
433
445
  def initialize(msg, response = nil) # :nodoc:
446
+ solr_error = find_solr_error(response)
447
+ msg += ": #{solr_error}" if solr_error
448
+
434
449
  super(msg)
450
+
435
451
  @response = response
436
452
  end
453
+
454
+ private
455
+
456
+ # If we can get the error out of a JSON response, please do,
457
+ # to include in error message.
458
+ def find_solr_error(response)
459
+ return nil unless response && response.body && response.content_type&.start_with?("application/json")
460
+
461
+ parsed = JSON.parse(response.body)
462
+
463
+ parsed && parsed.dig("error", "msg")
464
+ rescue JSON::ParserError
465
+ return nil
466
+ end
437
467
  end
438
468
 
439
469
  private
data/lib/traject/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Traject
2
- VERSION = "3.7.0"
2
+ VERSION = "3.8.0"
3
3
  end
data/test/solr_json_writer_test.rb CHANGED
@@ -19,7 +19,7 @@ describe "Traject::SolrJsonWriter" do
19
19
  class FakeHTTPClient
20
20
  # Always reply with this status, normally 200, can
21
21
  # be reset for testing error conditions.
22
- attr_accessor :response_status
22
+ attr_accessor :response_status, :body, :content_type
23
23
 
24
24
  def initialize(*args)
25
25
  @post_args = []
@@ -33,10 +33,7 @@ describe "Traject::SolrJsonWriter" do
33
33
  @post_args << args
34
34
  end
35
35
 
36
- resp = HTTP::Message.new_response("")
37
- resp.status = self.response_status
38
-
39
- return resp
36
+ return faked_response
40
37
  end
41
38
 
42
39
  def get(*args)
@@ -44,10 +41,7 @@ describe "Traject::SolrJsonWriter" do
44
41
  @get_args << args
45
42
  end
46
43
 
47
- resp = HTTP::Message.new_response("")
48
- resp.status = self.response_status
49
-
50
- return resp
44
+ return faked_response
51
45
  end
52
46
 
53
47
  def post_args
@@ -65,6 +59,16 @@ describe "Traject::SolrJsonWriter" do
65
59
  # Everything else, just return nil please
66
60
  def method_missing(*args)
67
61
  end
62
+
63
+ private
64
+
65
+ def faked_response
66
+ resp = HTTP::Message.new_response(self.body || "")
67
+ resp.status = self.response_status
68
+ resp.content_type = self.content_type if self.content_type
69
+
70
+ resp
71
+ end
68
72
  end
69
73
 
70
74
 
@@ -157,6 +161,26 @@ describe "Traject::SolrJsonWriter" do
157
161
  assert_length 1, JSON.parse(individual_update2[1])
158
162
  end
159
163
 
164
+ it "includes Solr reported error in base error message" do
165
+ @writer = create_writer("solr_writer.batch_size" => 1, "solr_writer.max_skipped" => 0)
166
+ @fake_http_client.response_status = 400
167
+ @fake_http_client.content_type = "application/json;charset=utf-8"
168
+ @fake_http_client.body =
169
+ { "responseHeader"=>{"status"=>400, "QTime"=>0},
170
+ "error"=>{
171
+ "metadata"=>["error-class", "org.apache.solr.common.SolrException", "root-error-class", "org.apache.solr.common.SolrException"],
172
+ "msg"=>"ERROR: this is a solr error",
173
+ "code"=>400
174
+ }
175
+ }.to_json
176
+
177
+ error = assert_raises(Traject::SolrJsonWriter::MaxSkippedRecordsExceeded) {
178
+ @writer.put context_with({"id" => "doc_1", "key" => "value"})
179
+ @writer.close
180
+ }
181
+ assert_match(/ERROR: this is a solr error/, error.message)
182
+ end
183
+
160
184
  it "can #flush" do
161
185
  2.times do |i|
162
186
  doc = {"id" => "doc_#{i}", "key" => "value"}
data/traject.gemspec CHANGED
@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
33
33
  spec.add_dependency 'marc-fastxmlwriter', '~>1.0' # fast marc->xml
34
34
  spec.add_dependency "nokogiri", "~> 1.9" # NokogiriIndexer
35
35
 
36
- spec.add_development_dependency 'bundler', '>= 1.7', '< 3'
36
+ spec.add_development_dependency 'bundler', '~>2.0'
37
37
 
38
38
  spec.add_development_dependency "rake"
39
39
  spec.add_development_dependency "minitest"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: traject
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.7.0
4
+ version: 3.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2022-02-08 00:00:00.000000000 Z
12
+ date: 2022-12-09 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: concurrent-ruby
@@ -167,22 +167,16 @@ dependencies:
167
167
  name: bundler
168
168
  requirement: !ruby/object:Gem::Requirement
169
169
  requirements:
170
- - - ">="
171
- - !ruby/object:Gem::Version
172
- version: '1.7'
173
- - - "<"
170
+ - - "~>"
174
171
  - !ruby/object:Gem::Version
175
- version: '3'
172
+ version: '2.0'
176
173
  type: :development
177
174
  prerelease: false
178
175
  version_requirements: !ruby/object:Gem::Requirement
179
176
  requirements:
180
- - - ">="
181
- - !ruby/object:Gem::Version
182
- version: '1.7'
183
- - - "<"
177
+ - - "~>"
184
178
  - !ruby/object:Gem::Version
185
- version: '3'
179
+ version: '2.0'
186
180
  - !ruby/object:Gem::Dependency
187
181
  name: rake
188
182
  requirement: !ruby/object:Gem::Requirement
@@ -402,7 +396,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
402
396
  - !ruby/object:Gem::Version
403
397
  version: '0'
404
398
  requirements: []
405
- rubygems_version: 3.1.6
399
+ rubygems_version: 3.3.3
406
400
  signing_key:
407
401
  specification_version: 4
408
402
  summary: An easy to use, high-performance, flexible and extensible metadata transformation
OSZAR »