RubyGems - sdr-replication - Versions diffs - 0.1.0 - Mend

sdr-replication 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (15) hide show

checksums.yaml +7 -0
data/lib/libdir.rb +3 -0
data/lib/replication/archive_catalog.rb +110 -0
data/lib/replication/bagit_bag.rb +337 -0
data/lib/replication/command_consumer.rb +55 -0
data/lib/replication/command_producer.rb +105 -0
data/lib/replication/dpn_check_rep.rb +83 -0
data/lib/replication/file_fixity.rb +98 -0
data/lib/replication/fixity.rb +155 -0
data/lib/replication/operating_system.rb +33 -0
data/lib/replication/replica.rb +62 -0
data/lib/replication/sdr_object_version.rb +63 -0
data/lib/replication/tarfile.rb +160 -0
data/lib/sdr_replication.rb +26 -0
metadata +198 -0

checksums.yaml ADDED Viewed

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e99a0814e4383ec6287dec7df41825d786e65919
+  data.tar.gz: 4890c2758dd820f22ce8aa9015621e1b33dec9ae
+SHA512:
+  metadata.gz: deb3400e53fcbdf16cf8263ffe1deed70be83889633b4fecfa0de58e41a076e7f502de61d22f079b09ff6b81ddb29586bb6e898782f22e45010ecfd46d93828d
+  data.tar.gz: a0533d23addcc264e7aeca97f985bdad1d70e8081230e09ab5337a03314b058374541b1587f47a5f89dd2f558115677190d49b320b892a77e7031bb15ac9c3ff

data/lib/libdir.rb ADDED Viewed

@@ -0,0 +1,3 @@
+libdir = File.expand_path(File.join(File.dirname(__FILE__)))
+$LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)

data/lib/replication/archive_catalog.rb ADDED Viewed

@@ -0,0 +1,110 @@
+require 'rubygems'
+require 'rest-client'
+module Replication
+  # A wrapper class based on {RestClient} used to interface with the Archive Catalog service.
+  # <br>
+  # <br>
+  # The default RestClient behavior is:
+  # * for results code between 200 and 207 a RestClient::Response will be returned
+  # * for results code 301, 302 or 307 the redirection will be followed if the request is a get or a head
+  # * for result code 303 the redirection will be followed and the request transformed into a get
+  # * for other cases a RestClient::Exception holding the Response will be raised
+  #
+  # But we are using a technique that forces RestClient to always provide the response
+  # <br>
+  # <br>
+  # RestClient::Response has these instance methods (some inherited from AbstractResponse):
+  # * args
+  # * body
+  # * code (e.g. 204)
+  # * description (e.g. "204 No Content |  0 bytes")
+  # * headers
+  # * net_http_res
+  #
+  # @see https://github.com/rest-client/rest-client
+  # @see http://rubydoc.info/gems/rest-client/1.6.7/frames
+  class ArchiveCatalog
+    @root_uri = 'http://localhost:3000'
+    @timeout = 120
+    # @see https://www.google.com/search?q="class+<<+self"+"attr_accessor"
+    class << self
+      # @return [String] The base or home URL of the Archive Catalog web service
+      attr_accessor :root_uri
+      # @return [Integer] seconds to wait for a response or to open a connection. Value nil disables the timeout.
+      attr_accessor :timeout
+      # The base RestClient resource to be used for requests
+      def root_resource
+        RestClient::Resource.new(@root_uri, {:open_timeout => @timeout, :timeout => @timeout})
+      end
+      # Get the item record from the specified table for the specified primary key.
+      # @param [String] table name of the database table
+      # @param [String] id primary key for the item in the database table
+      # @return [Hash] the row (in key,value hash) from the specified table for the specified identifier.
+      #    Response body contains the item data in JSON format, which is converted to a hash.
+      # @see http://tools.ietf.org/html/rfc2616#page-53
+      def get_item(table,id)
+        # Don't raise RestClient::Exception but return the response
+        headers = {:accept => 'application/json'}
+        response = root_resource["#{table}/#{id}.json"].get(headers) {|response, request, result| response }
+        case response.code.to_s
+          when '200'
+            JSON.parse(response.body)
+          else
+            raise response.description
+        end
+      end
+      # Retrieve an existing database record or add a new one using the data provided.
+      # @param [String] table name of the database table
+      # @param [Hash] hash the item data to be added to the database table
+      # @return [Hash] result containing the item data as if a GET were performed.
+      #    The HTTP response code for success is 201 (Created).
+      # @see http://en.wikipedia.org/wiki/POST_(HTTP)
+      # @see http://tools.ietf.org/html/rfc2616#page-54
+      def find_or_create_item(table,hash)
+        payload = hash.to_json
+        headers = {:content_type => :json, :accept => :json}
+        # Don't raise RestClient::Exception but return the response
+        response = root_resource["#{table}.json"].post(payload, headers) {|response, request, result| response }
+        case response.code.to_s
+          when '201'
+            JSON.parse(response.body)
+          else
+            raise response.description
+        end
+      end
+      # Update the database columns for the specified item using the hash data.
+      # @param [String] table name of the database table
+      # @param [String] id primary key for the item in the database table
+      # @param [Hash] hash the item data to be updated in the database table
+      # @return (Boolean) true if the HTTP response code is 204, per specification for PATCH or PUT request types.
+      #    Response body is empty, per same specification.
+      # @see https://tools.ietf.org/html/rfc5789
+      # @see http://stackoverflow.com/questions/797834/should-a-restful-put-operation-return-something/827045#827045
+      def update_item(table,id,hash)
+        payload = hash.to_json
+        headers = {:content_type => :json}
+        # Don't raise RestClient::Exception but return the response
+        response = root_resource["#{table}/#{id}.json"].patch(payload, headers) {|response, request, result| response }
+        case response.code.to_s
+          when '204'
+            true
+          else
+            raise response.description
+        end
+      end
+    end
+  end
+end

data/lib/replication/bagit_bag.rb ADDED Viewed

@@ -0,0 +1,337 @@
+require File.join(File.dirname(__FILE__),'../libdir')
+require 'sdr_replication'
+module Replication
+  # A BagIt bag contains a structured copy of a digital object for storage, transfer, or replication
+  # @see https://tools.ietf.org/html/draft-kunze-bagit-10
+  # This class can be used to create, parse, or validate a bag instance
+  #
+  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class BagitBag
+    # @param [Pathname,String] pathname The location of the bag home directory
+    # @return [BagitBag] Initialize a new bag, create home and payload folders, write bagit.txt file
+    def BagitBag.create_bag(pathname)
+      bag = BagitBag.new
+      bag.bag_pathname = pathname
+      bag.payload_pathname.mkpath
+      bag.write_bagit_txt
+      bag
+    end
+    # @param [Pathname,String] pathname The location of the bag home directory
+    # @return [BagitBag] Initialize a new bag, create home and payload folders, write bagit.txt file
+    def BagitBag.open_bag(pathname)
+      bag = BagitBag.new
+      bag.bag_pathname = pathname
+      raise "No bag found at #{bag.bag_pathname}" unless bag.bag_pathname.exist?
+      bagit_txt = bag.bag_pathname.join("bagit.txt")
+      raise "No bagit.txt file found at #{bagit_txt}" unless bagit_txt.exist?
+      bag
+    end
+    # @return [Pathname] The location of the bag home directory
+    def bag_pathname
+      @bag_pathname
+    end
+    # @param [Pathname,String] pathname The location of the bag home directory
+    # @return [Void] Set the location of the bag home directory
+    def bag_pathname=(pathname)
+      @bag_pathname = Pathname(pathname)
+    end
+    # @return [Pathname] The location of the bag data directory
+    def payload_pathname
+      bag_pathname.join('data')
+    end
+    # @return [Pathname] Generate the bagit.txt tag file
+    def write_bagit_txt
+      bagit_txt = bag_pathname.join("bagit.txt")
+      bagit_txt.open('w') do |f|
+       f.puts "Tag-File-Character-Encoding: UTF-8"
+       f.puts "BagIt-Version: 0.97"
+      end
+      bagit_txt
+    end
+    # @return [Hash<String,String] A hash containing the properties documented in the bagit.txt tagfile
+    def read_bagit_txt
+      properties = Hash.new
+      bagit_txt = bag_pathname.join("bagit.txt")
+      bagit_txt.readlines.each do |line|
+        line.chomp!.strip!
+        key,value = line.split(':',2)
+        properties[key.strip] = value.strip if value
+      end
+      properties
+    end
+    # @return [Array<Symbol>] The list of checksum types to be used when generating fixity data
+    def bag_checksum_types
+      @bag_checksum_types ||= Fixity.default_checksum_types
+    end
+    # @param [Object] types The list of checksum types to be used when generating fixity data
+    # @return [Void] Set the list of checksum types to be used when generating fixity data
+    def bag_checksum_types=(*types)
+      @bag_checksum_types = Fixity.validate_checksum_types(*types)
+    end
+    # @param [Symbol] link_mode Specifies whether to :copy, :link, or :symlink the files to the payload directory
+    # @param [Pathname] source_dir The source location of the directory whose contents are to be ingested
+    # @return [Pathname] Generate file_fixity_hash and send it to #add_payload_files
+    def add_payload_dir (link_mode, source_dir)
+      file_fixity_hash = Fixity.generate_checksums(source_dir, nil ,bag_checksum_types)
+      add_payload_files(link_mode, source_dir, file_fixity_hash)
+      payload_pathname
+    end
+    # @param [Symbol] link_mode Specifies whether to :copy, :link, or :symlink the files to the payload directory
+    # @param [Pathname] source_basepath The source location of the directory whose contents are to be ingested
+    # @param [Hash<String,FileFixity>] file_fixity_hash The list of files (with fixity data) to be added to the payload
+    # @return [Pathname] Copy or link the files specified in the file_fixity_hash to the payload directory,
+    #   then update the payload manifest files
+    def add_payload_files(link_mode, source_basepath, file_fixity_hash)
+      file_fixity_hash.keys.each do |file_id|
+        source_pathname = source_basepath.join(file_id)
+        target_pathname = payload_pathname.join(file_id)
+        copy_file(link_mode, source_pathname, target_pathname)
+      end
+      write_manifest_checksums('manifest', file_fixity_hash)
+      payload_pathname
+    end
+    # @param [Symbol] link_mode Specifies whether to :copy, :link, or :symlink the files to the payload directory
+    # @param [Pathname] source_pathname The source location of the file to be ingested
+    # @param [Pathname] target_pathname The location of the directory in which to place the file
+    # @return [Pathname] link or copy the specified file from source location to the target location
+    def copy_file(link_mode, source_pathname, target_pathname)
+      target_pathname.parent.mkpath
+      case link_mode
+        when :copy, nil
+          FileUtils.copy(source_pathname.to_s, target_pathname.to_s) # automatically dereferences symlinks
+        when :link
+          FileUtils.link(source_pathname.to_s, target_pathname.to_s) #, :force => true (false is default)
+        when :symlink
+          FileUtils.symlink(source_pathname.to_s, target_pathname.to_s) #, :force => true (false is default)
+        else
+          raise "Invalid link_mode: #{link_mode}, expected one of [:copy,:link,:symlink]"
+      end
+      target_pathname
+    end
+    # @param [Pathname,String] source_fullpath The location of the directory whose content will be tarred
+    # @param [Pathname,String] source_basepath The location of the directory to change to before doing the tar create
+    # @return [Tarfile] Create a tar archive of a directory into the payload directory,
+    #   generating checksums in parallel processes and recording those checksums in the payload manifests
+    def add_payload_tarfile(tarfile_id,source_fullpath, source_basepath)
+      tarfile = Tarfile.new
+      tarfile.source_basepath = Pathname(source_basepath)
+      tarfile.source_fullpath = Pathname(source_fullpath)
+      tarfile.tarfile_basepath = payload_pathname
+      tarfile.tarfile_fullpath = payload_pathname.join("#{tarfile_id}")
+      tarfile.create_tarfile
+      file_fixity_hash = Fixity.generate_checksums(tarfile.tarfile_basepath,[tarfile.tarfile_fullpath],bag_checksum_types)
+      write_manifest_checksums('manifest', file_fixity_hash)
+      tarfile
+    end
+    # @return [Pathname] Generate the bag-info.txt tag file to record the payload size
+    def write_bag_info_txt
+      payload_size = bag_payload_size
+      bag_info_txt = bag_pathname.join("bag-info.txt")
+      bag_info_txt.open('w') do |f|
+        f.puts "External-Identifier: #{bag_pathname.basename}"
+        f.puts "Payload-Oxum: #{payload_size[:bytes]}.#{payload_size[:files]}"
+        f.puts "Bag-Size: #{bag_size_human(payload_size[:bytes])}"
+      end
+      bag_info_txt
+    end
+    # @return [Hash<Symbol,Integer>] A hash contining the payload size in bytes, and the number of files,
+    #   derived from the payload directory contents
+    def bag_payload_size
+      payload_pathname.find.select{|f| f.file?}.inject({bytes: 0, files: 0}) do |hash,file|
+        hash[:bytes] += file.size
+        hash[:files] += 1
+        hash
+      end
+    end
+    # @param [Integer] bytes The total number of bytes in the payload
+    # @return [String] Human-readable rendition of the total payload size
+    def bag_size_human(bytes)
+      count = 0
+      size = bytes
+      while ( size >= 1024 and count < 4 )
+        size /= 1024.0
+        count += 1
+      end
+      if (count == 0)
+        return sprintf("%d B", size)
+      else
+        return sprintf("%.2f %s", size, %w[B KB MB GB TB][count] )
+      end
+    end
+    # @return [Hash<String,String] A hash containing the properties documented in the bag-info.txt tagfile
+    def read_bag_info_txt
+      properties = Hash.new
+      bag_info = bag_pathname.join("bag-info.txt")
+      bag_info.readlines.each do |line|
+        line.chomp!.strip!
+        key,value = line.split(':',2)
+        properties[key.strip] = value.strip if value
+      end
+      properties
+    end
+    # @return [Hash<Symbol,Integer>] A hash contining the payload size in bytes, and the number of files,
+    #   derived from the Payload-Oxum property
+    def info_payload_size
+      info = read_bag_info_txt
+      size_array = info['Payload-Oxum'].split('.')
+      size_hash = {:bytes => size_array[0].to_i, :files => size_array[1].to_i}
+      size_hash
+    end
+    # @return [Boolean] Compare the actual measured payload size against the value recorded in bag-info.txt
+    def verify_payload_size
+      info_size = info_payload_size
+      bag_size = bag_payload_size
+      if info_size != bag_size
+        raise "Failed payload size verification! Expected: #{info_size}, Found: #{bag_size}"
+      end
+      true
+    end
+    # @return [Hash<String,FileFixity>] create hash containing ids and checksums for all files in the bag's root directory
+    def generate_tagfile_checksums
+      tagfiles = bag_pathname.children.reject{|file| file.basename.to_s.start_with?('tagmanifest')}
+      Fixity.generate_checksums(bag_pathname, tagfiles, bag_checksum_types )
+    end
+    # @return [Hash<String,FileFixity>] create hash containing ids and checksums for all files in the bag's payload
+    def generate_payload_checksums
+      Fixity.generate_checksums(payload_pathname, nil, bag_checksum_types)
+    end
+    # @param [String] manifest_type The type of manifest file ('manifest' or 'tagmanifest') to be updated
+    # @param [Hash<String,FileFixity>] file_fixity_hash A hash containing file ids and fixity data
+    # @param [String] open_mode The file open mode (default is 'a')
+    # @return [Hash<Symbol,Pathname] Update each of the manifests with data from the file_fixity_hash
+    def write_manifest_checksums(manifest_type, file_fixity_hash, open_mode='a')
+      manifests = Hash.new
+      self.bag_checksum_types.each do |checksum_type|
+        manifest_pathname = bag_pathname.join("#{manifest_type}-#{checksum_type}.txt")
+        manifest_file = manifest_pathname.open(open_mode)
+        file_fixity_hash.values.each do |fixity|
+          checksum = fixity.get_checksum(checksum_type)
+          manifest_file.puts("#{checksum} #{fixity.file_id}") if checksum
+        end
+        manifest_file.close
+        manifests[checksum_type] = manifest_pathname
+      end
+      manifests
+    end
+    # @param [String] manifest_type The type of manifest file ('manifest' or 'tagmanifest') to be read
+    # @return [Hash<String,FileFixity>] A hash containing file ids and fixity data derived from the manifest files
+    def read_manifest_files(manifest_type)
+      file_fixity_hash = Hash.new
+      checksum_type_list = Array.new
+      Fixity.valid_checksum_ids.each do |checksum_type|
+        manifest_pathname = bag_pathname.join("#{manifest_type}-#{checksum_type}.txt")
+        if manifest_pathname.file?
+          checksum_type_list << checksum_type
+          manifest_pathname.readlines.each do |line|
+            line.chomp!.strip!
+            checksum,file_id = line.split(/[\s*]+/,2)
+            file_fixity = file_fixity_hash[file_id] || FileFixity.new(file_id: file_id)
+            file_fixity.set_checksum(checksum_type,checksum)
+            file_fixity_hash[file_id] = file_fixity
+          end
+        end
+      end
+      self.bag_checksum_types = self.bag_checksum_types | checksum_type_list
+      file_fixity_hash
+     end
+    # @return [Boolean] Compare fixity data from the tag manifest files against the values measured by digesting the files
+    def verify_tagfile_manifests
+      manifest_type = 'tagmanifest'
+      manifest_fixity_hash = read_manifest_files(manifest_type)
+      bag_fixity_hash = generate_tagfile_checksums
+      verify_manifests(manifest_type, manifest_fixity_hash, bag_fixity_hash)
+    end
+    # @return [Boolean] Compare fixity data from the payload manifest files against the values measured by digesting the files
+    def verify_payload_manifests
+      manifest_type = 'manifest'
+      manifest_fixity_hash = read_manifest_files(manifest_type)
+      bag_fixity_hash = generate_payload_checksums
+      verify_manifests(manifest_type, manifest_fixity_hash, bag_fixity_hash)
+    end
+    # @param [String] manifest_type The type of manifest file ('manifest' or 'tagmanifest') to be read
+    # @param [Hash<String,FileFixity>] manifest_fixity_hash A hash containing file ids and fixity data derived from the manifest files
+    # @param [Hash<String,FileFixity>] bag_fixity_hash A hash containing file ids and fixity data derived from the actual files
+    # @return [Boolean] Compare fixity data from the manifest files against the values measured by digesting the files,
+    #   returning true if equal or false if not equal
+    def verify_manifests(manifest_type, manifest_fixity_hash, bag_fixity_hash)
+      diff = manifest_diff(manifest_fixity_hash, bag_fixity_hash)
+      if diff.size > 0
+        raise "Failed #{manifest_type} verification! Differences: \n#{diff.inspect}"
+      end
+      true
+    end
+    # @param [Hash<String,FileFixity>] manifest_fixity_hash A hash containing file ids and fixity data derived from the manifest files
+    # @param [Hash<String,FileFixity>] bag_fixity_hash A hash containing file ids and fixity data derived from the actual files
+    # @return [Hash] A report of the differences between the fixity data from the manifest files
+    #   against the values measured by digesting the files
+    def manifest_diff(manifest_fixity_hash, bag_fixity_hash)
+      diff = Hash.new
+      (manifest_fixity_hash.keys | bag_fixity_hash.keys).each do |file_id|
+        manifest_fixity = manifest_fixity_hash[file_id] || FileFixity.new(file_id: file_id)
+        bag_fixity = bag_fixity_hash[file_id] || FileFixity.new(file_id: file_id)
+        if manifest_fixity != bag_fixity
+          diff[file_id] = manifest_fixity.diff(bag_fixity,'manifest','bag')
+        end
+      end
+      diff
+    end
+    # @return [Boolean] Validate the bag containing the digital object
+    def verify_bag
+      verify_bag_structure
+      verify_tagfile_manifests
+      verify_payload_size
+      verify_payload_manifests
+      true
+    end
+    # @return [Boolean] Test the existence of expected files, return true if files exist, raise exception if not
+    def verify_bag_structure
+      required_files = ['data','bagit.txt','bag-info.txt','manifest-sha256.txt','tagmanifest-sha256.txt']
+      required_files.each{|filename| verify_pathname(bag_pathname.join(filename))}
+      optional_files = []
+      true
+    end
+    # @param [Pathname] pathname The file whose existence should be verified
+    # @return [Boolean] Test the existence of the specified path.  Return true if file exists, raise exception if not
+    def verify_pathname(pathname)
+      raise "#{pathname.basename} not found at #{pathname}" unless pathname.exist?
+      true
+    end
+  end
+end