rakelib/hglib.rb
author Jan Vrany <jan.vrany@fit.cvut.cz>
Sun, 20 Nov 2016 23:28:58 +0000
changeset 76 df28d45f7f5a
parent 75 9b57c88b2ab3
child 78 2d09a485772f
permissions -rwxr-xr-x
On Windows use MSYS2 `ssh.exe` rather than `plink.exe` to clone/checkout repositories over SSH On Windows, most of users tend to use `plink.exe` as SSH client. Moreover, TortoiseHG comes with its own version if plink and it's pre-configured to use it. This results in a bad performance over high-speed LAN since plink uses 16k channel input buffer (!) leading to a pretty slow transfers (a lot of iowaits...) OTOH, OpenSSH client has 2MB input buffer which is much better. Sadly, does not help as much on Windows as TCP window size is fixed to 65k. Windows TCP window autotuning (CTCP) does not help as it's only enabled on connections with RTT > 1ms which is clearly not the case of high-speed low-latencly LAN. So, unless you hack OpenSSH sources to manually increase TCP window size to match 2MB channel buffer, you're doomed to slow transfers. How nice! Still, 65k is 4 times more than 16k, so still worth the hassle. As a workaround, look if MSYS2's OpenSSH client is installed and if so, use that one - but only if `ui.ssh` config option has the default value. This is soo ugly, isn't it? But it actually makes checkout over high-speed LAN ~8times faster on Windows, believe it or not! (at least on my setup). Sigh, I need a double scotch.

# This file is not a standalone script. It is a kind
# of lightweight Mercurial library used by other scripts.

require 'uri'
require 'open3'
require 'shellwords'

# Following hack is to make hglib.rb working wit both jv:scripts and
# Smalltalk/X rakefiles. 
begin
  require 'rakelib/inifile'
rescue
  begin
    require 'inifile'
  rescue LoadError => ex
    $LOGGER.error("Cannot load package 'inifile'")
    $LOGGER.error("Run 'gem install inifile' to install it")
    exit 1
  end
end

# Cross-platform way of finding an executable in the $PATH.
#
#   which('ruby') #=> /usr/bin/ruby
def which(cmd)
  exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
  ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
    exts.each { |ext|
      exe = File.join(path, "#{cmd}#{ext}")
      return exe if File.executable?(exe) && !File.directory?(exe)
    }
  end
  return nil
end

if not $LOGGER then
  if STDOUT.tty? or win32? then
    require 'logger'
    $LOGGER = Logger.new(STDOUT)
    
	if (VERBOSE != nil) then
	    $LOGGER.level = Logger::DEBUG
	else 
	   $LOGGER.level = Logger::INFO	
	end
  else 
    require 'syslog/logger'
    $LOGGER = Syslog::Logger.new($0)    
  end
end

module HG
  @@config = nil

  GLOBAL_OPTIONS= [:'cwd', :'repository', :'noninteractive', :'config', :'debug', :'debugger',
       :'encoding',:'encodingmode',:'traceback',:'time', :'profile',:'version', :'help',
       :'hidden' ]
  # Execute `hg` command with given positional arguments and
  # keyword arguments turned into command options. For example, 
  #
  #     HG::hg("heads", "default", cwd: '/tmp/testrepo')
  #
  # will result in executing
  # 
  #     hg --cwd '/tmp/testrepo' heads default
  #
  # In addition if block is passed, then the block is evaluate with
  # `hg` command exit status (as Process::Status) and (optionally)
  # with contents of `hg` command stdout and stderr. 
  # If no block is given, an exception is raised when `hg` command 
  # exit status IS NOT zero.
  def self.hg(command, *args, **options, &block)
    g_opts = []
    c_opts = []
    options.each do | k , v |       
      if v != false and v != nil
        o = k.size == 1 ? "-#{k}" : "--#{k}"                
        if GLOBAL_OPTIONS.include? k then                  
          if v.kind_of?(Array)
            v.each do | e |
              g_opts << o << (e == true ? '' : e)  
            end
          else
            g_opts << o << (v == true ? '' : v)
          end
        else
          if v.kind_of?(Array)
            v.each do | e |
              c_opts << o << (e == true ? '' : e)  
            end
          else
            c_opts << o << (v == true ? '' : v)
          end
        end
      end
    end
    c_opts.reject! { | e | e.size == 0 }
    cmd = ['hg'] + g_opts + [command] + c_opts + args      
    cmd_info = cmd.shelljoin.
                gsub(/username\\=\S+/, "username\\=***").
                gsub(/password\\=\S+/, "password\\=***")
    $LOGGER.debug("executing: #{cmd_info}")
    if defined? RakeFileUtils then
      puts cmd_info
    end
    if block_given? then
      stdout, stderr, status = Open3.capture3(*cmd)
      case block.arity
      when 1        
        yield status
      when 2        
        yield status, stdout
      when 3        
        yield status, stdout, stderr
      else
        raise Exception.new("invalid arity of given block")
      end
    else
      if not system(*cmd) then
        raise Exception.new("command failed: #{cmd.join(' ')}")
      end
    end    
  end

  def self.config()
    if @@config == nil
      files = Dir.glob('/etc/mercurial/hgrc.d/*.rc') + 
          [ '/etc/mercurial/hgrc' ,
          hgrc() ]
	  if Gem.win_platform? then
	     hg_exe = which("hg")
		 hgrc_d = File.join(File.dirname(hg_exe), "hgrc.d")
		 if File.directory? (hgrc_d) then
			files += Dir.glob("#{hgrc_d}\\*.rc".gsub('\\', '/'))
		 end
	  end
      @@config = IniFile.new()
      files.each do | file |
        if File.exist?(file)
          $LOGGER.debug("Loading global config from \"#{file}\"")
          @@config.merge!(IniFile.new(:filename => file))
        end
      end  
    end
    return @@config
  end

  def self.hgrc()
  	return File.expand_path('~/.hgrc')
  end

  class Repository
    attr_accessor :path, :config

    # Clone a repository from given `uri` to given `directory`. 
    # Returns an `HG::Repository` instance representing the repository
    # clone. 
    # If `noupdate` is true, working copy is not updated, i.e., will be
    # empty. Use this when you're going to issue `update(rev)` shortly after.
    #
    def self.clone(uri, directory, noupdate: false)
	  uri_obj = URI(uri)
      host = uri_obj.host
	  scheme = uri_obj.scheme
      # When cloning over LAN, use --uncompressed option
      # as it tends to be faster if bandwidth is good (1GB norm
      # these days) amd saves some CPU cycles.
	  local = false
      if host
        require 'resolv'
        addr = Resolv.getaddress(host)
        # Really poor detection of LAN, but since this is an 
        # optimization, getting this wrong does not hurt.         
        local = (addr.start_with? '192.168.') or (addr.start_with? '10.10.')
      end
	  # On Windows, most of users tend to use TortoiseHG and hg.exe comming with
	  # it. THG makes hg.exe to use (shipped) plink.exe which is bad for performance 
	  # since it uses 16k channel input buffer (!) leading to a pretty slow transfers 
	  # (a lot of iowaits...)
	  # OpenSSH OTOH has 2MB input buffer which is good though on Windows bit 
	  # oversized as Windows TCP window size is fixed to 65k for all connections with 
	  # RTT less than 1ms. Still, 65k better then 16k. 
	  # As a workaround, look if MSYS2's OpenSSH client is installed and if so, use that 
	  # one - but only if `ui.ssh` config option has the default value. 
	  # Ugly, isn't it? 
	  ssh = nil
	  puts "1 #{HG::config['ui'].has_key?('ssh')}"
	  if (scheme == 'ssh') and (Gem.win_platform?) and (HG::config['ui'].has_key?('ssh')) then
	    # For THG, `ui.ssh` is configured as: 
		# 
		#     "C:\Program Files\TortoiseHg\lib\TortoisePlink.exe" -ssh -2
		#
		# Be more relaxed and conver all "standard" plink.exe usages...
		if File.exist? "c:\\msys64\\usr\\bin\\ssh.exe" then
			# Rename. only if user did not override the setting! 
			if /^.*[pP]link.exe"?\s*(-ssh)?\s*(-2)?$/ =~ HG::config['ui']['ssh'] then
				ssh = "\"c:\\msys64\\usr\\bin\\ssh.exe\""
			end
			# Since we're messing wth ssh config anyway, add -C if we're cloning "over LAN"
			# to save some CPU cycles. Same reasosing as for --uncompressed above. 
			if local then
				ssh += " -C"
			end
		end
	  end
	  
	  # A downside of messing with ssh configuration is that OpenSSH client does not know how
	  # to connect to pageant. So issue a warning...
	  if ssh then
		$LOGGER.warn("Passing --ssh \"#{ssh}\" option to 'hg clone' for better performance.")
		if not ENV['SSH_AUTH_SOCK'] then
			$LOGGER.warn("Clone may fail since MSYS2 `ssh.exe` dont know how to talk to pageant. ")
			$LOGGER.warn("Consider using ssh-pageant")
		end
	  end
      if noupdate then
        HG::hg("clone", uri, directory, ssh: ssh, uncompressed: local, noupdate: true)
      else
        HG::hg("clone", uri, directory, ssh: ssh, uncompressed: local)
      end
      return HG::Repository.new(directory)
    end

    # Initializes an empty repository in given directory. Returns an 
    # `HG::Repository` instance representing the created (empty) repository.
    def self.init(directory)
      HG::hg("init", directory)
      return HG::Repository.new(directory)
    end

    # Like HG::hg, but passes --cwd @path
    def hg(command, *args, **options, &block)
      options[:cwd] = @path
      HG::hg(command, *args, **options, &block)
    end

    def hgrc() 
      return File.join(@path, '.hg', 'hgrc')
    end

    def initialize(directory)
      @path = directory
      config_file = hgrc()      
      if File.exist? ( config_file ) 
        $LOGGER.debug("Loading repository config from \"#{config_file}\"")
        @config = HG::config().merge(IniFile.new(:filename => config_file))
      else
        @config = HG::config()
      end
    end

    # Return a hashmap with defined paths (alias => uri)
    def paths() 
      return @config['paths'].clone
    end

    # Set paths for given repository
    def paths=(paths)
      config = IniFile.new(:filename => self.hgrc())
      config['paths'] = paths
      config.write()
    end

    def log(revset, template = "{node|short}\n")      
      log = []
      hg("log", rev: revset, template: template) do | status, out |     
        if status.success?
          puts out
          log = out.split("\n")
        end
      end
      return log
    end

    # Return changeset IDs of all head revisions. 
    # If `branch` is given, return only heads in given
    # branch.
    def heads(branch = nil) 
      if branch then
        return log("head() and branch('#{branch}')")
      else
        return log("head()")
      end
    end

    # Return name of an active bookmark or nil if no bookmark
    # is active
    def bookmark() 
      filename = File.join(@path, '.hg', 'bookmarks.current')
      if File.exist? filename then
        file = File.open(filename, "r")
        begin
          bookmark = file.read.chomp
        ensure
          file.close()
        end
        return bookmark
      else
        return nil
      end
    end

    # Return a hash "bookmark => revision" of all 
    # bookmarks. 
    def bookmarks(branch = nil)
      revset  = "bookmark()"
      revset += " and branch('#{branch}')" if branch
      bookmarks = {}
      self.log(revset, "{bookmarks}|{node|short}\n").each do | line |
        bookmark, changesetid = line.split("|")
        bookmarks[bookmark] = changesetid
      end
      return bookmarks
    end

    def pull(remote = 'default', user: nil, pass: nil, rev: nil, bookmarks: [])
      authconf = []
      if pass != nil then
        if user == nil then
          raise Exception.new("Password given but not username! Use user: named param to specify username.")
        end
        # If user/password is provided, make sure we don't have
        # username in remote URI. Otherwise Mercurial won't use 
        # password from config!        
        uri = URI.parse(self.paths[remote] || remote)
        uri.user = nil
        uri = uri.to_s
        uri_alias = if self.paths.has_key? remote then remote else 'xxx' end
        authconf << "auth.#{uri_alias}.prefix=#{uri}"
        authconf << "auth.#{uri_alias}.username=#{user}"        
        authconf << "auth.#{uri_alias}.password=#{pass}"        
      end
      hg("pull", remote, config: authconf, rev: nil) do | status |
        if not status.success? then
          raise Exception.new("Failed to pull from #{remote} (exit code #{status.exitstatus})")
        end
      end
    end

    def push(remote = 'default', user: nil, pass: nil, rev: nil)
      authconf = []
      if pass != nil then
        if user == nil then
          raise Exception.new("Password given but not username! Use user: named param to specify username.")
        end
        # If user/password is provided, make sure we don't have
        # username in remote URI. Otherwise Mercurial won't use 
        # password from config!        
        uri = URI.parse(self.paths[remote] || remote)
        uri.user = nil
        uri = uri.to_s
        uri_alias = if self.paths.has_key? remote then remote else 'xxx' end
        authconf << "auth.#{uri_alias}.prefix=#{uri}"
        authconf << "auth.#{uri_alias}.username=#{user}"        
        authconf << "auth.#{uri_alias}.password=#{pass}"        
      end      
      hg("push", remote, config: authconf, rev: rev) do | status |
        if status.exitstatus != 0 and status.exitstatus != 1 then
          raise Exception.new("Failed to push to #{remote} (exit code #{status.exitstatus})")
        end
      end            
    end

    # Create a shared clone in given directory, Return a new
    # HG::Repository object on the shared clone
    def share(dst, rev = nil)
      if File.exist? dst then
        raise Exception.new("Destination file exists: #{dst}")
      end
      if rev == nil then
        rev = log('.')[0]
      end
      if not has_revision?(rev) 
        raise Exception.new("Revision #{rev} does not exist")
      end
      mkdir_p File.dirname(dst);
      HG::hg("share", path, dst, config: 'extensions.share=', noupdate: true, bookmarks: false)
      share = Repository.new(dst)
      share.update(rev);
      return share
    end

    # Updates the repository's working copy to given 
    # revision if given. If not, update to most-recent
    # head, as plain
    #
    #   hg update
    #
    # would do. 
    def update(rev = nil)
      if rev 
        if not has_revision? rev then
          raise Exception.new("Revision #{rev} does not exist")
        end
        hg("update", rev: rev)
      else
        hg("update")
      end
    end

    # Merge given revision. Return true, if the merge was
    # successful, false otherwise
    def merge(rev)
      if not has_revision? rev then
        raise Exception.new("Revision #{rev} does not exist")
      end
      hg("merge", rev) do | status |
        return status.success?
      end
    end

    def commit(message)
      user = ''
      if not @config['ui'].has_key? 'username' then
        user = @config['ui']['username']
      end
      hg("commit", message: message, user: user)
    end

    def has_revision?(rev)
    	revs = log(rev)
    	return revs.size > 0      
    end

    # Lookup a repository in given `directory`. If found,
    # return it as instance of HG::Repository. If not,
    # `nil` is returned.
    def self.lookup(directory)
      return nil if not File.exist?(directory)
      repo_dir = directory
      while repo_dir != nil
        if HG::repository? repo_dir
          return Repository.new(repo_dir)
        end
        repo_dir_parent = File.dirname(repo_dir)
        if repo_dir_parent == repo_dir
          repo_dir = nil
        else 
          repo_dir = repo_dir_parent
        end
      end
    end    

    # Initializes and empty Mercurial repository in given `directory`
    def self.init(directory)
      FileUtils.mkdir_p File.dirname(directory)
      HG::hg("init", directory)
      return Repository.new(directory)
    end
  end # class Repository 

  # Return `true` if given `directory` is a root of mercurial
  # repository, `false` otherwise.
  def self.repository?(directory)
    return File.directory? File.join(directory, '.hg')
  end

  # Enumerate all repositories in given `directory`
  def self.forest(directory, &block)      
    if repository? directory  
      yield Repository.new(directory)
    end
    Dir.foreach(directory) do |x|
      path = File.join(directory, x)
      if File.directory? path       
        if x == "." or x == ".." or x == ".svn" or x == '.git'
          next    
        elsif File.directory?(path)
          forest(path, &block)
        end
      end
    end  
  end
end # module HG