Darcs Support: Parses logs of newer version of darcs. Use hashes as tokens instead of patch names.

.gitignore

@@ -2,3 +2,4 @@
 *.pyc
 pkg/
 *.cache
+TAGS

.ruby-version

@@ -0,0 +1 @@
+2.0.0

.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+env:
+  - TRAVIS_CONFIG_DIR=.travis
+before_install:
+  - cd $TRAVIS_CONFIG_DIR
+  - sh .travis_ssh_setup.sh
+install:
+  - sh .install_multiple_scms.sh
+  - sh .install_dependencies.sh
+  - gem install posix-spawn
+before_script:
+  - cd $TRAVIS_BUILD_DIR

.travis/.install_dependencies.sh

@@ -0,0 +1,13 @@
+#/usr/bin/env sh
+
+bazaar_plugins_path=`bzr --version |  awk '/bzrlib:/ {print $2}'`
+
+cd "$bazaar_plugins_path/plugins"
+
+sudo bzr branch lp:bzr-xmloutput
+
+sudo mv bzr-xmloutput xmloutput
+
+cd xmloutput
+
+python setup.py build_ext -i

.travis/.install_multiple_scms.sh

@@ -0,0 +1,5 @@
+sudo sh -c 'echo "deb http://opensource.wandisco.com/ubuntu precise svn18" >> /etc/apt/sources.list.d/subversion18.list'
+sudo wget -q http://opensource.wandisco.com/wandisco-debian.gpg -O- | sudo apt-key add -
+sudo apt-get update
+sudo apt-get install -y subversion cvs bzr mercurial
+sudo ln -s /usr/bin/cvs /usr/bin/cvsnt

.travis/.travis_ssh_setup.sh

@@ -0,0 +1,3 @@
+ssh-keygen -t rsa -f ~/.ssh/id_rsa -N "" -q
+cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+ssh-keyscan -t rsa `hostname` >> ~/.ssh/known_hosts

README.md

@@ -1,4 +1,4 @@
-[![Ohloh SCM on Ohloh](https://www.ohloh.net/p/ohloh_scm/widgets/project_partner_badge.gif)](https://www.ohloh.net/p/ohloh_scm)
+[![Ohloh SCM on Ohloh](https://www.ohloh.net/p/ohloh_scm/widgets/project_partner_badge.gif)](https://www.ohloh.net/p/ohloh_scm) [![Build Status](https://travis-ci.org/blackducksoftware/ohloh_scm.svg?branch=master)](https://travis-ci.org/blackducksoftware/ohloh_scm)
 
 # Ohloh SCM
 
@@ -49,10 +49,17 @@ hg 1.1.2
 If you are using CVS instead of CVSNT, you can potentially try creating
 a shell alias or symlink mapping 'cvsnt' to 'cvs'.
 
+Ohloh SCM uses [posix-spawn](https://github.com/rtomayko/posix-spawn) to
+execute commands so ensure *posix-spawn* gem is installed
+
+``gem install posix-spawn``
+
+
 ## Usage with Bundler
 
 ```
 gem 'ohloh_scm', git: 'https://github.com/blackducksw/ohloh_scm/', require: 'scm'
+gem 'posix-spawn'
 ```
 ## Running
 

lib/ohloh_scm.rb

@@ -19,6 +19,7 @@ module OhlohScm
 require_relative 'ohloh_scm/adapters/hglib_adapter'
 require_relative 'ohloh_scm/adapters/bzr_adapter'
 require_relative 'ohloh_scm/adapters/bzrlib_adapter'
+require_relative 'ohloh_scm/adapters/darcs_adapter'
 require_relative 'ohloh_scm/adapters/factory'
 
 require_relative 'ohloh_scm/parsers/parser'
@@ -32,6 +33,7 @@ module OhlohScm
 require_relative 'ohloh_scm/parsers/hg_styled_parser'
 require_relative 'ohloh_scm/parsers/bzr_xml_parser'
 require_relative 'ohloh_scm/parsers/bzr_parser'
+require_relative 'ohloh_scm/parsers/darcs_parser'
 
 require_relative 'ohloh_scm/parsers/array_writer'
 require_relative 'ohloh_scm/parsers/xml_writer'

lib/ohloh_scm/adapters/bzr/misc.rb

@@ -10,7 +10,7 @@ def exist?
 		end
 
 		def ls_tree(token)
-			run("cd #{path} && bzr ls -V -r #{to_rev_param(token)}").split("\n")
+      run("cd #{path} && bzr ls -V -r #{to_rev_param(token)}").split("\n")
 		end
 
 		def to_rev_param(r=nil)

lib/ohloh_scm/adapters/bzrlib/bzrlib_pipe_client.rb

@@ -1,14 +1,14 @@
 require 'rubygems'
-require 'open4'
+require 'posix/spawn'
 
 class BzrPipeClient
   def initialize(repository_url)
     @repository_url = repository_url
     @py_script = File.dirname(__FILE__) + '/bzrlib_pipe_server.py'
   end
-  
+
   def start
-    @pid, @stdin, @stdout, @stderr = Open4::popen4 "python #{@py_script}"
+    @pid, @stdin, @stdout, @stderr = POSIX::Spawn::popen4 "python #{@py_script}"
     open_repository
   end
 
@@ -27,8 +27,9 @@ def send_command(cmd)
     # send the command
     @stdin.puts cmd
     @stdin.flush
+    return if cmd == "QUIT"
 
-    # get status on stderr, first letter indicates state, 
+    # get status on stderr, first letter indicates state,
     # remaing value indicates length of the file content
     status = @stderr.read(10)
     flag = status[0,1]
@@ -46,6 +47,7 @@ def send_command(cmd)
 
   def shutdown
     send_command("QUIT")
+    [@stdout, @stdin, @stderr].each { |io| io.close unless io.closed? }
     Process.waitpid(@pid, Process::WNOHANG)
   end
 end

lib/ohloh_scm/adapters/bzrlib_adapter.rb

@@ -5,13 +5,13 @@ module OhlohScm::Adapters
 	class BzrlibAdapter < BzrAdapter
 
     def setup
-      @bzr_client = BzrPipeClient.new(url)
-      @bzr_client.start
+      bzr_client = BzrPipeClient.new(url)
+      bzr_client.start
+      bzr_client
     end
 
     def bzr_client
-      setup unless @bzr_client
-      return @bzr_client
+      @bzr_client ||= setup
     end
 
     def cleanup

lib/ohloh_scm/adapters/darcs/cat_file.rb

@@ -0,0 +1,28 @@
+module OhlohScm::Adapters
+	class DarcsAdapter < AbstractAdapter
+		def cat_file(commit, diff)
+			cat(commit.token, diff.path)
+		end
+
+		def cat_file_parent(commit, diff)
+			p = parent_tokens(commit)
+			cat(p.first, diff.path) if p.first
+		end
+
+		def cat(revision, path)
+			out, err = run_with_err("cd '#{url}' && darcs show contents -h '#{revision}' #{escape(path)}")
+			# show contents gives no error for non-existent paths
+			#return nil if err =~ /No such file in rev/
+			raise RuntimeError.new(err) unless err.to_s == ''
+			return nil if out == ''
+			out
+		end
+
+		# Escape bash-significant characters in the filename
+		# Example:
+		#     "Foo Bar & Baz" => "Foo\ Bar\ \&\ Baz"
+		def escape(path)
+			path.gsub(/[ '"&()<>|]/) { |c| '\\' + c }
+		end
+	end
+end

lib/ohloh_scm/adapters/darcs/commits.rb

@@ -0,0 +1,88 @@
+module OhlohScm::Adapters
+	class DarcsAdapter < AbstractAdapter
+
+		# Return the number of commits in the repository following +since+.
+		def commit_count(opts={})
+			commit_tokens(opts).size
+		end
+
+		# Return the list of commit tokens following +since+.
+		def commit_tokens(opts={})
+			after = opts[:after] ? " --from-match 'hash #{opts[:after]}'" : ""
+			up_to = opts[:up_to] ? " --to-match 'hash #{opts[:up_to]}'" : ""
+			tokens = string_to_patch_tokens(run("cd '#{self.url}' && darcs changes#{after}#{up_to}")).reverse
+
+			# Darcs returns everything after *and including* since.
+			# We want to exclude it.
+			if tokens.any? && tokens.first == opts[:after]
+				tokens[1..-1]
+			else
+				tokens
+			end
+		end
+
+		# Returns a list of shallow commits (i.e., the diffs are not populated).
+		# Not including the diffs is meant to be a memory savings when we encounter massive repositories.
+		# If you need all commits including diffs, you should use the each_commit() iterator, which only holds one commit
+		# in memory at a time.
+		def commits(opts={})
+			after = opts[:after] ? " --from-match 'hash #{opts[:after]}'" : ""
+			log = run("cd '#{self.url}' && darcs changes#{after} --reverse")
+			a = OhlohScm::Parsers::DarcsParser.parse(log)
+			if a.any? && a.first.token == opts[:after]
+				a[1..-1]
+			else
+				a
+			end
+		end
+
+		# Returns a single commit, including its diffs
+		def verbose_commit(token)
+			log = run("cd '#{self.url}' && darcs changes -v -h '#{token}'")
+			OhlohScm::Parsers::DarcsParser.parse(log).first
+		end
+
+		# Yields each commit after +since+, including its diffs.
+		# The log is stored in a temporary file.
+		# This is designed to prevent excessive RAM usage when we encounter a massive repository.
+		# Only a single commit is ever held in memory at once.
+		def each_commit(opts={})
+			open_log_file(opts) do |io|
+				OhlohScm::Parsers::DarcsParser.parse(io) do |commit|
+					yield commit if block_given? && commit.token != opts[:after]
+				end
+			end
+		end
+
+		# Not used by Ohloh proper, but handy for debugging and testing
+		def log(opts={})
+      after = opts[:after] ? " --from-match 'hash #{opts[:after]}'" : ""
+			run "cd '#{url}' && darcs changes -s#{after}"
+		end
+
+		# Returns a file handle to the log.
+		# In our standard, the log should include everything AFTER +since+. However, darcs doesn't work that way;
+		# it returns everything after and INCLUDING +since+. Therefore, consumers of this file should check for
+		# and reject the duplicate commit.
+		def open_log_file(opts={})
+			after = opts[:after] ? " --from-match 'hash #{opts[:after]}'" : ''
+			begin
+				if opts[:after] == head_token # There are no new commits
+					# As a time optimization, just create an empty file rather than fetch a log we know will be empty.
+					File.open(log_filename, 'w') { }
+				else
+					after = opts[:after] ? " --from-match 'hash #{opts[:after]}'" : ""
+					run "cd '#{url}' && darcs changes --reverse -v#{after} > #{log_filename}"
+				end
+				File.open(log_filename, 'r') { |io| yield io }
+			ensure
+				File.delete(log_filename) if FileTest.exist?(log_filename)
+			end
+		end
+
+		def log_filename
+		  File.join('/tmp', (self.url).gsub(/\W/,'') + '.log')
+		end
+
+	end
+end

lib/ohloh_scm/adapters/darcs/head.rb

@@ -0,0 +1,23 @@
+module OhlohScm::Adapters
+	class DarcsAdapter < AbstractAdapter
+		def head_token
+		  string_to_patch_tokens(run("cd '#{url}' && darcs changes --last 1"))[0]
+		end
+
+		def head
+			verbose_commit(head_token)
+		end
+
+		def parent_tokens(commit)
+		  string_to_patch_tokens(run("cd '#{url}' && darcs changes --to-match 'hash #{commit.token}'"))[1..-1]
+		end
+
+		def parents(commit)
+			parent_tokens(commit).map {|token| verbose_commit(token)}
+		end
+
+		def string_to_patch_tokens(s)
+		  s.split(/\n/).select {|s| s =~ /^patch /}.map {|s| s.sub(/^patch /,'')}
+		end
+	end
+end

lib/ohloh_scm/adapters/darcs/misc.rb

@@ -0,0 +1,21 @@
+module OhlohScm::Adapters
+	class DarcsAdapter < AbstractAdapter
+		def exist?
+			begin
+				!!(head_token)
+			rescue
+				logger.debug { $! }
+				false
+			end
+		end
+
+		def ls_tree(token)
+			run("cd '#{path}' && darcs show files --no-pending -h '#{token}'").split("\n")
+		end
+
+		def export(dest_dir, token=nil)
+			p = token ? " -h '#{token}'" : ""
+			run("cd '#{path}' && darcs dist#{p} && mv darcs.tar.gz '#{dest_dir}'")
+		end
+	end
+end

lib/ohloh_scm/adapters/darcs/patch.rb

@@ -0,0 +1,7 @@
+module OhlohScm::Adapters
+  class DarcsAdapter < AbstractAdapter
+    def patch_for_commit(commit)
+      run("cd '#{url}' && darcs changes -h'#{commit.token}' -v")
+    end
+  end
+end

lib/ohloh_scm/adapters/darcs/pull.rb

@@ -0,0 +1,23 @@
+module OhlohScm::Adapters
+	class DarcsAdapter < AbstractAdapter
+
+		def pull(from, &block)
+			raise ArgumentError.new("Cannot pull from #{from.inspect}") unless from.is_a?(DarcsAdapter)
+			logger.info { "Pulling #{from.url}" }
+
+			yield(0,1) if block_given? # Progress bar callback
+
+			unless self.exist?
+				run "mkdir -p '#{self.url}'"
+				run "rm -rf '#{self.url}'"
+				run "darcs get '#{from.url}' '#{self.url}'"
+			else
+				# might also need to unpull for an exact copy
+				run "cd '#{self.url}' && darcs revert --all && darcs pull --dont-allow-conflicts -a '#{from.url}'"
+			end
+
+			yield(1,1) if block_given? # Progress bar callback
+		end
+
+	end
+end

lib/ohloh_scm/adapters/darcs/push.rb

@@ -0,0 +1,50 @@
+module OhlohScm::Adapters
+	class DarcsAdapter < AbstractAdapter
+
+		def push(to, &block)
+			raise ArgumentError.new("Cannot push to #{to.inspect}") unless to.is_a?(DarcsAdapter)
+			logger.info { "Pushing to #{to.url}" }
+
+			yield(0,1) if block_given? # Progress bar callback
+
+			unless to.exist?
+				if to.local?
+					# Create a new repo on the same local machine. Just use existing pull code in reverse.
+					to.pull(self)
+				else
+					run "cd '#{self.url}' && darcs put #{to.hostname}:#{to.path}"
+				end
+			else
+				run "cd '#{self.url}' && darcs push -a '#{to.url}'"
+			end
+
+			yield(1,1) if block_given? # Progress bar callback
+		end
+
+		def local?
+			return true if hostname == Socket.gethostname
+			return true if url =~ /^file:\/\//
+			return true if url !~ /:/
+			false
+		end
+
+		def hostname
+			$1 if url =~ /^ssh:\/\/([^\/]+)/
+		end
+
+		def path
+			case url
+			when /^file:\/\/(.+)$/
+				$1
+			when /^ssh:\/\/[^\/]+(\/.+)$/
+				$1
+			when /^[^:]*$/
+				url
+			end
+		end
+
+		def darcs_path
+			path && File.join(path, '.darcs')
+		end
+	end
+end