Automatically download pre-built librun and stc when sources are not available.
authorJan Vrany <jan.vrany@fit.cvut.cz>
Wed, 25 May 2016 17:19:27 +0100
changeset 11 f7dc950d8df8
parent 10 cb3e0e3ca28f
child 12 5bd69db5ca2b
Automatically download pre-built librun and stc when sources are not available. ...from SWING Jenkins job. A courstesy to those unlucky ones who want to recompile Smalltalk/X but have no access to stc and librun sources.
rakelib/compile.rake
rakelib/support.rb
specs/reports-misc.rbspec
specs/stx-jv.rbspec
--- a/rakelib/compile.rake	Wed May 25 13:50:49 2016 +0100
+++ b/rakelib/compile.rake	Wed May 25 17:19:27 2016 +0100
@@ -14,18 +14,6 @@
   LIBRUN = STX_TOP_DIR / 'librun' / 'librun.so'
 end
 
-# Returns true iff this is machine of one of the core developers...
-def core_developer_machine?
-  # JV's box: jv@..., vranyj1@...
-  if (ENV['USER'] == 'jv') or (ENV['USER'] == 'vranyj1')
-    return true
-  end
-  if (ENV['USERNAME'] == 'jv') or (ENV['USERNAME'] == 'vranyj1')
-    return true
-  end 
-  return false
-end
-
 # Return suitable gcc for compilation. For now, GCC 5.x.x produces wrong code,
 # likely because some garbage C generated.
 def gcc()
@@ -42,7 +30,7 @@
 end
 
 def should_remove_librun_and_stc_sources
-    return ! core_developer_machine?
+    return ! core_developer?
 end
 
 def make_or_raise_error(args = '')
@@ -95,9 +83,7 @@
     #unix specific
     'stx-config.sh',
     'linuxIntern.h',
-    'macIntern.h',
-    'makefile',
-    'Makefile',
+    'macIntern.h',        
     'Make.proto',
     'stc',
     'stc.1',    
@@ -196,7 +182,7 @@
 
 # Setup flags for GCC (both, real GCC and MinGW)
 GCC_CFLAGS_OPT = ARCH == 'i386' ? '-O' : ''
-GCC_CFLAGS_DBG = core_developer_machine? ? '-ggdb3' : ''
+GCC_CFLAGS_DBG = core_developer? ? '-ggdb3' : ''
 GCC_CFLAGS_PIC = win32? ? '' : '-fPIC'
 GCC_CFLAGS = "-pipe -fno-omit-frame-pointer -fno-stack-protector -fwrapv #{GCC_CFLAGS_PIC} #{GCC_CFLAGS_OPT} #{GCC_CFLAGS_DBG}"
 
--- a/rakelib/support.rb	Wed May 25 13:50:49 2016 +0100
+++ b/rakelib/support.rb	Wed May 25 17:19:27 2016 +0100
@@ -1,3 +1,6 @@
+require 'net/http'
+require 'net/https'
+require 'json'
 require 'rakelib/extensions.rb'
 require 'rakelib/rbspec.rb'
 require 'rakelib/vcs.rb'
@@ -25,5 +28,120 @@
 def jenkins?
   return (ENV.has_key? 'WORKSPACE'   and 
           ENV.has_key? 'JOB_NAME'    and 
-          ENV.has_key> 'BUILD_NAME')
+          ENV.has_key? 'BUILD_ID')
+end
+
+# Returns true if and only if this is machine of one of the core developers
+# (currently only JV). 
+# Indeed this is a feeble check and can be easily bypassed by chaning the
+# code or by setting a proper environment variable. But this should not hurt 
+# much as in that case, unauthorized person wouldn't be able to connect to 
+# stc and librun repository so the build will fail. 
+def core_developer?
+  # JV's box: jv@..., vranyj1@...
+  if (ENV['USER'] == 'jv') or (ENV['USER'] == 'vranyj1')
+    return true
+  end
+  if (ENV['USERNAME'] == 'jv') or (ENV['USERNAME'] == 'vranyj1')
+    return true
+  end 
+  return false
+end
+
+# A super simple API for Jenkins used to download pre-built stc and librun.
+module Jenkins
+  # Return an a Jenkins build with pre-built stc and librun. 
+  # Used to download pre-build stc and librun
+  def self.smalltalkx_jv_branch_build()
+    plat = nil
+    if win32? then        
+        plat = 'Windows'
+    elsif linux?        
+      plat = 'Linux'
+    else        
+      error_unsupported_platform()
+    end
+    return Jenkins::Build.new(%Q{https://swing.fit.cvut.cz/jenkins/job/stx_jv_new/ARCH=#{ARCH},PLATFORM=#{plat}N/lastSuccessfulBuild})
+  end
+
+  class Artifact
+    attr_reader :name
+    attr_reader :uri
+
+    def initialize(name, uri)
+      @name = name
+      @uri = uri
+    end
+
+    def download_to(destination)       
+      if not File.exist? destination
+        if not File.directory? File.dirname(destination)
+          raise Exception.new("Invalid destination for download: #{destination}")
+        end
+      else
+        if not File.directory? destination
+          raise Exception.new("Invalid destination for download: #{destination}")
+        else
+          destination = File.join(destination, @name)
+        end
+      end
+      Jenkins::get(@uri) do | response |
+        File.open(destination, "wb") do | file |
+          response.read_body do | part |
+            file.write part
+          end
+        end
+      end
+    end
+  end
+
+  class Build
+    attr_reader :data
+    attr_reader :uri
+
+    def initialize(uri)
+      @uri = uri
+      @data = JSON.parse(Jenkins::get(URI(uri.to_s + '/api/json')))
+    end
+
+    # Return a list of artifacts (as instances of `Jenkins::Artifact`) 
+    # associated with this build. 
+    def artifacts()
+      if not @artifacts then
+        @artifacts = @data["artifacts"].collect do | each | 
+          Artifact.new(each['fileName'], URI(@uri.to_s + '/artifact/' + each['relativePath']))
+        end
+      end
+      return @artifacts      
+    end
+  end
+
+  # A private method to GET data over HTTP(S)  
+  def self.get(uri, &block) 
+    # http parameters       
+    http_host = Net::HTTP.new(uri.host, uri.port)
+    http_host.use_ssl = (uri.scheme == 'https') # simple true enough
+
+    if false # host verification not used (yet)
+      http_host.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      http_host.cert_store = OpenSSL::X509::Store.new
+      http_host.cert_store.set_default_paths
+      http_host.cert_store.add_file('cacert.pem') # file downloaded from curl.haxx.se/ca/cacert.pem
+    else
+      http_host.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+
+    # actual download
+    body = nil
+    http_host.start do | http |        
+      http.request_get(uri) do | response |
+        if block then
+          yield response
+        else
+          body = response.body
+        end
+      end
+    end
+    return body
+  end
 end
\ No newline at end of file
--- a/specs/reports-misc.rbspec	Wed May 25 13:50:49 2016 +0100
+++ b/specs/reports-misc.rbspec	Wed May 25 17:19:27 2016 +0100
@@ -84,7 +84,7 @@
     end
 
     task :'test:post' do
-      if not core_developer_machine?
+      if not core_developer?
         rm_rf BUILD_DIR / 'exept' / 'regression'
       end
     end
--- a/specs/stx-jv.rbspec	Wed May 25 13:50:49 2016 +0100
+++ b/specs/stx-jv.rbspec	Wed May 25 17:19:27 2016 +0100
@@ -1,5 +1,43 @@
 load "jv-branch.deps.rake"
 
+
+# A helper function to download and unpack pre-built stc and librun
+# for those who are not lucky enough to have an access to sources
+def download_blob_matching(pattern, directory)
+  plat = nil
+  blob = nil
+  sha256 = nil
+  if win32? then        
+      plat = 'Windows'
+  elsif linux?        
+    plat = 'Linux'
+  else        
+    error_unsupported_platform()
+  end
+  build = Jenkins::Build.new(%Q{https://swing.fit.cvut.cz/jenkins/job/stx_jv_new/ARCH=#{ARCH},PLATFORM=#{plat}N/lastSuccessfulBuild})
+  build.artifacts.each do | artifact |
+    if pattern =~ artifact.name then
+      if artifact.name.end_with? '.sha256' then
+        sha256 = artifact
+      else
+        blob = artifact
+      end
+    end
+  end
+  if not blob then
+    error "No artifact matching given pattern found"
+  end
+  puts "Downloading binary component #{blob.name}"
+  blob.download_to(directory)
+  if sha256 then
+    sha256.download_to(directory)
+  end
+  unzip directory / blob.name, remove: true
+  rm_f directory / sha256.name
+end
+
+
+
 project :'stx:jv-branch-core' do
   # Core Smalltalk/X - does contain only standard libraries,
   # and development tools. Does not contain any other 'features'
@@ -51,30 +89,18 @@
     end
 
     redefine BUILD_DIR / 'stx' / 'stc' => BUILD_DIR do
-      if core_developer_machine? then
+      if core_developer? or jenkins? then
         checkout :'swing:private:hg', 'stx/stc', :branch => 'jv'
       else
-        lastSicessfulBuild = Jenkins::Build.new(%q{https://swing.fit.cvut.cz/jenkins/job/stx_jv_new/ARCH=i386,PLATFORM=LinuxN/lastSuccessfulBuild})
-        lastSicessfulBuild.artifacs(/prebuilt\-stc/).each do | each |
-          if not each.name.end_with? '.sha256' then
-            each.download_to(BUILD_DIR / 'stx')
-            unzip BUILD_DIR / 'stx' / each.name
-          end
-        end
+        download_blob_matching(/prebuilt-stc/, BUILD_DIR / 'stx')
       end
     end
 
     redefine BUILD_DIR / 'stx' / 'librun' => BUILD_DIR do      
-      if core_developer_machine? then
+      if core_developer? or jenkins? then
         checkout :'swing:private:hg', 'stx/librun', :branch => 'jv'
       else        
-        lastSicessfulBuild = Jenkins::Build.new(%q{https://swing.fit.cvut.cz/jenkins/job/stx_jv_new/ARCH=i386,PLATFORM=LinuxN/lastSuccessfulBuild})
-        lastSicessfulBuild.artifacs(/prebuilt\-librun/).each do | each |
-          if not each.name.end_with? '.sha256' then
-            each.download_to(BUILD_DIR / 'stx')
-            unzip BUILD_DIR / 'stx' / each.name
-          end
-        end
+        download_blob_matching(/prebuilt-librun/, BUILD_DIR / 'stx')
       end
     end
 
@@ -374,14 +400,14 @@
       # Just to be sure, refuse to create stc and librun archives
       # if the directory contain sources. This is to prevent accidental
       # leak of non-disclosed sources. Better safe than sorry!
-      if core_developer_machine? || (File.exist? BUILD_DIR / 'stx' / 'stc' / '.hg') || (File.exist? BUILD_DIR / 'stx' / 'stc' / 'CVS') then
+      if core_developer? || (File.exist? BUILD_DIR / 'stx' / 'stc' / '.hg') || (File.exist? BUILD_DIR / 'stx' / 'stc' / 'CVS') then
         puts "WARNING: NOT CREATING stc archive since directory contains sources!!!"
       else
         # Be paranoid, do cleanup here. Should have been done, but who knows...
         cleanup_stc() 
     	  zip BUILD_DIR / 'stx' / 'stc' , archive: ARTIFACTS_DIR / "#{project.app_name}-#{project.app_version}_#{ARCH}-#{win32? ? 'win32' : RbConfig::CONFIG['host_os']}_prebuilt-stc"
       end
-      if core_developer_machine? || (File.exist? BUILD_DIR / 'stx' / 'librun' / '.hg') || (File.exist? BUILD_DIR / 'stx' / 'librun' / 'CVS') then
+      if core_developer? || (File.exist? BUILD_DIR / 'stx' / 'librun' / '.hg') || (File.exist? BUILD_DIR / 'stx' / 'librun' / 'CVS') then
         puts  "WARNING: NOT CREATING librun archive since directory contains sources!!!"
       else
         # Be paranoid, do cleanup here. Should have been done, but who knows...