diff --git a/.gitignore b/.gitignore index 262403e49..21d8a9803 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +output/ # Leiningen /target/ diff --git a/tasks/build.rake b/tasks/build.rake index e19587020..669470d73 100644 --- a/tasks/build.rake +++ b/tasks/build.rake @@ -12,11 +12,30 @@ require 'tmpdir' # To be fixed one of these days. Relevant stuff: # https://github.com/puppetlabs/ezbake/blob/aeb7735a16d2eecd389a6bd9e5c0cfc7c62e61a5/resources/puppetlabs/lein-ezbake/template/global/tasks/build.rake # https://github.com/puppetlabs/ezbake/blob/aeb7735a16d2eecd389a6bd9e5c0cfc7c62e61a5/resources/puppetlabs/lein-ezbake/template/global/ext/fpm.rb -deb_platforms = ENV['DEB_PLATFORMS'] || 'ubuntu-20.04,ubuntu-22.04,ubuntu-24.04,debian-10,debian-11,debian-12' -rpm_platforms = ENV['RPM_PLATFORMS'] || 'el-8,el-9,el-10,sles-15,amazon-2023' +deb_platforms = ENV['DEB_PLATFORMS'] || 'ubuntu-20.04,ubuntu-22.04,ubuntu-24.04,ubuntu-25.04,debian-11,debian-12,debian-13' +rpm_platforms = ENV['RPM_PLATFORMS'] || 'el-8,el-9,el-10,sles-15,sles-16,amazon-2,amazon-2023,fedora-42,fedora-43' @debs = deb_platforms.split(',').map{ |p| "base-#{p.split('-').join}-i386.cow" }.join(' ') @rpms = rpm_platforms.split(',').map{ |p| "pl-#{p}-x86_64" }.join(' ') +# The deps must be built in this order due to dependencies between them. +# There is a circular dependency between clj-http-client and trapperkeeper-webserver-jetty10, +# but only for tests, so the build *should* work. +DEP_BUILD_ORDER = [ + 'clj-parent', + 'clj-kitchensink', + 'clj-i18n', + 'comidi', + 'jvm-ssl-utils', + 'trapperkeeper', + 'trapperkeeper-filesystem-watcher', + 'trapperkeeper-webserver-jetty10', + 'trapperkeeper-authorization', + 'trapperkeeper-metrics', + 'trapperkeeper-status', + 'stockpile', + 'structured-logging', +].freeze + def image_exists !`docker images -q #{@image}`.strip.empty? end @@ -34,7 +53,7 @@ def teardown end def start_container(ezbake_dir) - run_command("docker run -d --name #{@container} -v .:/code -v #{ezbake_dir}:/ezbake #{@image} /bin/sh -c 'tail -f /dev/null'", silent: false, print_command: true) + run_command("docker run -d --name #{@container} -v .:/code -v #{ezbake_dir}:/deps #{@image} /bin/sh -c 'tail -f /dev/null'", silent: false, print_command: true) end def run(cmd) @@ -60,24 +79,60 @@ namespace :vox do run_command("docker build -t ezbake-builder .", silent: false, print_command: true) end - puts "Checking out ezbake" - tmp = Dir.mktmpdir("ezbake") - ezbake_dir = "#{tmp}/ezbake" - ezbake_repo = ENV.fetch('EZBAKE_REPO', 'https://github.com/openvoxproject/ezbake') - ezbake_branch = ENV['EZBAKE_BRANCH'] || 'main' - run_command("git clone -b #{ezbake_branch} #{ezbake_repo} #{ezbake_dir}", silent: false, print_command: true) + libs_to_build_manually = {} + if ENV['EZBAKE_BRANCH'] && !ENV['EZBAKE_BRANCH'].strip.empty? + libs_to_build_manually['ezbake'] = { + :repo => ENV.fetch('EZBAKE_REPO', 'https://github.com/openvoxproject/ezbake'), + :branch => ENV.fetch('EZBAKE_BRANCH', 'main'), + } + end + + deps_to_build = [] + dep_branch = nil + + full_rebuild_branch = ENV['FULL_DEP_REBUILD_BRANCH'] + subset_list = (ENV['DEP_REBUILD'] || '').split(',').map(&:strip).reject(&:empty?) + subset_branch = ENV.fetch('DEP_REBUILD_BRANCH', 'main').to_s + rebuild_org = ENV.fetch('DEP_REBUILD_ORG', 'openvoxproject').to_s + + if full_rebuild_branch && !full_rebuild_branch.strip.empty? + dep_branch = full_rebuild_branch.strip + deps_to_build = DEP_BUILD_ORDER.dup + elsif !subset_list.empty? + dep_branch = subset_branch + unknown = subset_list.reject { |lib| DEP_BUILD_ORDER.include?(lib) } + puts "WARNING: Unknown deps in DEP_REBUILD (will be ignored): #{unknown.join(', ')}" unless unknown.empty? + deps_to_build = DEP_BUILD_ORDER.select { |lib| subset_list.include?(lib) } + end + + deps_to_build.each do |lib| + libs_to_build_manually[lib] = { + :repo => "https://github.com/#{rebuild_org}/#{lib}", + :branch => dep_branch, + } + end + + deps_tmp = Dir.mktmpdir("deps") + + libs_to_build_manually.each do |lib, config| + puts "Checking out #{lib}" + run_command("git clone -b #{config[:branch]} #{config[:repo]} #{deps_tmp}/#{lib}", silent: false, print_command: true) + end puts "Starting container" teardown if container_exists - start_container(ezbake_dir) + start_container(deps_tmp) - puts "Installing ezbake from source" - run("cd /ezbake && lein install") + libs_to_build_manually.each do |lib, _| + puts "Building and installing #{lib} from source" + run("cd /deps/#{lib} && lein install") + end + fips = !ENV['FIPS'].nil? puts "Building openvoxdb" ezbake_version_var = ENV['EZBAKE_VERSION'] ? "EZBAKE_VERSION=#{ENV['EZBAKE_VERSION']}" : '' run("cd /code && rm -rf ruby && rm -rf output && bundle install --without test && lein install") - run("cd /code && COW=\"#{@debs}\" MOCK=\"#{@rpms}\" GEM_SOURCE='https://rubygems.org' #{ezbake_version_var} EZBAKE_ALLOW_UNREPRODUCIBLE_BUILDS=true EZBAKE_NODEPLOY=true LEIN_PROFILES=ezbake lein with-profile user,ezbake,provided,internal ezbake local-build") + run("cd /code && COW=\"#{@debs}\" MOCK=\"#{@rpms}\" GEM_SOURCE='https://rubygems.org' #{ezbake_version_var} EZBAKE_ALLOW_UNREPRODUCIBLE_BUILDS=true EZBAKE_NODEPLOY=true LEIN_PROFILES=ezbake lein with-profile #{fips ? 'fips,' : ''}user,ezbake,provided,internal ezbake local-build") run_command("sudo chown -R $USER output", print_command: true) Dir.glob('output/**/*i386*').each { |f| FileUtils.rm_rf(f) } Dir.glob('output/puppetdb-*.tar.gz').each { |f| FileUtils.mv(f, f.sub('puppetdb','openvoxdb'))} diff --git a/tasks/upload.rake b/tasks/upload.rake index 4d88dcb85..8a834c5b8 100644 --- a/tasks/upload.rake +++ b/tasks/upload.rake @@ -1,6 +1,6 @@ namespace :vox do desc 'Upload artifacts from the output directory to S3. Requires the AWS CLI to be installed and configured appropriately.' - task :upload, [:tag, :platform] do |_, args| + task :upload, [:platform] do |_, args| endpoint = ENV.fetch('ENDPOINT_URL') bucket = ENV.fetch('BUCKET_NAME') component = 'openvoxdb' @@ -14,15 +14,23 @@ namespace :vox do abort 'You must set the ENDPOINT_URL environment variable to the S3 server you want to upload to.' if endpoint.nil? || endpoint.empty? abort 'You must set the BUCKET_NAME environment variable to the S3 bucket you are uploading to.' if bucket.nil? || bucket.empty? - abort 'You must provide a tag.' if args[:tag].nil? || args[:tag].empty? - munged_tag = args[:tag].gsub('-', '.') s3 = "aws s3 --endpoint-url=#{endpoint}" # Ensure the AWS CLI isn't going to fail with the given parameters run_command("#{s3} ls s3://#{bucket}/") - glob = "#{__dir__}/../output/**/*#{munged_tag}*" + config = File.expand_path("../target/staging/ezbake.rb", __dir__) + abort "Could not find ezbake config from the build at #{config}" unless File.exist?(config) + load config + version = EZBake::Config.fetch(:version) + release = EZBake::Config.fetch(:release) + # If release is a digit, then we built a tagged version. Otherwise, + # we built a snapshot and want to include that in the path to upload to. + tag = release =~ /^\d{1,2}$/ ? version : "#{version}-#{release}" + + + glob = "#{__dir__}/../output/**/*#{tag}*" if os # "arch" is not used here because it's all noarch glob += "#{os}*" @@ -30,7 +38,7 @@ namespace :vox do files = Dir.glob(glob) abort 'No files for the given tag found in the output directory.' if files.empty? - path = "s3://#{bucket}/#{component}/#{args[:tag]}" + path = "s3://#{bucket}/#{component}/#{tag}" files.each do |f| run_command("#{s3} cp #{f} #{path}/#{File.basename(f)}", silent: false) end