draft build concurrency limits

Gemfile

@@ -4,6 +4,7 @@ ruby '2.6.5' if ENV['DYNO']
 
 gem 'travis-config',      '~> 1.1.3'
 gem 'travis-lock'
+gem 'travis-conditions',  '~> 1.0.10'
 gem 'travis-metrics',     git: 'https://github.com/travis-ci/travis-metrics'
 gem 'travis-rollout',     git: 'https://github.com/travis-ci/travis-rollout'
 gem 'travis-exceptions',  git: 'https://github.com/travis-ci/travis-exceptions'

Gemfile.lock

@@ -151,6 +151,7 @@ GEM
     multipart-post (2.0.0)
     net-http-persistent (2.9.4)
     net-http-pipeline (1.0.1)
+    parslet (1.8.2)
     pg (0.21.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
@@ -182,6 +183,7 @@ GEM
     safe_yaml (1.0.4)
     sentry-raven (2.6.3)
       faraday (>= 0.7.6, < 1.0)
+    sh_vars (1.0.2)
     sidekiq (4.2.6)
       concurrent-ruby (~> 1.0)
       connection_pool (~> 2.2, >= 2.2.0)
@@ -190,6 +192,9 @@ GEM
     sidekiq-pro (3.4.0)
       sidekiq (>= 4.1.5)
     thread_safe (0.3.6)
+    travis-conditions (1.0.13)
+      parslet (~> 1.8.2)
+      sh_vars (~> 1.0.2)
     travis-config (1.1.3)
       hashr (~> 2.0)
     travis-lock (0.1.1)
@@ -237,6 +242,7 @@ DEPENDENCIES
   rspec
   sentry-raven
   sidekiq-pro!
+  travis-conditions (~> 1.0.10)
   travis-config (~> 1.1.3)
   travis-exceptions!
   travis-lock

lib/travis/scheduler/helper/condition.rb

@@ -0,0 +1,153 @@
+require 'travis/conditions'
+
+module Travis
+  module Scheduler
+    class Condition
+      attr_reader :cond, :job, :conf, :data
+
+      def initialize(cond, job)
+        @cond = cond.is_a?(Array) || cond.is_a?(Hash) || cond.nil? ? cond : cond.to_s
+        @conf = job.config
+        @data = Data.new(job, conf).to_h
+      end
+
+      def applies?
+        return true if cond.nil?
+        version = conf[:conditions].to_s == 'v0' ? :v0 : :v1
+        Travis::Conditions.eval(cond, data, version: version)
+      rescue Travis::Conditions::Error, ArgumentError, RegexpError, TypeError => e
+        Gatekeeper.logger.error "Unable to process condition: #{cond.inspect}. (#{e.message})"
+        # the old implentation returned `true` on parse errors, unfortunately
+        return true if version == :v0
+        raise BadCondition, e.message
+      end
+
+      def to_s
+        "IF #{cond}"
+      end
+
+      class Data < Struct.new(:job, :conf)
+        def to_h
+          {
+            type: event,
+            repo: repo.slug,
+            head_repo: head_repo,
+            os: os,
+            dist: dist,
+            group: group,
+            sudo: sudo.to_s, # this can be a boolean once we're on v1
+            language: language,
+            sender: sender,
+            fork: fork?.to_s,
+            branch: branch,
+            head_branch: head_branch,
+            tag: tag,
+            commit_message: commit_message,
+            env: env
+          }
+        end
+
+        def repo
+          job.repository
+        end
+
+        def request
+          job.source.request
+        end
+
+        def event
+          request.event_type.to_s
+        end
+
+        def head_repo
+          pull_request&.head_repo_slug
+        end
+
+        def os
+          conf[:os]
+        end
+
+        def dist
+          conf[:dist]
+        end
+
+        def group
+          conf[:group]
+        end
+
+        def sudo
+          conf[:sudo]
+        end
+
+        def language
+          conf[:language]
+        end
+
+        def commit_message
+          commit&.message
+        end
+
+        def fork?
+          pull_request ? repo.slug != pull_request.head_repo_slug : repo.fork?
+        end
+
+        def branch
+          commit.branch
+        end
+
+        def head_branch
+          pull_request&.head_ref
+        end
+
+        def tag
+          commit.tag && commit.tag.name
+        end
+
+        def sender
+          request.sender&.login
+        end
+
+        def pull_request
+          request.pull_request
+        end
+
+        def commit
+          job.commit
+        end
+
+        def env
+          env = conf[:env] || {}
+          env = env[:global] if global?(env)
+          env = to_strings(env)
+          env = settings_env + global_env + env
+          env.compact.flatten
+        end
+
+        def global_env
+          Array(to_strings(conf[:global_env]))
+        end
+
+        def settings_env
+          repo.settings.env_vars.map { |v| "#{v.name}=#{v.value.decrypt}" }
+        end
+
+        def to_strings(env)
+          return Array(env) unless env.is_a?(Hash)
+          env.map { |key, value| [key, value].join('=') }
+        end
+
+        def global?(env)
+          env.is_a?(Hash) && [Hash, Array].include?(env[:global].class)
+        end
+
+        def conf
+          @conf ||= super || {}
+        end
+
+        def compact(hash)
+          hash.reject { |_, value| value.nil? }.to_h
+        end
+      end
+    end
+  end
+end

lib/travis/scheduler/jobs/limit/build.rb

@@ -0,0 +1,49 @@
+require 'travis/scheduler/helper/condition'
+
+module Travis
+  module Scheduler
+    module Jobs
+      module Limit
+        class Build < Base
+          def accept?(job)
+            return true unless applies?(job) && max(job)
+            super
+          end
+
+          private
+
+            def max(job)
+              return unless applies?(job)
+              num = job.repository.settings.maximum_number_of_builds
+              num > 0 ? num : nil
+            end
+
+            def applies?(job)
+              # remove this once the setting for concurrent jobs has been renamed
+              return unless cond = condition(job)
+              Condition.new(cond, job).applies?
+            end
+
+            def condition(job)
+              job.repository.settings.maximum_number_of_builds_condition
+            end
+
+            def running(job)
+              state.running.select(&method(:applies?)).size
+            end
+
+            def report(status, job)
+              {
+                type: :limit,
+                name: :build,
+                status: status,
+                repo_slug: job.repository.slug,
+                id: job.id,
+                max: max(job)
+              }
+            end
+        end
+      end
+    end
+  end
+end

lib/travis/scheduler/jobs/limit/repo.rb

@@ -11,7 +11,10 @@ def accept?(job)
           private
 
             def max(job)
-              num = job.repository.settings.maximum_number_of_builds
+              settings = job.repository.settings
+              # rename the setting to maximum_number_of_jobs, and remove this
+              return if settings.maximum_number_of_builds_condition
+              num = settings.maximum_number_of_builds
               num > 0 ? num : nil
             end
 

lib/travis/scheduler/jobs/limits.rb

@@ -1,5 +1,6 @@
 require 'travis/scheduler/helper/memoize'
 require 'travis/scheduler/jobs/limit/base'
+require 'travis/scheduler/jobs/limit/build'
 require 'travis/scheduler/jobs/limit/queue'
 require 'travis/scheduler/jobs/limit/repo'
 require 'travis/scheduler/jobs/limit/stages'
@@ -12,7 +13,7 @@ class Limits < Struct.new(:context, :owners, :state)
 
         # These are ordered by how specific the limit is, from most specific to least.
         # In other orders, we may apply a stricter limit than is intended.
-        NAMES = %w(stages repo queue)
+        NAMES = %w(stages build repo queue)
 
         def accept(job)
           yield job if accept?(job)

lib/travis/scheduler/jobs/reports/limits.rb

@@ -7,7 +7,8 @@ module Reports
         class Limits < Struct.new(:owners, :reports)
           MSGS = {
             queue:  '%s limited by queue %s: max=%s rejected=%s selected=%s',
-            repo:   'repo %s limited by repo settings: max=%s rejected=%s selected=%s',
+            repo:   'repo %s limited by repo max jobs settings: max=%s rejected=%s selected=%s',
+            build:  'repo %s limited by repo max builds settings: max=%s rejected=%s selected=%s',
             stages: 'repo %s limited by stage on build_id=%s: rejected=%s selected=%s',
           }
 
@@ -31,6 +32,12 @@ def report_repo(data)
               end
             end
 
+            def report_build(data)
+              map_repos(data) do |slug, data|
+                msg :build, slug, data[0][:max], rejected(data), selected(data)
+              end
+            end
+
             def report_stages(data)
               map_repos(data) do |slug, data|
                 map_builds(data) do |build_id, data|

lib/travis/scheduler/jobs/state.rb

@@ -10,7 +10,7 @@ class State < Struct.new(:context, :owners)
         include FilterMigratedJobs
 
         ATTRS = {
-          running:  %i(repository_id private queue org_id restarted_at),
+          running:  %i(repository_id source_id source_type commit_id private queue org_id restarted_at),
           by_build: %i(id state stage_number)
         }
 
@@ -38,15 +38,21 @@ def count_running_by_repo(id)
           counts[:repo][id] ||= running.select { |job| job.repository_id == id }.size
         end
 
+        def count_running_by_build(id)
+          counts[:build][id] ||= running.select { |job| job.source_id == id }.size
+        end
+
         def count_running_by_queue(name)
           counts[:queue][name] ||= running.select { |job| job.queue == name }.size
         end
 
         private
 
           def read_running
-            result = Job.by_owners(owners.all).running.select(*ATTRS[:running]).includes(:repository).to_a
-            filter_migrated_jobs(result)
+            result = Job.by_owners(owners.all).running
+            result = result.includes(:repository, commit: :tag, source: :request)
+            result = result.select(*ATTRS[:running])
+            filter_migrated_jobs(result.to_a)
           end
           time :read_queueable, key: 'scheduler.running_jobs'
 
@@ -60,7 +66,7 @@ def cache
           end
 
           def counts
-            @counts ||= { repo: {}, queue: {} }
+            @counts ||= { repo: {}, build: {}, queue: {} }
           end
       end
     end

lib/travis/scheduler/record.rb

@@ -16,5 +16,6 @@
 require 'travis/scheduler/record/ssl_key'
 require 'travis/scheduler/record/stage'
 require 'travis/scheduler/record/subscription'
+require 'travis/scheduler/record/tag'
 require 'travis/scheduler/record/trial'
 require 'travis/scheduler/record/user'

lib/travis/scheduler/record/commit.rb

@@ -1,4 +1,5 @@
 class Commit < ActiveRecord::Base
   has_one :request
   belongs_to :repository
+  belongs_to :tag
 end

lib/travis/scheduler/record/repository/settings.rb

@@ -87,6 +87,7 @@ def custom_timeouts?(settings)
   attribute :build_pushes, Boolean, default: true
   attribute :build_pull_requests, Boolean, default: true
   attribute :maximum_number_of_builds, Integer
+  attribute :maximum_number_of_builds_condition, String
   attribute :ssh_key, SshKey
   attribute :timeout_hard_limit
   attribute :timeout_log_silence

lib/travis/scheduler/record/request.rb

@@ -9,6 +9,7 @@ class Request < ActiveRecord::Base
   belongs_to :pull_request
   belongs_to :repository
   belongs_to :owner, polymorphic: true
+  belongs_to :sender, polymorphic: true
 
   serialize :payload
 

lib/travis/scheduler/record/tag.rb

@@ -0,0 +1,2 @@
+class Tag < ActiveRecord::Base
+end