Merge branch 'import-timeout' of https://dev.gitlab.org/dzaporozhets/gitlabhq into dzaporozhets/gitlabhq-import-timeout

Signed-off-by: Dmitriy Zaporozhets <[email protected]>

Conflicts:
	CHANGELOG
	db/schema.rb

Author: dzaporozhets

CHANGELOG

@@ -17,6 +17,7 @@ v 6.7.0
   - Add GFM autocompletion for MergeRequests (Robert Speicher)
   - Add webhook when a new tag is pushed (Jeroen van Baarsen)
   - Add button for toggling inline comments in diff view
+  - Add retry feature for repository import
 
 v 6.6.2
   - Fix 500 error on branch/tag create or remove via UI

app/controllers/projects_controller.rb

@@ -5,7 +5,7 @@ class ProjectsController < ApplicationController
 
   # Authorize
   before_filter :authorize_read_project!, except: [:index, :new, :create]
-  before_filter :authorize_admin_project!, only: [:edit, :update, :destroy, :transfer, :archive, :unarchive]
+  before_filter :authorize_admin_project!, only: [:edit, :update, :destroy, :transfer, :archive, :unarchive, :retry_import]
   before_filter :require_non_empty_project, only: [:blob, :tree, :graph]
 
   layout 'navless', only: [:new, :create, :fork]
@@ -21,16 +21,9 @@ def edit
 
   def create
     @project = ::Projects::CreateService.new(current_user, params[:project]).execute
+    flash[:notice] = 'Project was successfully created.' if @project.saved?
 
     respond_to do |format|
-      flash[:notice] = 'Project was successfully created.' if @project.saved?
-      format.html do
-        if @project.saved?
-          redirect_to @project
-        else
-          render "new"
-        end
-      end
       format.js
     end
   end
@@ -55,6 +48,11 @@ def transfer
   end
 
   def show
+    if @project.import_in_progress?
+      redirect_to import_project_path(@project)
+      return
+    end
+
     return authenticate_user! unless @project.public? || current_user
 
     limit = (params[:limit] || 20).to_i
@@ -67,16 +65,36 @@ def show
         if @project.empty_repo?
           render "projects/empty", layout: user_layout
         else
-          if current_user
-            @last_push = current_user.recent_push(@project.id)
-          end
+          @last_push = current_user.recent_push(@project.id) if current_user
           render :show, layout: user_layout
         end
       end
       format.json { pager_json("events/_events", @events.count) }
     end
   end
 
+  def import
+    if project.import_finished?
+      redirect_to @project
+      return
+    end
+  end
+
+  def retry_import
+    unless @project.import_failed?
+      redirect_to import_project_path(@project)
+    end
+
+    @project.import_url = params[:project][:import_url]
+
+    if @project.save
+      @project.reload
+      @project.import_retry
+    end
+
+    redirect_to import_project_path(@project)
+  end
+
   def destroy
     return access_denied! unless can?(current_user, :remove_project, project)
 

app/models/project.rb

@@ -28,7 +28,6 @@ class Project < ActiveRecord::Base
   include Gitlab::VisibilityLevel
   extend Enumerize
 
-  default_value_for :imported, false
   default_value_for :archived, false
 
   ActsAsTaggableOn.strict_case_match = true
@@ -59,13 +58,10 @@ class Project < ActiveRecord::Base
   has_one :gemnasium_service, dependent: :destroy
   has_one :forked_project_link, dependent: :destroy, foreign_key: "forked_to_project_id"
   has_one :forked_from_project, through: :forked_project_link
-
   # Merge Requests for target project should be removed with it
   has_many :merge_requests,     dependent: :destroy, foreign_key: "target_project_id"
-
   # Merge requests from source project should be kept when source project was removed
   has_many :fork_merge_requests, foreign_key: "source_project_id", class_name: MergeRequest
-
   has_many :issues, -> { order "state DESC, created_at DESC" }, dependent: :destroy
   has_many :services,           dependent: :destroy
   has_many :events,             dependent: :destroy
@@ -74,10 +70,8 @@ class Project < ActiveRecord::Base
   has_many :snippets,           dependent: :destroy, class_name: "ProjectSnippet"
   has_many :hooks,              dependent: :destroy, class_name: "ProjectHook"
   has_many :protected_branches, dependent: :destroy
-
   has_many :users_projects, dependent: :destroy
   has_many :users, through: :users_projects
-
   has_many :deploy_keys_projects, dependent: :destroy
   has_many :deploy_keys, through: :deploy_keys_projects
 
@@ -97,15 +91,12 @@ class Project < ActiveRecord::Base
   validates :issues_enabled, :wall_enabled, :merge_requests_enabled,
             :wiki_enabled, inclusion: { in: [true, false] }
   validates :issues_tracker_id, length: { maximum: 255 }, allow_blank: true
-
   validates :namespace, presence: true
   validates_uniqueness_of :name, scope: :namespace_id
   validates_uniqueness_of :path, scope: :namespace_id
-
   validates :import_url,
     format: { with: URI::regexp(%w(git http https)), message: "should be a valid url" },
     if: :import?
-
   validate :check_limit, on: :create
 
   # Scopes
@@ -118,14 +109,36 @@ class Project < ActiveRecord::Base
   scope :sorted_by_activity, -> { reorder("projects.last_activity_at DESC") }
   scope :personal, ->(user) { where(namespace_id: user.namespace_id) }
   scope :joined, ->(user) { where("namespace_id != ?", user.namespace_id) }
-
   scope :public_only, -> { where(visibility_level: Project::PUBLIC) }
   scope :public_and_internal_only, -> { where(visibility_level: Project.public_and_internal_levels) }
-
   scope :non_archived, -> { where(archived: false) }
 
   enumerize :issues_tracker, in: (Gitlab.config.issues_tracker.keys).append(:gitlab), default: :gitlab
 
+  state_machine :import_status, initial: :none do
+    event :import_start do
+      transition :none => :started
+    end
+
+    event :import_finish do
+      transition :started => :finished
+    end
+
+    event :import_fail do
+      transition :started => :failed
+    end
+
+    event :import_retry do
+      transition :failed => :started
+    end
+
+    state :started
+    state :finished
+    state :failed
+
+    after_transition any => :started, :do => :add_import_job
+  end
+
   class << self
     def public_and_internal_levels
       [Project::PUBLIC, Project::INTERNAL]
@@ -202,12 +215,28 @@ def saved?
     id && persisted?
   end
 
+  def add_import_job
+    RepositoryImportWorker.perform_in(2.seconds, id)
+  end
+
   def import?
     import_url.present?
   end
 
   def imported?
-    imported
+    import_finished?
+  end
+
+  def import_in_progress?
+    import? && import_status == 'started'
+  end
+
+  def import_failed?
+    import_status == 'failed'
+  end
+
+  def import_finished?
+    import_status == 'finished'
   end
 
   def check_limit

app/observers/project_observer.rb

@@ -1,30 +1,6 @@
 class ProjectObserver < BaseObserver
   def after_create(project)
-    project.update_column(:last_activity_at, project.created_at)
-
-    return true if project.forked?
-
-    if project.import?
-      RepositoryImportWorker.perform_in(5.seconds, project.id)
-    else
-      GitlabShellWorker.perform_async(
-        :add_repository,
-        project.path_with_namespace
-      )
-
-      log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
-    end
-
-    if project.wiki_enabled?
-      begin
-        # force the creation of a wiki,
-        GollumWiki.new(project, project.owner).wiki
-      rescue GollumWiki::CouldNotCreateWikiError => ex
-        # Prevent project observer crash
-        # if failed to create wiki
-        nil
-      end
-    end
+    log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
   end
 
   def after_update(project)

app/services/projects/create_service.rb

@@ -58,6 +58,29 @@ def execute
             user: current_user
           )
         end
+
+        @project.update_column(:last_activity_at, @project.created_at)
+
+        if @project.import?
+          @project.import_start
+        else
+          GitlabShellWorker.perform_async(
+            :add_repository,
+            @project.path_with_namespace
+          )
+
+        end
+
+        if @project.wiki_enabled?
+          begin
+            # force the creation of a wiki,
+            GollumWiki.new(@project, @project.owner).wiki
+          rescue GollumWiki::CouldNotCreateWikiError => ex
+            # Prevent project observer crash
+            # if failed to create wiki
+            nil
+          end
+        end
       end
 
       @project

app/views/projects/create.js.haml

@@ -1,6 +1,10 @@
 - if @project.saved?
-  :plain
-    location.href = "#{project_path(@project)}";
+  - if @project.import?
+    :plain
+      location.href = "#{import_project_path(@project)}";
+  - else
+    :plain
+      location.href = "#{project_path(@project)}";
 - else
   :plain
     $(".project-edit-errors").html("#{escape_javascript(render('errors'))}");

app/views/projects/empty.html.haml

@@ -1,46 +1,34 @@
 = render "home_panel"
 
-- if @project.import? && [email protected]
-  .save-project-loader
-    %center
-      %h2
-        %i.icon-spinner.icon-spin
-        Importing repository.
-      %p.monospace git clone --bare #{@project.import_url}
-      %p Please wait while we import the repository for you. Refresh at will.
-      :javascript
-        new ProjectImport();
+%div.git-empty
+  %fieldset
+    %legend Git global setup:
+    %pre.dark
+      :preserve
+        git config --global user.name "#{git_user_name}"
+        git config --global user.email "#{git_user_email}"
 
-- else
-  %div.git-empty
-    %fieldset
-      %legend Git global setup:
-      %pre.dark
-        :preserve
-          git config --global user.name "#{git_user_name}"
-          git config --global user.email "#{git_user_email}"
+  %fieldset
+    %legend Create Repository
+    %pre.dark
+      :preserve
+        mkdir #{@project.path}
+        cd #{@project.path}
+        git init
+        touch README
+        git add README
+        git commit -m 'first commit'
+        git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
+        git push -u origin master
 
-    %fieldset
-      %legend Create Repository
-      %pre.dark
-        :preserve
-          mkdir #{@project.path}
-          cd #{@project.path}
-          git init
-          touch README
-          git add README
-          git commit -m 'first commit'
-          git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
-          git push -u origin master
+  %fieldset
+    %legend Existing Git Repo?
+    %pre.dark
+      :preserve
+        cd existing_git_repo
+        git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
+        git push -u origin master
 
-    %fieldset
-      %legend Existing Git Repo?
-      %pre.dark
-        :preserve
-          cd existing_git_repo
-          git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
-          git push -u origin master
-
-  - if can? current_user, :remove_project, @project
-    .prepend-top-20
-      = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right"
+- if can? current_user, :remove_project, @project
+  .prepend-top-20
+    = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right"

app/views/projects/import.html.haml

@@ -0,0 +1,30 @@
+- if @project.import_in_progress?
+  .save-project-loader
+    %center
+      %h2
+        %i.icon-spinner.icon-spin
+        Import in progress.
+      %p.monospace git clone --bare #{@project.import_url}
+      %p Please wait while we import the repository for you. Refresh at will.
+      :javascript
+        new ProjectImport();
+
+- elsif @project.import_failed?
+  .save-project-loader
+    %center
+      %h2
+        Import failed. Retry?
+  %hr
+  - if can?(current_user, :admin_project, @project)
+    = form_for @project, url: retry_import_project_path(@project), method: :put, html: { class: 'form-horizontal' } do |f|
+      .form-group.import-url-data
+        = f.label :import_url, class: 'control-label' do
+          %span Import existing repo
+        .col-sm-10
+          = f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git'
+          .bs-callout.bs-callout-info
+            This url must be publicly accessible or you can add a username and password like this: https://username:[email protected]/company/project.git.
+            %br
+            The import will time out after 2 minutes. For big repositories, use a clone/push combination.
+      .form-actions
+        = f.submit 'Retry import', class: "btn btn-create", tabindex: 4