Github Importer

Gemfile

@@ -263,3 +263,5 @@ group :production do
 end
 
 gem "newrelic_rpm"
+
+gem 'octokit', '3.7.0'

Gemfile.lock

@@ -318,6 +318,8 @@ GEM
       jwt (~> 0.1.4)
       multi_json (~> 1.0)
       rack (~> 1.2)
+    octokit (3.7.0)
+      sawyer (~> 0.6.0, >= 0.5.3)
     omniauth (1.1.4)
       hashie (>= 1.2, < 3)
       rack
@@ -472,6 +474,9 @@ GEM
       sass (~> 3.2.0)
       sprockets (~> 2.8, <= 2.11.0)
       sprockets-rails (~> 2.0)
+    sawyer (0.6.0)
+      addressable (~> 2.3.5)
+      faraday (~> 0.8, < 0.10)
     sdoc (0.3.20)
       json (>= 1.1.3)
       rdoc (~> 3.10)
@@ -671,6 +676,7 @@ DEPENDENCIES
   mysql2
   newrelic_rpm
   nprogress-rails
+  octokit (= 3.7.0)
   omniauth (~> 1.1.3)
   omniauth-github
   omniauth-google-oauth2

app/controllers/github_imports_controller.rb

@@ -0,0 +1,75 @@
+class GithubImportsController < ApplicationController
+  before_filter :github_auth, except: :callback
+
+  rescue_from Octokit::Unauthorized, with: :github_unauthorized
+  
+  def callback
+    token = client.auth_code.get_token(params[:code]).token
+    current_user.github_access_token = token
+    current_user.save
+    redirect_to status_github_import_url
+  end
+
+  def status
+    @repos = octo_client.repos
+    octo_client.orgs.each do |org|
+      @repos += octo_client.repos(org.login)
+    end
+
+    @already_added_projects = current_user.created_projects.where(import_type: "github")
+    already_added_projects_names = @already_added_projects.pluck(:import_source)
+
+    @repos.reject!{|repo| already_added_projects_names.include? repo.full_name}
+  end
+
+  def create
+    @repo_id = params[:repo_id].to_i
+    repo = octo_client.repo(@repo_id)
+    target_namespace = params[:new_namespace].presence || repo.owner.login
+    existing_namespace = Namespace.find_by("path = ? OR name = ?", target_namespace, target_namespace)
+
+    if existing_namespace
+      if existing_namespace.owner == current_user
+        namespace = existing_namespace
+      else
+        @already_been_taken = true
+        @target_namespace = target_namespace
+        @project_name = repo.name
+        render and return
+      end
+    else
+      namespace = Group.create(name: target_namespace, path: target_namespace, owner: current_user)
+      namespace.add_owner(current_user)
+    end
+
+    Gitlab::Github::ProjectCreator.new(repo, namespace, current_user).execute
+  end
+
+  private
+
+  def client
+    @client ||= Gitlab::Github::Client.new.client
+  end
+
+  def octo_client
+    Octokit.auto_paginate = true
+    @octo_client ||= Octokit::Client.new(:access_token => current_user.github_access_token)
+  end
+
+  def github_auth
+    if current_user.github_access_token.blank?
+      go_to_gihub_for_permissions
+    end
+  end
+
+  def go_to_gihub_for_permissions
+    redirect_to client.auth_code.authorize_url({
+      redirect_uri: callback_github_import_url,
+      scope: "repo, user, user:email"
+    })
+  end
+
+  def github_unauthorized
+    go_to_gihub_for_permissions
+  end
+end

app/controllers/omniauth_callbacks_controller.rb

@@ -65,7 +65,7 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController
         redirect_to omniauth_error_path(oauth['provider'], error: error_message) and return
       end
     end
-  rescue ForbiddenAction => e
+  rescue Gitlab::OAuth::ForbiddenAction => e
     flash[:notice] = e.message
     redirect_to new_user_session_path
   end

app/helpers/projects_helper.rb

@@ -237,4 +237,20 @@ module ProjectsHelper
     result.password = '*****' if result.password.present?
     result
   end
+
+  def project_status_css_class(status)
+    case status
+    when "started"
+      "active"
+    when "failed"
+      "danger"
+    when "finished"
+      "success"
+    end
+  end
+
+  def github_import_enabled?
+    Gitlab.config.omniauth.enabled && enabled_oauth_providers.include?(:github)
+  end
 end
+

app/views/github_imports/create.js.haml

@@ -0,0 +1,18 @@
+- if @already_been_taken
+  :plain
+    target_field = $("tr#repo_#{@repo_id} .import-target")
+    origin_target = target_field.text()
+    project_name = "#{@project_name}"
+    origin_namespace = "#{@target_namespace}"
+    target_field.empty()
+    target_field.append("<p class='alert alert-danger'>This namespace already been taken! Please choose another one</p>")
+    target_field.append("<input type='text' name='target_namespace' />")
+    target_field.append("/" + project_name)
+    target_field.data("project_name", project_name)
+    target_field.find('input').prop("value", origin_namespace)
+- else
+  :plain
+    $("table.import-jobs tbody").prepend($("tr#repo_#{@repo_id}"))
+    $("tr#repo_#{@repo_id}").addClass("active").find(".import-actions").text("started")
+
+  

app/views/github_imports/status.html.haml

@@ -0,0 +1,41 @@
+%h3.page-title
+  Import repositories from github
+
+%hr
+%h4
+  Select projects you want to import.
+  
+%table.table.table-bordered.import-jobs
+  %thead
+    %tr
+      %th From GitHub
+      %th To GitLab
+      %th Status
+  %tbody
+    - @already_added_projects.each do |repo|
+      %tr{id: "repo_#{repo.id}", class: "#{project_status_css_class(repo.import_status)}"}
+        %td= repo.import_source
+        %td= repo.name_with_namespace
+        %td= repo.human_import_status_name
+    
+    - @repos.each do |repo|
+      %tr{id: "repo_#{repo.id}"}
+        %td= repo.full_name
+        %td.import-target 
+          = repo.full_name
+        %td.import-actions
+          = button_tag "Add", class: "btn btn-add-to-import"
+        
+
+:coffeescript
+  $(".btn-add-to-import").click () ->
+    new_namespace = null
+    tr = $(this).closest("tr")
+    id = tr.attr("id").replace("repo_", "")
+    if tr.find(".import-target input").length > 0
+      new_namespace = tr.find(".import-target input").prop("value")
+      tr.find(".import-target").empty().append(new_namespace + "/" + tr.find(".import-target").data("project_name"))
+    $.post "#{github_import_url}", {repo_id: id, new_namespace: new_namespace}, dataType: 'script'
+
+
+        

app/views/projects/new.html.haml

@@ -39,7 +39,15 @@
                 %br
                 The import will time out after 4 minutes. For big repositories, use a clone/push combination.
                 For SVN repositories, check #{link_to "this migrating from SVN doc.", "http://doc.gitlab.com/ce/workflow/migrating_from_svn.html"}
-      %hr
+      
+      - if github_import_enabled?
+        .project-import.form-group
+          .col-sm-2
+          .col-sm-10
+            %i.fa.fa-bars
+            = link_to "Import projects from github", status_github_import_path
+
+      %hr.prepend-botton-10
 
       .form-group
         = f.label :description, class: 'control-label' do

app/workers/repository_import_worker.rb

@@ -10,7 +10,13 @@ class RepositoryImportWorker
                                project.path_with_namespace,
                                project.import_url)
 
-    if result
+    if project.import_type == 'github'
+      result_of_data_import = Gitlab::Github::Importer.new(project).execute
+    else
+      result_of_data_import = true
+    end
+
+    if result && result_of_data_import
       project.import_finish
       project.save
       project.satellite.create unless project.satellite.exists?

config/routes.rb

@@ -51,6 +51,14 @@ Gitlab::Application.routes.draw do
   end
   get "/s/:username" => "snippets#user_index", as: :user_snippets, constraints: { username: /.*/ }
 
+  #
+  # Github importer area
+  #
+  resource :github_import, only: [:create, :new] do
+    get :status
+    get :callback
+  end
+
   #
   # Explroe area
   #

db/migrate/20141223135007_add_import_data_to_project_table.rb

@@ -0,0 +1,8 @@
+class AddImportDataToProjectTable < ActiveRecord::Migration
+  def change
+    add_column :projects, :import_type, :string
+    add_column :projects, :import_source, :string
+
+    add_column :users, :github_access_token, :string
+  end
+end

db/schema.rb

@@ -314,6 +314,8 @@ ActiveRecord::Schema.define(version: 20141226080412) do
     t.string   "import_status"
     t.float    "repository_size",        default: 0.0
     t.integer  "star_count",             default: 0,        null: false
+    t.string   "import_type"
+    t.string   "import_source"
   end
 
   add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree
@@ -411,6 +413,7 @@ ActiveRecord::Schema.define(version: 20141226080412) do
     t.integer  "notification_level",       default: 1,     null: false
     t.datetime "password_expires_at"
     t.integer  "created_by_id"
+    t.datetime "last_credential_check_at"
     t.string   "avatar"
     t.string   "confirmation_token"
     t.datetime "confirmed_at"
@@ -418,7 +421,7 @@ ActiveRecord::Schema.define(version: 20141226080412) do
     t.string   "unconfirmed_email"
     t.boolean  "hide_no_ssh_key",          default: false
     t.string   "website_url",              default: "",    null: false
-    t.datetime "last_credential_check_at"
+    t.string   "github_access_token"
   end
 
   add_index "users", ["admin"], name: "index_users_on_admin", using: :btree

lib/gitlab/github/client.rb

@@ -0,0 +1,29 @@
+module Gitlab
+  module Github
+    class Client
+      attr_reader :client
+
+      def initialize
+        @client = ::OAuth2::Client.new(
+          config.app_id,
+          config.app_secret,
+          github_options
+        )
+      end
+
+      private
+
+      def config
+        Gitlab.config.omniauth.providers.select{|provider| provider.name == "github"}.first
+      end
+
+      def github_options
+        {
+          :site => 'https://api.github.com',
+          :authorize_url => 'https://github.com/login/oauth/authorize',
+          :token_url => 'https://github.com/login/oauth/access_token'
+        }
+      end
+    end
+  end
+end

lib/gitlab/github/importer.rb

@@ -0,0 +1,48 @@
+module Gitlab
+  module Github
+    class Importer
+      attr_reader :project
+
+      def initialize(project)
+        @project = project
+      end
+
+      def execute
+        client = octo_client(project.creator.github_access_token)
+       
+        #Issues && Comments
+        client.list_issues(project.import_source, state: :all).each do |issue|
+          if issue.pull_request.nil?
+            body = "*Created by: #{issue.user.login}*\n\n#{issue.body}"
+            
+            if issue.comments > 0
+              body += "\n\n\n**Imported comments:**\n"
+              client.issue_comments(project.import_source, issue.number).each do |c|
+                body += "\n\n*By #{c.user.login} on #{c.created_at}*\n\n#{c.body}"
+              end
+            end
+
+            project.issues.create!(
+              description: body, 
+              title: issue.title,
+              state: issue.state == 'closed' ? 'closed' : 'opened',
+              author_id: gl_user_id(project, issue.user.id)
+            )
+          end
+        end
+      end
+
+      private
+
+      def octo_client(access_token)
+        ::Octokit.auto_paginate = true
+        ::Octokit::Client.new(:access_token => access_token)
+      end
+
+      def gl_user_id(project, github_id)
+        user = User.joins(:identities).find_by("identities.extern_uid = ?", github_id.to_s)
+        (user && user.id) || project.creator_id
+      end
+    end
+  end
+end

lib/gitlab/github/project_creator.rb

@@ -0,0 +1,37 @@
+module Gitlab
+  module Github
+    class ProjectCreator
+      attr_reader :repo, :namespace, :current_user
+
+      def initialize(repo, namespace, current_user)
+        @repo = repo
+        @namespace = namespace
+        @current_user = current_user
+      end
+
+      def execute
+        @project = Project.new(
+          name: repo.name,
+          path: repo.name,
+          description: repo.description,
+          namespace: namespace,
+          creator: current_user,
+          visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
+          import_type: "github",
+          import_source: repo.full_name,
+          import_url: repo.clone_url.sub("https://", "https://#{current_user.github_access_token}@")
+        )
+
+        if @project.save!
+          @project.reload
+
+          if @project.import_failed?
+            @project.import_retry
+          else
+            @project.import_start
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/regex.rb

@@ -11,7 +11,7 @@ module Gitlab
     end
 
     def project_name_regex
-      /\A[a-zA-Z0-9_][a-zA-Z0-9_\-\. ]*\z/
+      /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\. ]*\z/
     end
 
     def project_regex_message

spec/controllers/github_imports_controller_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+describe GithubImportsController do
+  let(:user) { create(:user, github_access_token: 'asd123') }
+
+  before do
+    sign_in(user)
+  end
+
+  describe "GET callback" do
+    it "updates access token" do
+      token = "asdasd12345"
+      Gitlab::Github::Client.any_instance.stub_chain(:client, :auth_code, :get_token, :token).and_return(token)
+
+      get :callback
+      
+      user.reload.github_access_token.should == token
+      controller.should redirect_to(status_github_import_url)
+    end
+  end
+
+  describe "GET status" do
+    before do
+      @repo = OpenStruct.new(login: 'vim', full_name: 'asd/vim')
+    end
+
+    it "assigns variables" do
+      @project = create(:project, import_type: 'github', creator_id: user.id)
+      controller.stub_chain(:octo_client, :repos).and_return([@repo])
+      controller.stub_chain(:octo_client, :orgs).and_return([])
+
+      get :status
+
+      expect(assigns(:already_added_projects)).to eq([@project])
+      expect(assigns(:repos)).to eq([@repo])
+    end
+
+    it "does not show already added project" do
+      @project = create(:project, import_type: 'github', creator_id: user.id, import_source: 'asd/vim')
+      controller.stub_chain(:octo_client, :repos).and_return([@repo])
+      controller.stub_chain(:octo_client, :orgs).and_return([])
+
+      get :status
+
+      expect(assigns(:already_added_projects)).to eq([@project])
+      expect(assigns(:repos)).to eq([])
+    end
+  end
+
+  describe "POST create" do
+    before do
+      @repo = OpenStruct.new(login: 'vim', full_name: 'asd/vim', owner: OpenStruct.new(login: "john"))
+    end
+
+    it "takes already existing namespace" do
+      namespace = create(:namespace, name: "john", owner: user)
+      Gitlab::Github::ProjectCreator.should_receive(:new).with(@repo, namespace, user).
+        and_return(double(execute: true))
+      controller.stub_chain(:octo_client, :repo).and_return(@repo)
+
+      post :create, format: :js
+    end
+  end
+end

spec/helpers/projects_helper_spec.rb

@@ -20,4 +20,13 @@ describe ProjectsHelper do
           "<option value=\"gitlab\">GitLab</option>"
     end
   end
+
+  describe "#project_status_css_class" do
+    it "returns appropriate class" do
+      project_status_css_class("started").should == "active"
+      project_status_css_class("failed").should == "danger"
+      project_status_css_class("finished").should == "success"
+    end
+
+  end
 end

spec/lib/gitlab/github/project_creator.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe Gitlab::Github::ProjectCreator do
+  let(:user) { create(:user, github_access_token: "asdffg") }
+  let(:repo) { OpenStruct.new(
+    login: 'vim',
+    name: 'vim',
+    private: true,
+    full_name: 'asd/vim',
+    clone_url: "https://gitlab.com/asd/vim.git",
+    owner: OpenStruct.new(login: "john"))
+  }
+  let(:namespace){ create(:namespace) }
+
+  it 'creates project' do
+    Project.any_instance.stub(:add_import_job)
+    
+    project_creator = Gitlab::Github::ProjectCreator.new(repo, namespace, user)
+    project_creator.execute
+    project = Project.last
+    
+    project.import_url.should ==  "https://asdffg@gitlab.com/asd/vim.git"
+    project.visibility_level.should == Gitlab::VisibilityLevel::PRIVATE
+  end
+end