Merge branch 'feature/custom-attributes-on-projects-and-groups' into 'master'

Support custom attributes on groups and projects

See merge request gitlab-org/gitlab-ce!14593

app/finders/groups_finder.rb

@@ -15,6 +15,8 @@
 # Anonymous users will never return any `owned` groups. They will return all
 # public groups instead, even if `all_available` is set to false.
 class GroupsFinder < UnionFinder
+  include CustomAttributesFilter
+
   def initialize(current_user = nil, params = {})
     @current_user = current_user
     @params = params
@@ -22,8 +24,12 @@ class GroupsFinder < UnionFinder
 
   def execute
     items = all_groups.map do |item|
-      by_parent(item)
+      item = by_parent(item)
+      item = by_custom_attributes(item)
+
+      item
     end
+
     find_union(items, Group).with_route.order_id_desc
   end
 

app/finders/projects_finder.rb

@@ -18,6 +18,8 @@
 #     non_archived: boolean
 #
 class ProjectsFinder < UnionFinder
+  include CustomAttributesFilter
+
   attr_accessor :params
   attr_reader :current_user, :project_ids_relation
 
@@ -44,6 +46,7 @@ class ProjectsFinder < UnionFinder
     collection = by_tags(collection)
     collection = by_search(collection)
     collection = by_archived(collection)
+    collection = by_custom_attributes(collection)
 
     sort(collection)
   end

app/models/group.rb

@@ -26,6 +26,7 @@ class Group < Namespace
   has_many :notification_settings, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent
   has_many :labels, class_name: 'GroupLabel'
   has_many :variables, class_name: 'Ci::GroupVariable'
+  has_many :custom_attributes, class_name: 'GroupCustomAttribute'
 
   validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? }
   validate :visibility_level_allowed_by_projects

app/models/group_custom_attribute.rb

@@ -0,0 +1,6 @@
+class GroupCustomAttribute < ActiveRecord::Base
+  belongs_to :group
+
+  validates :group, :key, :value, presence: true
+  validates :key, uniqueness: { scope: [:group_id] }
+end

app/models/project.rb

@@ -215,6 +215,7 @@ class Project < ActiveRecord::Base
   has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
 
   has_one :auto_devops, class_name: 'ProjectAutoDevops'
+  has_many :custom_attributes, class_name: 'ProjectCustomAttribute'
 
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature, update_only: true

app/models/project_custom_attribute.rb

@@ -0,0 +1,6 @@
+class ProjectCustomAttribute < ActiveRecord::Base
+  belongs_to :project
+
+  validates :project, :key, :value, presence: true
+  validates :key, uniqueness: { scope: [:project_id] }
+end

changelogs/unreleased/feature-custom-attributes-on-projects-and-groups.yml

@@ -0,0 +1,5 @@
+---
+title: Support custom attributes on groups and projects
+merge_request: 14593
+author: Markus Koller
+type: changed

db/migrate/20170918111708_create_project_custom_attributes.rb

@@ -0,0 +1,15 @@
+class CreateProjectCustomAttributes < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :project_custom_attributes do |t|
+      t.timestamps_with_timezone null: false
+      t.references :project, null: false, foreign_key: { on_delete: :cascade }
+      t.string :key, null: false
+      t.string :value, null: false
+
+      t.index [:project_id, :key], unique: true
+      t.index [:key, :value]
+    end
+  end
+end

db/migrate/20170918140927_create_group_custom_attributes.rb

@@ -0,0 +1,19 @@
+class CreateGroupCustomAttributes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :group_custom_attributes do |t|
+      t.timestamps_with_timezone null: false
+      t.references :group, null: false
+      t.string :key, null: false
+      t.string :value, null: false
+
+      t.index [:group_id, :key], unique: true
+      t.index [:key, :value]
+    end
+
+    add_foreign_key :group_custom_attributes, :namespaces, column: :group_id, on_delete: :cascade # rubocop: disable Migration/AddConcurrentForeignKey
+  end
+end

db/schema.rb

@@ -750,6 +750,17 @@ ActiveRecord::Schema.define(version: 20171101134435) do
   add_index "gpg_signatures", ["gpg_key_subkey_id"], name: "index_gpg_signatures_on_gpg_key_subkey_id", using: :btree
   add_index "gpg_signatures", ["project_id"], name: "index_gpg_signatures_on_project_id", using: :btree
 
+  create_table "group_custom_attributes", force: :cascade do |t|
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.integer "group_id", null: false
+    t.string "key", null: false
+    t.string "value", null: false
+  end
+
+  add_index "group_custom_attributes", ["group_id", "key"], name: "index_group_custom_attributes_on_group_id_and_key", unique: true, using: :btree
+  add_index "group_custom_attributes", ["key", "value"], name: "index_group_custom_attributes_on_key_and_value", using: :btree
+
   create_table "identities", force: :cascade do |t|
     t.string "extern_uid"
     t.string "provider"
@@ -1270,6 +1281,17 @@ ActiveRecord::Schema.define(version: 20171101134435) do
 
   add_index "project_auto_devops", ["project_id"], name: "index_project_auto_devops_on_project_id", unique: true, using: :btree
 
+  create_table "project_custom_attributes", force: :cascade do |t|
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.integer "project_id", null: false
+    t.string "key", null: false
+    t.string "value", null: false
+  end
+
+  add_index "project_custom_attributes", ["key", "value"], name: "index_project_custom_attributes_on_key_and_value", using: :btree
+  add_index "project_custom_attributes", ["project_id", "key"], name: "index_project_custom_attributes_on_project_id_and_key", unique: true, using: :btree
+
   create_table "project_features", force: :cascade do |t|
     t.integer "project_id"
     t.integer "merge_requests_access_level"
@@ -1890,6 +1912,7 @@ ActiveRecord::Schema.define(version: 20171101134435) do
   add_foreign_key "gpg_signatures", "gpg_key_subkeys", on_delete: :nullify
   add_foreign_key "gpg_signatures", "gpg_keys", on_delete: :nullify
   add_foreign_key "gpg_signatures", "projects", on_delete: :cascade
+  add_foreign_key "group_custom_attributes", "namespaces", column: "group_id", on_delete: :cascade
   add_foreign_key "issue_assignees", "issues", name: "fk_b7d881734a", on_delete: :cascade
   add_foreign_key "issue_assignees", "users", name: "fk_5e0c8d9154", on_delete: :cascade
   add_foreign_key "issue_metrics", "issues", on_delete: :cascade
@@ -1920,6 +1943,7 @@ ActiveRecord::Schema.define(version: 20171101134435) do
   add_foreign_key "project_authorizations", "projects", on_delete: :cascade
   add_foreign_key "project_authorizations", "users", on_delete: :cascade
   add_foreign_key "project_auto_devops", "projects", on_delete: :cascade
+  add_foreign_key "project_custom_attributes", "projects", on_delete: :cascade
   add_foreign_key "project_features", "projects", name: "fk_18513d9b92", on_delete: :cascade
   add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade
   add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade

doc/api/custom_attributes.md

@@ -2,17 +2,22 @@
 
 Every API call to custom attributes must be authenticated as administrator.
 
+Custom attributes are currently available on users, groups, and projects,
+which will be referred to as "resource" in this documentation.
+
 ## List custom attributes
 
-Get all custom attributes on a user.
+Get all custom attributes on a resource.
 
 ```
 GET /users/:id/custom_attributes
+GET /groups/:id/custom_attributes
+GET /projects/:id/custom_attributes
 ```
 
 | Attribute | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
-| `id` | integer | yes | The ID of a user |
+| `id` | integer | yes | The ID of a resource |
 
 ```bash
 curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/users/42/custom_attributes
@@ -35,15 +40,17 @@ Example response:
 
 ## Single custom attribute
 
-Get a single custom attribute on a user.
+Get a single custom attribute on a resource.
 
 ```
 GET /users/:id/custom_attributes/:key
+GET /groups/:id/custom_attributes/:key
+GET /projects/:id/custom_attributes/:key
 ```
 
 | Attribute | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
-| `id` | integer | yes | The ID of a user |
+| `id` | integer | yes | The ID of a resource |
 | `key` | string | yes | The key of the custom attribute |
 
 ```bash
@@ -61,16 +68,18 @@ Example response:
 
 ## Set custom attribute
 
-Set a custom attribute on a user. The attribute will be updated if it already exists,
+Set a custom attribute on a resource. The attribute will be updated if it already exists,
 or newly created otherwise.
 
 ```
 PUT /users/:id/custom_attributes/:key
+PUT /groups/:id/custom_attributes/:key
+PUT /projects/:id/custom_attributes/:key
 ```
 
 | Attribute | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
-| `id` | integer | yes | The ID of a user |
+| `id` | integer | yes | The ID of a resource |
 | `key` | string | yes | The key of the custom attribute |
 | `value` | string | yes | The value of the custom attribute |
 
@@ -89,15 +98,17 @@ Example response:
 
 ## Delete custom attribute
 
-Delete a custom attribute on a user.
+Delete a custom attribute on a resource.
 
 ```
 DELETE /users/:id/custom_attributes/:key
+DELETE /groups/:id/custom_attributes/:key
+DELETE /projects/:id/custom_attributes/:key
 ```
 
 | Attribute | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
-| `id` | integer | yes | The ID of a user |
+| `id` | integer | yes | The ID of a resource |
 | `key` | string | yes | The key of the custom attribute |
 
 ```bash

doc/api/groups.md

@@ -74,6 +74,12 @@ GET /groups?statistics=true
 
 You can search for groups by name or path, see below.
 
+You can filter by [custom attributes](custom_attributes.md) with:
+
+```
+GET /groups?custom_attributes[key]=value&custom_attributes[other_key]=other_value
+```
+
 ## List a group's projects
 
 Get a list of projects in this group. When accessed without authentication, only

doc/api/projects.md

@@ -192,6 +192,12 @@ GET /projects
 ]
 ```
 
+You can filter by [custom attributes](custom_attributes.md) with:
+
+```
+GET /projects?custom_attributes[key]=value&custom_attributes[other_key]=other_value
+```
+
 ## List user projects
 
 Get a list of visible projects for the given user. When accessed without

lib/api/groups.rb

@@ -37,6 +37,8 @@ module API
     end
 
     resource :groups do
+      include CustomAttributesEndpoints
+
       desc 'Get a groups list' do
         success Entities::Group
       end
@@ -51,7 +53,12 @@ module API
         use :pagination
       end
       get do
-        find_params = { all_available: params[:all_available], owned: params[:owned] }
+        find_params = {
+          all_available: params[:all_available],
+          owned: params[:owned],
+          custom_attributes: params[:custom_attributes]
+        }
+
         groups = GroupsFinder.new(current_user, find_params).execute
         groups = groups.search(params[:search]) if params[:search].present?
         groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?

lib/api/helpers.rb

@@ -328,6 +328,7 @@ module API
       finder_params[:archived] = params[:archived]
       finder_params[:search] = params[:search] if params[:search]
       finder_params[:user] = params.delete(:user) if params[:user]
+      finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes]
       finder_params
     end
 

lib/api/projects.rb

@@ -119,6 +119,8 @@ module API
     end
 
     resource :projects do
+      include CustomAttributesEndpoints
+
       desc 'Get a list of visible projects for authenticated user' do
         success Entities::BasicProjectDetails
       end

lib/gitlab/import_export/import_export.yml

@@ -63,6 +63,7 @@ project_tree:
   - protected_tags:
     - :create_access_levels
   - :project_feature
+  - :custom_attributes
 
 # Only include the following attributes for the models specified.
 included_attributes:

lib/gitlab/import_export/relation_factory.rb

@@ -17,7 +17,8 @@ module Gitlab
                     labels: :project_labels,
                     priorities: :label_priorities,
                     auto_devops: :project_auto_devops,
-                    label: :project_label }.freeze
+                    label: :project_label,
+                    custom_attributes: 'ProjectCustomAttribute' }.freeze
 
       USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze
 

spec/factories/group_custom_attributes.rb

@@ -0,0 +1,7 @@
+FactoryGirl.define do
+  factory :group_custom_attribute do
+    group
+    sequence(:key) { |n| "key#{n}" }
+    sequence(:value) { |n| "value#{n}" }
+  end
+end

spec/factories/project_custom_attributes.rb

@@ -0,0 +1,7 @@
+FactoryGirl.define do
+  factory :project_custom_attribute do
+    project
+    sequence(:key) { |n| "key#{n}" }
+    sequence(:value) { |n| "value#{n}" }
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -286,6 +286,7 @@ project:
 - root_of_fork_network
 - fork_network_member
 - fork_network
+- custom_attributes
 award_emoji:
 - awardable
 - user

spec/lib/gitlab/import_export/project.json

@@ -7408,5 +7408,23 @@
     "snippets_access_level": 20,
     "updated_at": "2016-09-23T11:58:28.000Z",
     "wiki_access_level": 20
-  }
+  },
+  "custom_attributes": [
+    {
+      "id": 1,
+      "created_at": "2017-10-19T15:36:23.466Z",
+      "updated_at": "2017-10-19T15:36:23.466Z",
+      "project_id": 5,
+      "key": "foo",
+      "value": "foo"
+    },
+    {
+      "id": 2,
+      "created_at": "2017-10-19T15:37:21.904Z",
+      "updated_at": "2017-10-19T15:37:21.904Z",
+      "project_id": 5,
+      "key": "bar",
+      "value": "bar"
+    }
+  ]
 }

spec/lib/gitlab/import_export/project_tree_restorer_spec.rb

@@ -133,6 +133,10 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
         expect(@project.project_feature).not_to be_nil
       end
 
+      it 'has custom attributes' do
+        expect(@project.custom_attributes.count).to eq(2)
+      end
+
       it 'restores the correct service' do
         expect(CustomIssueTrackerService.first).not_to be_nil
       end

spec/lib/gitlab/import_export/project_tree_saver_spec.rb

@@ -168,6 +168,10 @@ describe Gitlab::ImportExport::ProjectTreeSaver do
         expect(project_feature["builds_access_level"]).to eq(ProjectFeature::PRIVATE)
       end
 
+      it 'has custom attributes' do
+        expect(saved_project_json['custom_attributes'].count).to eq(2)
+      end
+
       it 'does not complain about non UTF-8 characters in MR diffs' do
         ActiveRecord::Base.connection.execute("UPDATE merge_request_diffs SET st_diffs = '---\n- :diff: !binary |-\n    LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n    KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n    YXR'")
 
@@ -279,6 +283,9 @@ describe Gitlab::ImportExport::ProjectTreeSaver do
     create(:event, :created, target: milestone, project: project, author: user)
     create(:service, project: project, type: 'CustomIssueTrackerService', category: 'issue_tracker')
 
+    create(:project_custom_attribute, project: project)
+    create(:project_custom_attribute, project: project)
+
     project
   end
 

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -525,4 +525,11 @@ ProjectAutoDevops:
 - updated_at
 IssueAssignee:
 - user_id
-- issue_id
+- issue_id
+ProjectCustomAttribute:
+- id
+- created_at
+- updated_at
+- project_id
+- key
+- value

spec/models/group_custom_attribute_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe GroupCustomAttribute do
+  describe 'assocations' do
+    it { is_expected.to belong_to(:group) }
+  end
+
+  describe 'validations' do
+    subject { build :group_custom_attribute }
+
+    it { is_expected.to validate_presence_of(:group) }
+    it { is_expected.to validate_presence_of(:key) }
+    it { is_expected.to validate_presence_of(:value) }
+    it { is_expected.to validate_uniqueness_of(:key).scoped_to(:group_id) }
+  end
+end

spec/models/group_spec.rb

@@ -17,6 +17,7 @@ describe Group do
     it { is_expected.to have_many(:variables).class_name('Ci::GroupVariable') }
     it { is_expected.to have_many(:uploads).dependent(:destroy) }
     it { is_expected.to have_one(:chat_team) }
+    it { is_expected.to have_many(:custom_attributes).class_name('GroupCustomAttribute') }
 
     describe '#members & #requesters' do
       let(:requester) { create(:user) }

spec/models/project_custom_attribute_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe ProjectCustomAttribute do
+  describe 'assocations' do
+    it { is_expected.to belong_to(:project) }
+  end
+
+  describe 'validations' do
+    subject { build :project_custom_attribute }
+
+    it { is_expected.to validate_presence_of(:project) }
+    it { is_expected.to validate_presence_of(:key) }
+    it { is_expected.to validate_presence_of(:value) }
+    it { is_expected.to validate_uniqueness_of(:key).scoped_to(:project_id) }
+  end
+end

spec/models/project_spec.rb

@@ -79,6 +79,7 @@ describe Project do
     it { is_expected.to have_many(:pipeline_schedules) }
     it { is_expected.to have_many(:members_and_requesters) }
     it { is_expected.to have_one(:cluster) }
+    it { is_expected.to have_many(:custom_attributes).class_name('ProjectCustomAttribute') }
 
     context 'after initialized' do
       it "has a project_feature" do

spec/requests/api/groups_spec.rb

@@ -618,4 +618,14 @@ describe API::Groups do
       end
     end
   end
+
+  it_behaves_like 'custom attributes endpoints', 'groups' do
+    let(:attributable) { group1 }
+    let(:other_attributable) { group2 }
+    let(:user) { user1 }
+
+    before do
+      group2.add_owner(user1)
+    end
+  end
 end

spec/requests/api/projects_spec.rb

@@ -1856,4 +1856,9 @@ describe API::Projects do
       end
     end
   end
+
+  it_behaves_like 'custom attributes endpoints', 'projects' do
+    let(:attributable) { project }
+    let(:other_attributable) { project2 }
+  end
 end

spec/requests/api/users_spec.rb

@@ -1880,7 +1880,8 @@ describe API::Users do
     end
   end
 
-  include_examples 'custom attributes endpoints', 'users' do
+  it_behaves_like 'custom attributes endpoints', 'users' do
     let(:attributable) { user }
+    let(:other_attributable) { admin }
   end
 end

spec/support/shared_examples/requests/api/custom_attributes_shared_examples.rb

@@ -3,7 +3,9 @@ shared_examples 'custom attributes endpoints' do |attributable_name|
   let!(:custom_attribute2) { attributable.custom_attributes.create key: 'bar', value: 'bar' }
 
   describe "GET /#{attributable_name} with custom attributes filter" do
-    let!(:other_attributable) { create attributable.class.name.underscore }
+    before do
+      other_attributable
+    end
 
     context 'with an unauthorized user' do
       it 'does not filter by custom attributes' do
@@ -11,6 +13,7 @@ shared_examples 'custom attributes endpoints' do |attributable_name|
 
         expect(response).to have_gitlab_http_status(200)
         expect(json_response.size).to be 2
+        expect(json_response.map { |r| r['id'] }).to contain_exactly attributable.id, other_attributable.id
       end
     end