root
/
gitlab-ce
mirror of https://gitlab.com/free-releases/gitlab-ce.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Add support for Jupyter in GitLab via Kubernetes 

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
This commit is contained in:
Dmitriy Zaporozhets 2018-05-16 12:01:13 +03:00
parent 8a1ac8f4ce
commit 4220e914db
18 changed files with 234 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/assets/javascripts/clusters/clusters_bundle.js
View File

@ -31,6 +31,7 @@ export default class Clusters {
installHelmPath,
installIngressPath,
installRunnerPath,
installJupyterPath,
installPrometheusPath,
managePrometheusPath,
clusterStatus,
@ -51,6 +52,7 @@ export default class Clusters {
installIngressEndpoint: installIngressPath,
installRunnerEndpoint: installRunnerPath,
installPrometheusEndpoint: installPrometheusPath,
installJupyterEndpoint: installJupyterPath,
});
this.installApplication = this.installApplication.bind(this);

99
app/assets/javascripts/clusters/components/applications.vue
View File

@ -37,6 +37,11 @@ export default {
default: '',
},
},
data() {
return {
jupyterSuggestHostnameValue: '',
};
},
computed: {
generalApplicationDescription() {
return sprintf(
@ -121,6 +126,20 @@ export default {
false,
);
},
jupyterInstalled() {
return this.applications.jupyter.status === APPLICATION_INSTALLED;
},
jupyterHostname() {
return this.applications.jupyter.hostname;
},
jupyterSuggestHostname() {
return `jupyter.${this.applications.ingress.externalIp}.xip.io`;
},
},
watch: {
jupyterSuggestHostname() {
this.jupyterSuggestHostnameValue = this.jupyterSuggestHostname;
},
},
};
</script>
@ -278,11 +297,89 @@ export default {
applications to production.`) }}
</div>
</application-row>
<application-row
id="jupyter"
:title="applications.jupyter.title"
title-link="https://jupyterhub.readthedocs.io/en/stable/"
:status="applications.jupyter.status"
:status-reason="applications.jupyter.statusReason"
:request-status="applications.jupyter.requestStatus"
:request-reason="applications.jupyter.requestReason"
>
<div slot="description">
<p>
{{ s__(`ClusterIntegration|JupyterHub, a multi-user Hub, spawns,
manages, and proxies multiple instances of the single-user
Jupyter notebook server. JupyterHub can be used to serve
notebooks to a class of students, a corporate data science group,
or a scientific research group.`) }}
</p>
<template v-if="jupyterInstalled">
<div class="form-group">
<label for="jupyter-hostname">
{{ s__('ClusterIntegration|Jupyter Hostname') }}
</label>
<div
v-if="jupyterHostname"
class="input-group"
>
<input
type="text"
id="jupyter-hostname"
class="form-control js-hostname"
:value="jupyterHostname"
readonly
/>
<span class="input-group-btn">
<clipboard-button
:text="jupyterHostname"
:title="s__('ClusterIntegration|Copy Jupyter Hostname to clipboard')"
class="js-clipboard-btn"
/>
</span>
</div>
</div>
</template>
<template v-else-if="ingressInstalled">
<div class="form-group">
<label for="jupyter-hostname">
{{ s__('ClusterIntegration|Jupyter Hostname') }}
</label>
<div class="input-group">
<input
type="text"
id="jupyter-hostname"
class="form-control js-hostname"
v-model="jupyterSuggestHostnameValue"
/>
<span class="input-group-btn">
<clipboard-button
:text="jupyterHostname"
:title="s__('ClusterIntegration|Copy Jupyter Hostname to clipboard')"
class="js-clipboard-btn"
/>
</span>
</div>
</div>
<p>
{{ s__(`ClusterIntegration|Replace this with your own hostname if you want.
If you do so, point hostname to Ingress IP Address from above.`) }}
<a
:href="ingressDnsHelpPath"
target="_blank"
rel="noopener noreferrer"
>
{{ __('More information') }}
</a>
</p>
</template>
</div>
</application-row>
<!--
NOTE: Don't forget to update `clusters.scss`
min-height for this block and uncomment `application_spec` tests
-->
<!-- Add GitLab Runner row, all other plumbing is complete -->
<!-- Add Jupyter row, all other plumbing is complete -->
</div>
</div>
</section>

1
app/assets/javascripts/clusters/constants.js
View File

@ -11,3 +11,4 @@ export const REQUEST_LOADING = 'request-loading';
export const REQUEST_SUCCESS = 'request-success';
export const REQUEST_FAILURE = 'request-failure';
export const INGRESS = 'ingress';
export const JUPYTER = 'jupyter';

10
app/assets/javascripts/clusters/services/clusters_service.js
View File

@ -1,4 +1,5 @@
import axios from '../../lib/utils/axios_utils';
import { JUPYTER } from '../constants';
export default class ClusterService {
constructor(options = {}) {
@ -8,6 +9,7 @@ export default class ClusterService {
ingress: this.options.installIngressEndpoint,
runner: this.options.installRunnerEndpoint,
prometheus: this.options.installPrometheusEndpoint,
jupyter: this.options.installJupyterEndpoint,
};
}
@ -16,7 +18,13 @@ export default class ClusterService {
}
installApplication(appId) {
return axios.post(this.appInstallEndpointMap[appId]);
const data = {};
if (appId === JUPYTER) {
data.hostname = document.getElementById('jupyter-hostname').value;
}
return axios.post(this.appInstallEndpointMap[appId], data);
}
static updateCluster(endpoint, data) {

12
app/assets/javascripts/clusters/stores/clusters_store.js
View File

@ -1,5 +1,5 @@
import { s__ } from '../../locale';
import { INGRESS } from '../constants';
import { INGRESS, JUPYTER } from '../constants';
export default class ClusterStore {
constructor() {
@ -38,6 +38,14 @@ export default class ClusterStore {
requestStatus: null,
requestReason: null,
},
jupyter: {
title: s__('ClusterIntegration|JupyterHub'),
status: null,
statusReason: null,
requestStatus: null,
requestReason: null,
hostname: null,
},
},
};
}
@ -83,6 +91,8 @@ export default class ClusterStore {
if (appId === INGRESS) {
this.state.applications.ingress.externalIp = serverAppEntry.external_ip;
} else if (appId === JUPYTER) {
this.state.applications.jupyter.hostname = serverAppEntry.hostname;
}
});
}

2
app/assets/stylesheets/pages/clusters.scss
View File

@ -6,7 +6,7 @@
.cluster-applications-table {
// Wait for the Vue to kick-in and render the applications block
min-height: 400px;
min-height: 500px;
}
.clusters-dropdown-menu {

1
app/controllers/projects/clusters/applications_controller.rb
View File

@ -6,6 +6,7 @@ class Projects::Clusters::ApplicationsController < Projects::ApplicationControll
def create
application = @application_class.find_or_create_by!(cluster: @cluster)
application.update(hostname: params[:hostname]) if application.respond_to?(:hostname)
Clusters::Applications::ScheduleInstallationService.new(project, current_user).execute(application)

28
app/models/clusters/applications/jupyter.rb
View File

@ -12,17 +12,39 @@ module Clusters
default_value_for :version, VERSION
def chart
# TODO: publish jupyterhub charts that we can use for our installation
# and provide path to it here.
"#{name}/jupyterhub"
end
def repository
'https://jupyterhub.github.io/helm-chart/'
end
def values
content_values.to_yaml
end
def install_command
Gitlab::Kubernetes::Helm::InstallCommand.new(
name,
chart: chart,
values: values
values: values,
repository: repository
)
end
private
def specification
{
"ingress" => { "hosts" => [hostname] },
"hub" => { "cookieSecret" => SecureRandom.hex(32) },
"proxy" => { "secretToken" => SecureRandom.hex(32) }
}
end
def content_values
YAML.load_file(chart_values_file).deep_merge!(specification)
end
end
end
end

4
app/models/clusters/cluster.rb
View File

@ -27,6 +27,7 @@ module Clusters
has_one :application_ingress, class_name: 'Clusters::Applications::Ingress'
has_one :application_prometheus, class_name: 'Clusters::Applications::Prometheus'
has_one :application_runner, class_name: 'Clusters::Applications::Runner'
has_one :application_jupyter, class_name: 'Clusters::Applications::Jupyter'
accepts_nested_attributes_for :provider_gcp, update_only: true
accepts_nested_attributes_for :platform_kubernetes, update_only: true
@ -75,7 +76,8 @@ module Clusters
application_helm || build_application_helm,
application_ingress || build_application_ingress,
application_prometheus || build_application_prometheus,
application_runner || build_application_runner
application_runner || build_application_runner,
application_jupyter || build_application_jupyter
]
end

1
app/serializers/cluster_application_entity.rb
View File

@ -3,4 +3,5 @@ class ClusterApplicationEntity < Grape::Entity
expose :status_name, as: :status
expose :status_reason
expose :external_ip, if: -> (e, _) { e.respond_to?(:external_ip) }
expose :hostname, if: -> (e, _) { e.respond_to?(:hostname) }
end

4
app/services/clusters/applications/install_service.rb
View File

@ -12,8 +12,8 @@ module Clusters
ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
rescue Kubeclient::HttpError => ke
app.make_errored!("Kubernetes error: #{ke.message}")
rescue StandardError
app.make_errored!("Can't start installation process")
rescue StandardError => e
app.make_errored!("Can't start installation process. #{e.message}")
end
end
end

1
app/views/projects/clusters/show.html.haml
View File

@ -11,6 +11,7 @@
install_ingress_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :ingress),
install_prometheus_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :prometheus),
install_runner_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :runner),
install_jupyter_path: install_applications_namespace_project_cluster_path(@cluster.project.namespace, @cluster.project, @cluster, :jupyter),
toggle_status: @cluster.enabled? ? 'true': 'false',
cluster_status: @cluster.status_name,
cluster_status_reason: @cluster.status_reason,

22
db/migrate/20180511131058_create_clusters_applications_jupyter.rb Normal file
View File

@ -0,0 +1,22 @@
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class CreateClustersApplicationsJupyter < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
create_table :clusters_applications_jupyters do |t|
t.references :cluster, null: false, unique: true, foreign_key: { on_delete: :cascade }
t.integer :status, null: false
t.string :version, null: false
t.string :hostname
t.text :status_reason
t.timestamps_with_timezone null: false
end
end
end

60
db/schema.rb
View File

@ -135,16 +135,13 @@ ActiveRecord::Schema.define(version: 20180521171529) do
t.boolean "clientside_sentry_enabled", default: false, null: false
t.string "clientside_sentry_dsn"
t.boolean "prometheus_metrics_enabled", default: true, null: false
t.boolean "authorized_keys_enabled", default: true, null: false
t.boolean "help_page_hide_commercial_content", default: false
t.string "help_page_support_url"
t.integer "performance_bar_allowed_group_id"
t.boolean "hashed_storage_enabled", default: false, null: false
t.boolean "project_export_enabled", default: true, null: false
t.boolean "auto_devops_enabled", default: false, null: false
t.integer "circuitbreaker_failure_count_threshold", default: 3
t.integer "circuitbreaker_failure_reset_time", default: 1800
t.integer "circuitbreaker_storage_timeout", default: 15
t.integer "circuitbreaker_access_retries", default: 3
t.boolean "throttle_unauthenticated_enabled", default: false, null: false
t.integer "throttle_unauthenticated_requests_per_period", default: 3600, null: false
t.integer "throttle_unauthenticated_period_in_seconds", default: 3600, null: false
@ -154,13 +151,16 @@ ActiveRecord::Schema.define(version: 20180521171529) do
t.boolean "throttle_authenticated_web_enabled", default: false, null: false
t.integer "throttle_authenticated_web_requests_per_period", default: 7200, null: false
t.integer "throttle_authenticated_web_period_in_seconds", default: 3600, null: false
t.integer "circuitbreaker_check_interval", default: 1, null: false
t.boolean "password_authentication_enabled_for_web"
t.boolean "password_authentication_enabled_for_git", default: true
t.integer "circuitbreaker_failure_count_threshold", default: 3
t.integer "circuitbreaker_failure_reset_time", default: 1800
t.integer "circuitbreaker_storage_timeout", default: 15
t.integer "circuitbreaker_access_retries", default: 3
t.integer "gitaly_timeout_default", default: 55, null: false
t.integer "gitaly_timeout_medium", default: 30, null: false
t.integer "gitaly_timeout_fast", default: 10, null: false
t.boolean "authorized_keys_enabled", default: true, null: false
t.boolean "password_authentication_enabled_for_web"
t.boolean "password_authentication_enabled_for_git", default: true, null: false
t.integer "circuitbreaker_check_interval", default: 1, null: false
t.string "auto_devops_domain"
t.boolean "pages_domain_verification_enabled", default: true, null: false
t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false
@ -375,12 +375,12 @@ ActiveRecord::Schema.define(version: 20180521171529) do
t.integer "project_id", null: false
t.integer "job_id", null: false
t.integer "file_type", null: false
t.integer "file_store"
t.integer "size", limit: 8
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.datetime_with_timezone "expire_at"
t.string "file"
t.integer "file_store"
t.binary "file_sha256"
end
@ -448,8 +448,8 @@ ActiveRecord::Schema.define(version: 20180521171529) do
t.integer "auto_canceled_by_id"
t.integer "pipeline_schedule_id"
t.integer "source"
t.integer "config_source"
t.boolean "protected"
t.integer "config_source"
t.integer "failure_reason"
end
@ -495,8 +495,8 @@ ActiveRecord::Schema.define(version: 20180521171529) do
t.boolean "run_untagged", default: true, null: false
t.boolean "locked", default: false, null: false
t.integer "access_level", default: 0, null: false
t.string "ip_address"
t.integer "maximum_timeout"
t.string "ip_address"
t.integer "runner_type", limit: 2, null: false
end
@ -635,6 +635,16 @@ ActiveRecord::Schema.define(version: 20180521171529) do
t.string "external_ip"
end
create_table "clusters_applications_jupyters", force: :cascade do |t|
t.integer "cluster_id", null: false
t.integer "status", null: false
t.string "version", null: false
t.string "hostname"
t.text "status_reason"
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
end
create_table "clusters_applications_prometheus", force: :cascade do |t|
t.integer "cluster_id", null: false
t.integer "status", null: false
@ -904,8 +914,8 @@ ActiveRecord::Schema.define(version: 20180521171529) do
add_index "gpg_signatures", ["project_id"], name: "index_gpg_signatures_on_project_id", using: :btree
create_table "group_custom_attributes", force: :cascade do |t|
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.integer "group_id", null: false
t.string "key", null: false
t.string "value", null: false
@ -987,6 +997,7 @@ ActiveRecord::Schema.define(version: 20180521171529) do
add_index "issues", ["moved_to_id"], name: "index_issues_on_moved_to_id", where: "(moved_to_id IS NOT NULL)", using: :btree
add_index "issues", ["project_id", "created_at", "id", "state"], name: "index_issues_on_project_id_and_created_at_and_id_and_state", using: :btree
add_index "issues", ["project_id", "due_date", "id", "state"], name: "idx_issues_on_project_id_and_due_date_and_id_and_state_partial", where: "(due_date IS NOT NULL)", using: :btree
add_index "issues", ["project_id", "due_date", "id", "state"], name: "index_issues_on_project_id_and_due_date_and_id_and_state", using: :btree
add_index "issues", ["project_id", "iid"], name: "index_issues_on_project_id_and_iid", unique: true, using: :btree
add_index "issues", ["project_id", "updated_at", "id", "state"], name: "index_issues_on_project_id_and_updated_at_and_id_and_state", using: :btree
add_index "issues", ["relative_position"], name: "index_issues_on_relative_position", using: :btree
@ -1203,6 +1214,7 @@ ActiveRecord::Schema.define(version: 20180521171529) do
t.boolean "merge_when_pipeline_succeeds", default: false, null: false
t.integer "merge_user_id"
t.string "merge_commit_sha"
t.string "rebase_commit_sha"
t.string "in_progress_merge_commit_sha"
t.integer "lock_version"
t.text "title_html"
@ -1215,7 +1227,6 @@ ActiveRecord::Schema.define(version: 20180521171529) do
t.string "merge_jid"
t.boolean "discussion_locked"
t.integer "latest_merge_request_diff_id"
t.string "rebase_commit_sha"
t.boolean "allow_maintainer_to_push"
end
@ -1475,8 +1486,8 @@ ActiveRecord::Schema.define(version: 20180521171529) do
add_index "project_ci_cd_settings", ["project_id"], name: "index_project_ci_cd_settings_on_project_id", unique: true, using: :btree
create_table "project_custom_attributes", force: :cascade do |t|
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.integer "project_id", null: false
t.string "key", null: false
t.string "value", null: false
@ -1568,8 +1579,10 @@ ActiveRecord::Schema.define(version: 20180521171529) do
t.string "avatar"
t.string "import_status"
t.integer "star_count", default: 0, null: false
t.boolean "merge_requests_rebase_enabled", default: false, null: false
t.string "import_type"
t.string "import_source"
t.boolean "merge_requests_ff_only_enabled", default: false, null: false
t.text "import_error"
t.integer "ci_id"
t.boolean "shared_runners_enabled", default: true, null: false
@ -1585,6 +1598,7 @@ ActiveRecord::Schema.define(version: 20180521171529) do
t.boolean "only_allow_merge_if_pipeline_succeeds", default: false, null: false
t.boolean "has_external_issue_tracker"
t.string "repository_storage", default: "default", null: false
t.boolean "repository_read_only"
t.boolean "request_access_enabled", default: false, null: false
t.boolean "has_external_wiki"
t.string "ci_config_path"
@ -1599,9 +1613,6 @@ ActiveRecord::Schema.define(version: 20180521171529) do
t.datetime "last_repository_updated_at"
t.integer "storage_version", limit: 2
t.boolean "resolve_outdated_diff_discussions"
t.boolean "repository_read_only"
t.boolean "merge_requests_ff_only_enabled", default: false
t.boolean "merge_requests_rebase_enabled", default: false, null: false
t.integer "jobs_cache_index"
t.boolean "pages_https_only", default: true
t.boolean "remote_mirror_available_overridden"
@ -1945,9 +1956,9 @@ ActiveRecord::Schema.define(version: 20180521171529) do
t.string "model_type"
t.string "uploader", null: false
t.datetime "created_at", null: false
t.integer "store"
t.string "mount_point"
t.string "secret"
t.integer "store"
end
add_index "uploads", ["checksum"], name: "index_uploads_on_checksum", using: :btree
@ -2179,8 +2190,9 @@ ActiveRecord::Schema.define(version: 20180521171529) do
add_foreign_key "cluster_providers_gcp", "clusters", on_delete: :cascade
add_foreign_key "clusters", "users", on_delete: :nullify
add_foreign_key "clusters_applications_helm", "clusters", on_delete: :cascade
add_foreign_key "clusters_applications_ingress", "clusters", name: "fk_753a7b41c1", on_delete: :cascade
add_foreign_key "clusters_applications_prometheus", "clusters", name: "fk_557e773639", on_delete: :cascade
add_foreign_key "clusters_applications_ingress", "clusters", on_delete: :cascade
add_foreign_key "clusters_applications_jupyters", "clusters", on_delete: :cascade
add_foreign_key "clusters_applications_prometheus", "clusters", on_delete: :cascade
add_foreign_key "clusters_applications_runners", "ci_runners", column: "runner_id", name: "fk_02de2ded36", on_delete: :nullify
add_foreign_key "clusters_applications_runners", "clusters", on_delete: :cascade
add_foreign_key "container_repositories", "projects"
@ -2285,8 +2297,8 @@ ActiveRecord::Schema.define(version: 20180521171529) do
add_foreign_key "u2f_registrations", "users"
add_foreign_key "user_callouts", "users", on_delete: :cascade
add_foreign_key "user_custom_attributes", "users", on_delete: :cascade
add_foreign_key "user_interacted_projects", "projects", name: "fk_722ceba4f7", on_delete: :cascade
add_foreign_key "user_interacted_projects", "users", name: "fk_0894651f08", on_delete: :cascade
add_foreign_key "user_interacted_projects", "projects", on_delete: :cascade
add_foreign_key "user_interacted_projects", "users", on_delete: :cascade
add_foreign_key "user_synced_attributes_metadata", "users", on_delete: :cascade
add_foreign_key "users", "application_setting_terms", column: "accepted_term_id", name: "fk_789cd90b35", on_delete: :cascade
add_foreign_key "users_star_projects", "projects", name: "fk_22cd27ddfc", on_delete: :cascade

1
spec/factories/clusters/applications/helm.rb
View File

@ -35,5 +35,6 @@ FactoryBot.define do
factory :clusters_applications_ingress, class: Clusters::Applications::Ingress
factory :clusters_applications_prometheus, class: Clusters::Applications::Prometheus
factory :clusters_applications_runner, class: Clusters::Applications::Runner
factory :clusters_applications_jupyter, class: Clusters::Applications::Jupyter
end
end

3
spec/fixtures/api/schemas/cluster_status.json vendored
View File

@ -31,7 +31,8 @@
}
},
"status_reason": { "type": ["string", "null"] },
"external_ip": { "type": ["string", "null"] }
"external_ip": { "type": ["string", "null"] },
"hostname": { "type": ["string", "null"] }
},
"required" : [ "name", "status" ]
}

3
spec/models/clusters/cluster_spec.rb
View File

@ -234,9 +234,10 @@ describe Clusters::Cluster do
let!(:ingress) { create(:clusters_applications_ingress, cluster: cluster) }
let!(:prometheus) { create(:clusters_applications_prometheus, cluster: cluster) }
let!(:runner) { create(:clusters_applications_runner, cluster: cluster) }
let!(:jupyter) { create(:clusters_applications_jupyter, cluster: cluster) }
it 'returns a list of created applications' do
is_expected.to contain_exactly(helm, ingress, prometheus, runner)
is_expected.to contain_exactly(helm, ingress, prometheus, runner, jupyter)
end
end
end

16
vendor/jupyter/values.yaml vendored Normal file
View File

@ -0,0 +1,16 @@
rbac:
enabled: false
hub:
extraEnv:
JUPYTER_ENABLE_LAB: 1
extraConfig: |
c.KubeSpawner.cmd = ['jupyter-labhub']
singleuser:
defaultUrl: "/lab"
ingress:
enabled: true
annotations:
kubernetes.io/ingress.class: "nginx"