diff --git a/.drone.star b/.drone.star index b9d6ab67c91..10719f988f5 100644 --- a/.drone.star +++ b/.drone.star @@ -4,12 +4,12 @@ # 3. Run `make drone` # More information about this process here: https://github.com/grafana/deployment_tools/blob/master/docs/infrastructure/drone/signing.md -load('scripts/drone/pipelines/pr.star', 'pr_pipelines') -load('scripts/drone/pipelines/main.star', 'main_pipelines') +load('scripts/drone/events/pr.star', 'pr_pipelines') +load('scripts/drone/events/main.star', 'main_pipelines') load('scripts/drone/pipelines/docs.star', 'docs_pipelines') -load('scripts/drone/pipelines/release.star', 'release_pipelines', 'publish_image_pipelines', 'publish_artifacts_pipelines', 'publish_npm_pipelines', 'publish_packages_pipeline', 'artifacts_page_pipeline') +load('scripts/drone/events/release.star', 'release_pipelines', 'publish_image_pipelines', 'publish_artifacts_pipelines', 'publish_npm_pipelines', 'publish_packages_pipeline', 'artifacts_page_pipeline') load('scripts/drone/version.star', 'version_branch_pipelines') -load('scripts/drone/pipelines/cron.star', 'cronjobs') +load('scripts/drone/events/cron.star', 'cronjobs') load('scripts/drone/vault.star', 'secrets') def main(ctx): diff --git a/.drone.yml b/.drone.yml index 12bce0afd0b..cb881e4910d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -413,6 +413,13 @@ steps: NODE_OPTIONS: --max_old_space_size=4096 image: grafana/build-container:1.5.8 name: build-storybook +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package + image: grafana/build-container:1.5.8 + name: copy-packages-for-docker - commands: - yarn wait-on http://$HOST:$PORT - pa11y-ci --config .pa11yci-pr.conf.js @@ -426,13 +433,6 @@ steps: failure: always image: grafana/docker-puppeteer:1.0.0 name: test-a11y-frontend -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.5.8 - name: copy-packages-for-docker - commands: - ./bin/grabpl build-docker --edition oss -archs amd64 depends_on: @@ -850,12 +850,6 @@ steps: - verify-gen-cue image: grafana/build-container:1.5.8 name: wire-install -- commands: - - ./bin/grabpl verify-drone - depends_on: - - grabpl - image: byrnedo/alpine-curl:0.1.8 - name: lint-drone - commands: - |- echo -e "unknwon @@ -894,6 +888,12 @@ steps: - wire-install image: grafana/build-container:1.5.8 name: test-backend-integration +- commands: + - ./bin/grabpl verify-drone + depends_on: + - grabpl + image: byrnedo/alpine-curl:0.1.8 + name: lint-drone trigger: branch: main event: @@ -1133,6 +1133,26 @@ steps: NODE_OPTIONS: --max_old_space_size=4096 image: grafana/build-container:1.5.8 name: build-storybook +- commands: + - ls dist/*.tar.gz* + - cp dist/*.tar.gz* packaging/docker/ + depends_on: + - package + image: grafana/build-container:1.5.8 + name: copy-packages-for-docker +- commands: + - yarn wait-on http://$HOST:$PORT + - pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json + depends_on: + - grafana-server + environment: + GRAFANA_MISC_STATS_API_KEY: + from_secret: grafana_misc_stats_api_key + HOST: grafana-server + PORT: 3001 + failure: ignore + image: grafana/docker-puppeteer:1.0.0 + name: test-a11y-frontend - commands: - ./bin/grabpl store-storybook --deployment canary --src-bucket grafana-storybook depends_on: @@ -1151,19 +1171,6 @@ steps: when: repo: - grafana/grafana -- commands: - - yarn wait-on http://$HOST:$PORT - - pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json - depends_on: - - grafana-server - environment: - GRAFANA_MISC_STATS_API_KEY: - from_secret: grafana_misc_stats_api_key - HOST: grafana-server - PORT: 3001 - failure: ignore - image: grafana/docker-puppeteer:1.0.0 - name: test-a11y-frontend - commands: - ./scripts/ci-frontend-metrics.sh | ./bin/grabpl publish-metrics $${GRAFANA_MISC_STATS_API_KEY} depends_on: @@ -1177,13 +1184,6 @@ steps: when: repo: - grafana/grafana -- commands: - - ls dist/*.tar.gz* - - cp dist/*.tar.gz* packaging/docker/ - depends_on: - - package - image: grafana/build-container:1.5.8 - name: copy-packages-for-docker - commands: - ./bin/grabpl build-docker --edition oss depends_on: @@ -4874,6 +4874,6 @@ kind: secret name: gcp_upload_artifacts_key --- kind: signature -hmac: d7d85528bb22e0a126fb5fb04ec3bdde3001ab0d5a9a88a34db3cd9985eeb9fa +hmac: fdbc2724cf738ea9a2cd2be46f70d13928c96cc29179a79b7108650af40740ff ... diff --git a/scripts/drone/pipelines/cron.star b/scripts/drone/events/cron.star similarity index 100% rename from scripts/drone/pipelines/cron.star rename to scripts/drone/events/cron.star diff --git a/scripts/drone/events/main.star b/scripts/drone/events/main.star new file mode 100644 index 00000000000..7209dda83aa --- /dev/null +++ b/scripts/drone/events/main.star @@ -0,0 +1,88 @@ +load( + 'scripts/drone/utils/utils.star', + 'pipeline', + 'notify_pipeline', + 'failure_template', + 'drone_change_template', +) + +load( + 'scripts/drone/pipelines/docs.star', + 'docs_pipelines', +) + +load( + 'scripts/drone/pipelines/test_frontend.star', + 'test_frontend', +) + +load( + 'scripts/drone/pipelines/test_backend.star', + 'test_backend', +) + +load( + 'scripts/drone/pipelines/integration_tests.star', + 'integration_tests', +) + +load( + 'scripts/drone/pipelines/build.star', + 'build_e2e', +) + +load( + 'scripts/drone/pipelines/windows.star', + 'windows', +) + +load( + 'scripts/drone/pipelines/publish.star', + 'publish', +) + +load('scripts/drone/vault.star', 'from_secret') + + +ver_mode = 'main' +trigger = { + 'event': ['push',], + 'branch': 'main', +} + +def main_pipelines(edition): + drone_change_trigger = { + 'event': ['push',], + 'branch': 'main', + 'repo': [ + 'grafana/grafana', + ], + 'paths': { + 'include': [ + '.drone.yml', + ], + 'exclude': [ + 'exclude', + ], + }, + } + + pipelines = [ + docs_pipelines(edition, ver_mode, trigger), + test_frontend(trigger, ver_mode), + test_backend(trigger, ver_mode), + build_e2e(trigger, ver_mode, edition), + integration_tests(trigger, ver_mode, edition), + windows(trigger, edition, ver_mode), + notify_pipeline( + name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger, + template=drone_change_template, secret='drone-changes-webhook', + ), + publish(trigger, ver_mode, edition), + notify_pipeline( + name='main-notify', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']), + depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', 'main-publish'], + template=failure_template, secret='slack_webhook' + )] + + return pipelines diff --git a/scripts/drone/events/pr.star b/scripts/drone/events/pr.star new file mode 100644 index 00000000000..9c3acbf60bd --- /dev/null +++ b/scripts/drone/events/pr.star @@ -0,0 +1,81 @@ +load( + 'scripts/drone/utils/utils.star', + 'pipeline', +) + +load( + 'scripts/drone/pipelines/test_frontend.star', + 'test_frontend', +) + +load( + 'scripts/drone/pipelines/test_backend.star', + 'test_backend', +) + +load( + 'scripts/drone/pipelines/integration_tests.star', + 'integration_tests', +) + +load( + 'scripts/drone/pipelines/build.star', + 'build_e2e', +) + +load( + 'scripts/drone/pipelines/verify_drone.star', + 'verify_drone', +) + +load( + 'scripts/drone/pipelines/docs.star', + 'docs_pipelines', + 'trigger_docs', +) + +ver_mode = 'pr' +trigger = { + 'event': [ + 'pull_request', + ], + 'paths': { + 'exclude': [ + '*.md', + 'docs/**', + 'latest.json', + ], + }, +} + + +def pr_pipelines(edition): + return [ + verify_drone(get_pr_trigger(include_paths=['scripts/drone/**', '.drone.yml', '.drone.star']), ver_mode), + test_frontend(get_pr_trigger(exclude_paths=['pkg/**', 'packaging/**', 'go.sum', 'go.mod']), ver_mode), + test_backend(get_pr_trigger(include_paths=['pkg/**', 'packaging/**', '.drone.yml', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json']), ver_mode), + build_e2e(trigger, ver_mode, edition), + integration_tests(trigger, ver_mode, edition), + docs_pipelines(edition, ver_mode, trigger_docs()) + ] + + +def get_pr_trigger(include_paths=None, exclude_paths=None): + paths_ex = ['docs/**', '*.md'] + paths_in = [] + if include_paths: + for path in include_paths: + paths_in.extend([path]) + if exclude_paths: + for path in exclude_paths: + paths_ex.extend([path]) + return { + 'event': [ + 'pull_request', + ], + 'paths': { + 'exclude': paths_ex, + 'include': paths_in, + }, + } + diff --git a/scripts/drone/pipelines/release.star b/scripts/drone/events/release.star similarity index 100% rename from scripts/drone/pipelines/release.star rename to scripts/drone/events/release.star diff --git a/scripts/drone/pipelines/build.star b/scripts/drone/pipelines/build.star new file mode 100644 index 00000000000..6e5b246e822 --- /dev/null +++ b/scripts/drone/pipelines/build.star @@ -0,0 +1,107 @@ +load( + 'scripts/drone/steps/lib.star', + 'download_grabpl_step', + 'build_image', + 'identify_runner_step', + 'gen_version_step', + 'wire_install_step', + 'yarn_install_step', + 'build_backend_step', + 'build_frontend_step', + 'build_frontend_package_step', + 'build_plugins_step', + 'package_step', + 'grafana_server_step', + 'e2e_tests_step', + 'e2e_tests_artifacts', + 'build_storybook_step', + 'copy_packages_for_docker_step', + 'build_docker_images_step', + 'publish_images_step', + 'postgres_integration_tests_step', + 'mysql_integration_tests_step', + 'redis_integration_tests_step', + 'memcached_integration_tests_step', + 'get_windows_steps', + 'benchmark_ldap_step', + 'enterprise_downstream_step', + 'frontend_metrics_step', + 'store_storybook_step', + 'release_canary_npm_packages_step', + 'upload_packages_step', + 'store_packages_step', + 'upload_cdn_step', + 'verify_gen_cue_step', + 'test_a11y_frontend_step', + 'trigger_oss', + 'betterer_frontend_step', + 'trigger_test_release' +) + +load( + 'scripts/drone/utils/utils.star', + 'pipeline', +) + +def build_e2e(trigger, ver_mode, edition): + variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64',] + init_steps = [ + identify_runner_step(), + download_grabpl_step(), + gen_version_step(ver_mode), + verify_gen_cue_step(edition="oss"), + wire_install_step(), + yarn_install_step(), + ] + build_steps = [] + if ver_mode == 'main': + build_steps.extend([trigger_test_release()]) + build_steps.extend([ + enterprise_downstream_step(edition=edition, ver_mode=ver_mode), + build_backend_step(edition=edition, ver_mode=ver_mode), + build_frontend_step(edition=edition, ver_mode=ver_mode), + build_frontend_package_step(edition=edition, ver_mode=ver_mode), + build_plugins_step(edition=edition, ver_mode=ver_mode), + ]) + if ver_mode == 'main': + build_steps.extend([package_step(edition=edition, ver_mode=ver_mode)]) + elif ver_mode == 'pr': + build_steps.extend([package_step(edition=edition, ver_mode=ver_mode, variants=variants)]) + + build_steps.extend([ + grafana_server_step(edition=edition), + e2e_tests_step('dashboards-suite', edition=edition), + e2e_tests_step('smoke-tests-suite', edition=edition), + e2e_tests_step('panels-suite', edition=edition), + e2e_tests_step('various-suite', edition=edition), + e2e_tests_artifacts(edition=edition), + build_storybook_step(edition=edition, ver_mode=ver_mode), + copy_packages_for_docker_step(), + test_a11y_frontend_step(ver_mode=ver_mode, edition=edition), + ]) + if ver_mode == 'main': + build_steps.extend([ + store_storybook_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss), + frontend_metrics_step(edition=edition, trigger=trigger_oss) + ]) + + if ver_mode == 'main': + build_steps.extend([ + build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False), + build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False), + publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana', trigger=trigger_oss), + publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana-oss', trigger=trigger_oss), + release_canary_npm_packages_step(edition, trigger=trigger_oss), + upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss), + upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss) + ]) + elif ver_mode == 'pr': + build_steps.extend([build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64', ])]) + + publish_suffix = '' + if ver_mode == 'main': + publish_suffix = '-publish' + + return pipeline( + name='{}-build-e2e{}'.format(ver_mode, publish_suffix), edition="oss", trigger=trigger, services=[], steps=init_steps + build_steps, + ) diff --git a/scripts/drone/pipelines/integration_tests.star b/scripts/drone/pipelines/integration_tests.star new file mode 100644 index 00000000000..72d1d8a0489 --- /dev/null +++ b/scripts/drone/pipelines/integration_tests.star @@ -0,0 +1,40 @@ +load( + 'scripts/drone/steps/lib.star', + 'identify_runner_step', + 'download_grabpl_step', + 'verify_gen_cue_step', + 'wire_install_step', + 'postgres_integration_tests_step', + 'mysql_integration_tests_step', +) + +load( + 'scripts/drone/services/services.star', + 'integration_test_services', + 'integration_test_services_volumes', + 'ldap_service', +) + +load( + 'scripts/drone/utils/utils.star', + 'pipeline', +) + +def integration_tests(trigger, ver_mode, edition): + services = integration_test_services(edition) + volumes = integration_test_services_volumes() + init_steps = [ + download_grabpl_step(), + identify_runner_step(), + verify_gen_cue_step(edition="oss"), + wire_install_step(), + ] + test_steps = [ + postgres_integration_tests_step(edition=edition, ver_mode=ver_mode), + mysql_integration_tests_step(edition=edition, ver_mode=ver_mode), + ] + + return pipeline( + name='{}-integration-tests'.format(ver_mode), edition="oss", trigger=trigger, services=services, steps=init_steps + test_steps, + volumes=volumes + ) diff --git a/scripts/drone/pipelines/main.star b/scripts/drone/pipelines/main.star deleted file mode 100644 index 27c8b678c52..00000000000 --- a/scripts/drone/pipelines/main.star +++ /dev/null @@ -1,243 +0,0 @@ -load( - 'scripts/drone/steps/lib.star', - 'download_grabpl_step', - 'build_image', - 'identify_runner_step', - 'gen_version_step', - 'wire_install_step', - 'yarn_install_step', - 'lint_drone_step', - 'lint_backend_step', - 'lint_frontend_step', - 'codespell_step', - 'shellcheck_step', - 'test_backend_step', - 'test_backend_integration_step', - 'test_frontend_step', - 'build_backend_step', - 'build_frontend_step', - 'build_frontend_package_step', - 'build_plugins_step', - 'package_step', - 'grafana_server_step', - 'e2e_tests_step', - 'e2e_tests_artifacts', - 'build_storybook_step', - 'copy_packages_for_docker_step', - 'build_docker_images_step', - 'publish_images_step', - 'postgres_integration_tests_step', - 'mysql_integration_tests_step', - 'redis_integration_tests_step', - 'memcached_integration_tests_step', - 'get_windows_steps', - 'benchmark_ldap_step', - 'enterprise_downstream_step', - 'frontend_metrics_step', - 'store_storybook_step', - 'release_canary_npm_packages_step', - 'upload_packages_step', - 'store_packages_step', - 'upload_cdn_step', - 'verify_gen_cue_step', - 'test_a11y_frontend_step', - 'trigger_oss', - 'betterer_frontend_step' -) - -load( - 'scripts/drone/services/services.star', - 'integration_test_services', - 'integration_test_services_volumes', - 'ldap_service', -) - -load( - 'scripts/drone/utils/utils.star', - 'pipeline', - 'notify_pipeline', - 'failure_template', - 'drone_change_template', -) - -load( - 'scripts/drone/pipelines/docs.star', - 'docs_pipelines', -) - -load('scripts/drone/vault.star', 'from_secret') - - -ver_mode = 'main' -trigger = { - 'event': ['push',], - 'branch': 'main', -} - -def main_test_frontend(): - init_steps = [ - identify_runner_step(), - download_grabpl_step(), - gen_version_step(ver_mode), - yarn_install_step(), - ] - test_steps = [ - lint_frontend_step(), - betterer_frontend_step(), - test_frontend_step(), - ] - return pipeline( - name='main-test-frontend', edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps, - ) - -def main_test_backend(): - init_steps = [ - identify_runner_step(), - download_grabpl_step(), - gen_version_step(ver_mode), - verify_gen_cue_step(edition="oss"), - wire_install_step(), - ] - test_steps = [ - lint_drone_step(), - codespell_step(), - shellcheck_step(), - lint_backend_step(edition="oss"), - test_backend_step(edition="oss"), - test_backend_integration_step(edition="oss"), - ] - return pipeline( - name='main-test-backend', edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps, - ) - - -def get_steps(edition): - init_steps = [ - identify_runner_step(), - download_grabpl_step(), - gen_version_step(ver_mode), - verify_gen_cue_step(edition="oss"), - wire_install_step(), - yarn_install_step(), - ] - build_steps = [ - trigger_test_release(), - enterprise_downstream_step(edition=edition, ver_mode=ver_mode), - build_backend_step(edition=edition, ver_mode=ver_mode), - build_frontend_step(edition=edition, ver_mode=ver_mode), - build_frontend_package_step(edition=edition, ver_mode=ver_mode), - build_plugins_step(edition=edition, ver_mode=ver_mode), - ] - integration_test_steps = [ - postgres_integration_tests_step(edition=edition, ver_mode=ver_mode), - mysql_integration_tests_step(edition=edition, ver_mode=ver_mode), - ] - - # Insert remaining steps - build_steps.extend([ - package_step(edition=edition, ver_mode=ver_mode), - grafana_server_step(edition=edition), - e2e_tests_step('dashboards-suite', edition=edition), - e2e_tests_step('smoke-tests-suite', edition=edition), - e2e_tests_step('panels-suite', edition=edition), - e2e_tests_step('various-suite', edition=edition), - e2e_tests_artifacts(edition=edition), - build_storybook_step(edition=edition, ver_mode=ver_mode), - store_storybook_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss), - test_a11y_frontend_step(ver_mode=ver_mode, edition=edition), - frontend_metrics_step(edition=edition, trigger=trigger_oss), - copy_packages_for_docker_step(), - build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False), - build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False), - publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana', trigger=trigger_oss), - publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana-oss', trigger=trigger_oss) - ]) - - build_steps.extend([ - release_canary_npm_packages_step(edition, trigger=trigger_oss), - upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss), - upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss) - ]) - - windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode) - store_steps = [store_packages_step(edition=edition, ver_mode=ver_mode),] - - return init_steps, build_steps, integration_test_steps, windows_steps, store_steps - -def trigger_test_release(): - return { - 'name': 'trigger-test-release', - 'image': build_image, - 'environment': { - 'GITHUB_TOKEN': from_secret('github_token'), - 'DOWNSTREAM_REPO': from_secret('downstream'), - 'TEST_TAG': 'v0.0.0-test', - }, - 'commands': [ - 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" --depth=1', - 'cd grafana-enterprise', - 'git fetch origin "refs/tags/*:refs/tags/*"', - 'git tag -d $${TEST_TAG} && git push --delete origin $${TEST_TAG} && git tag $${TEST_TAG} && git push origin $${TEST_TAG}', - 'cd -', - 'git fetch origin "refs/tags/*:refs/tags/*"', - 'git remote add downstream https://$${GITHUB_TOKEN}@github.com/grafana/$${DOWNSTREAM_REPO}.git', - 'git tag -d $${TEST_TAG} && git push --delete downstream --quiet $${TEST_TAG} && git tag $${TEST_TAG} && git push downstream $${TEST_TAG} --quiet', - ], - 'failure': 'ignore', - 'when': { - 'paths': { - 'include': [ - '.drone.yml', - ] - }, - 'repo': [ - 'grafana/grafana', - ] - } - } - -def main_pipelines(edition): - services = integration_test_services(edition) - volumes = integration_test_services_volumes() - drone_change_trigger = { - 'event': ['push',], - 'branch': 'main', - 'repo': [ - 'grafana/grafana', - ], - 'paths': { - 'include': [ - '.drone.yml', - ], - 'exclude': [ - 'exclude', - ], - }, - } - init_steps, build_steps, integration_test_steps, windows_steps, store_steps = get_steps(edition=edition) - - pipelines = [docs_pipelines(edition, ver_mode, trigger), main_test_frontend(), main_test_backend(), pipeline( - name='main-build-e2e-publish', edition=edition, trigger=trigger, services=[], - steps=init_steps + build_steps, - ), pipeline( - name='main-integration-tests', edition=edition, trigger=trigger, services=services, - steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(edition="oss"), wire_install_step(), ] + integration_test_steps, - volumes=volumes, - ), pipeline( - name='main-windows', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), - steps=[identify_runner_step('windows')] + windows_steps, - depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows', - ), notify_pipeline( - name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger, - template=drone_change_template, secret='drone-changes-webhook', - ), pipeline( - name='main-publish', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), - steps=[download_grabpl_step(), gen_version_step(ver_mode), identify_runner_step(),] + store_steps, - depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', ], - ), notify_pipeline( - name='main-notify', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']), - depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', 'main-publish'], - template=failure_template, secret='slack_webhook' - )] - - return pipelines diff --git a/scripts/drone/pipelines/pr.star b/scripts/drone/pipelines/pr.star deleted file mode 100644 index 5e7f50d5238..00000000000 --- a/scripts/drone/pipelines/pr.star +++ /dev/null @@ -1,192 +0,0 @@ -load( - 'scripts/drone/steps/lib.star', - 'download_grabpl_step', - 'gen_version_step', - 'yarn_install_step', - 'wire_install_step', - 'identify_runner_step', - 'lint_drone_step', - 'lint_backend_step', - 'lint_frontend_step', - 'codespell_step', - 'shellcheck_step', - 'build_backend_step', - 'build_frontend_step', - 'build_frontend_package_step', - 'build_plugins_step', - 'test_backend_step', - 'test_backend_integration_step', - 'test_frontend_step', - 'package_step', - 'grafana_server_step', - 'e2e_tests_step', - 'e2e_tests_artifacts', - 'build_storybook_step', - 'copy_packages_for_docker_step', - 'build_docker_images_step', - 'postgres_integration_tests_step', - 'mysql_integration_tests_step', - 'redis_integration_tests_step', - 'memcached_integration_tests_step', - 'benchmark_ldap_step', - 'verify_gen_cue_step', - 'test_a11y_frontend_step', - 'enterprise_downstream_step', - 'betterer_frontend_step', -) - -load( - 'scripts/drone/services/services.star', - 'integration_test_services', - 'integration_test_services_volumes', - 'ldap_service', -) - -load( - 'scripts/drone/utils/utils.star', - 'notify_pipeline', - 'pipeline', - 'failure_template', - 'drone_change_template', -) - -load( - 'scripts/drone/pipelines/docs.star', - 'docs_pipelines', - 'trigger_docs', -) - -ver_mode = 'pr' -trigger = { - 'event': [ - 'pull_request', - ], - 'paths': { - 'exclude': [ - '*.md', - 'docs/**', - 'latest.json', - ], - }, -} - -def pr_verify_drone(): - steps = [ - identify_runner_step(), - download_grabpl_step(), - lint_drone_step(), - ] - return pipeline( - name='pr-verify-drone', edition="oss", trigger=get_pr_trigger(include_paths=['scripts/drone/**', '.drone.yml', '.drone.star']), services=[], steps=steps, - ) - - -def pr_test_frontend(): - init_steps = [ - identify_runner_step(), - download_grabpl_step(), - gen_version_step(ver_mode), - yarn_install_step(), - ] - test_steps = [ - lint_frontend_step(), - betterer_frontend_step(), - test_frontend_step(), - ] - return pipeline( - name='pr-test-frontend', edition="oss", trigger=get_pr_trigger(exclude_paths=['pkg/**', 'packaging/**', 'go.sum', 'go.mod']), services=[], steps=init_steps + test_steps, - ) - - -def pr_test_backend(): - init_steps = [ - identify_runner_step(), - download_grabpl_step(), - gen_version_step(ver_mode), - verify_gen_cue_step(edition="oss"), - wire_install_step(), - ] - test_steps = [ - codespell_step(), - shellcheck_step(), - lint_backend_step(edition="oss"), - test_backend_step(edition="oss"), - test_backend_integration_step(edition="oss"), - ] - return pipeline( - name='pr-test-backend', edition="oss", trigger=get_pr_trigger(include_paths=['pkg/**', 'packaging/**', '.drone.yml', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json']), services=[], steps=init_steps + test_steps, - ) - - -def pr_pipelines(edition): - services = integration_test_services(edition) - volumes = integration_test_services_volumes() - variants = ['linux-amd64', 'linux-amd64-musl', 'darwin-amd64', 'windows-amd64',] - init_steps = [ - identify_runner_step(), - download_grabpl_step(), - gen_version_step(ver_mode), - verify_gen_cue_step(edition="oss"), - wire_install_step(), - yarn_install_step(), - ] - build_steps = [ - enterprise_downstream_step(edition=edition, ver_mode=ver_mode), - build_backend_step(edition=edition, ver_mode=ver_mode), - build_frontend_step(edition=edition, ver_mode=ver_mode), - build_frontend_package_step(edition=edition, ver_mode=ver_mode), - build_plugins_step(edition=edition, ver_mode=ver_mode), - ] - integration_test_steps = [ - postgres_integration_tests_step(edition=edition, ver_mode=ver_mode), - mysql_integration_tests_step(edition=edition, ver_mode=ver_mode), - ] - - # Insert remaining build_steps - build_steps.extend([ - package_step(edition=edition, ver_mode=ver_mode, variants=variants), - grafana_server_step(edition=edition), - e2e_tests_step('dashboards-suite', edition=edition), - e2e_tests_step('smoke-tests-suite', edition=edition), - e2e_tests_step('panels-suite', edition=edition), - e2e_tests_step('various-suite', edition=edition), - e2e_tests_artifacts(edition=edition), - build_storybook_step(edition=edition, ver_mode=ver_mode), - test_a11y_frontend_step(ver_mode=ver_mode, edition=edition), - copy_packages_for_docker_step(), - build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64', ]), - ]) - - return [ - pr_verify_drone(), - pr_test_frontend(), - pr_test_backend(), - pipeline( - name='pr-build-e2e', edition=edition, trigger=trigger, services=[], steps=init_steps + build_steps, - ), pipeline( - name='pr-integration-tests', edition=edition, trigger=trigger, services=services, - steps=[download_grabpl_step(), identify_runner_step(), verify_gen_cue_step(edition="oss"), wire_install_step(), ] + integration_test_steps, - volumes=volumes, - ), docs_pipelines(edition, ver_mode, trigger_docs()) - ] - - -def get_pr_trigger(include_paths=None, exclude_paths=None): - paths_ex = ['docs/**', '*.md'] - paths_in = [] - if include_paths: - for path in include_paths: - paths_in.extend([path]) - if exclude_paths: - for path in exclude_paths: - paths_ex.extend([path]) - return { - 'event': [ - 'pull_request', - ], - 'paths': { - 'exclude': paths_ex, - 'include': paths_in, - }, - } - diff --git a/scripts/drone/pipelines/publish.star b/scripts/drone/pipelines/publish.star new file mode 100644 index 00000000000..1f75ba89271 --- /dev/null +++ b/scripts/drone/pipelines/publish.star @@ -0,0 +1,25 @@ +load( + 'scripts/drone/steps/lib.star', + 'identify_runner_step', + 'gen_version_step', + 'download_grabpl_step', + 'store_packages_step' +) + +load( + 'scripts/drone/utils/utils.star', + 'pipeline', +) + +def publish(trigger, ver_mode, edition): + steps = [ + download_grabpl_step(), + gen_version_step(ver_mode), + identify_runner_step(), + store_packages_step(edition=edition, ver_mode=ver_mode), + ] + return pipeline( + name='main-publish', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), + steps=steps, + depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows', ], + ) diff --git a/scripts/drone/pipelines/test_backend.star b/scripts/drone/pipelines/test_backend.star new file mode 100644 index 00000000000..25b7193a272 --- /dev/null +++ b/scripts/drone/pipelines/test_backend.star @@ -0,0 +1,41 @@ +load( + 'scripts/drone/steps/lib.star', + 'identify_runner_step', + 'download_grabpl_step', + 'gen_version_step', + 'wire_install_step', + 'codespell_step', + 'shellcheck_step', + 'lint_backend_step', + 'lint_drone_step', + 'test_backend_step', + 'test_backend_integration_step', + 'verify_gen_cue_step', +) + +load( + 'scripts/drone/utils/utils.star', + 'pipeline', +) + +def test_backend(trigger, ver_mode): + init_steps = [ + identify_runner_step(), + download_grabpl_step(), + gen_version_step(ver_mode), + verify_gen_cue_step(edition="oss"), + wire_install_step(), + ] + test_steps = [ + codespell_step(), + shellcheck_step(), + lint_backend_step(edition="oss"), + test_backend_step(edition="oss"), + test_backend_integration_step(edition="oss"), + ] + if ver_mode == 'main': + test_steps.extend([lint_drone_step()]) + + return pipeline( + name='{}-test-backend'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps, + ) diff --git a/scripts/drone/pipelines/test_frontend.star b/scripts/drone/pipelines/test_frontend.star new file mode 100644 index 00000000000..0231f19fb6d --- /dev/null +++ b/scripts/drone/pipelines/test_frontend.star @@ -0,0 +1,31 @@ +load( + 'scripts/drone/steps/lib.star', + 'identify_runner_step', + 'download_grabpl_step', + 'gen_version_step', + 'yarn_install_step', + 'lint_frontend_step', + 'betterer_frontend_step', + 'test_frontend_step', +) + +load( + 'scripts/drone/utils/utils.star', + 'pipeline', +) + +def test_frontend(trigger, ver_mode): + init_steps = [ + identify_runner_step(), + download_grabpl_step(), + gen_version_step(ver_mode), + yarn_install_step(), + ] + test_steps = [ + lint_frontend_step(), + betterer_frontend_step(), + test_frontend_step(), + ] + return pipeline( + name='{}-test-frontend'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=init_steps + test_steps, + ) diff --git a/scripts/drone/pipelines/verify_drone.star b/scripts/drone/pipelines/verify_drone.star new file mode 100644 index 00000000000..46b247d737a --- /dev/null +++ b/scripts/drone/pipelines/verify_drone.star @@ -0,0 +1,21 @@ +load( + 'scripts/drone/steps/lib.star', + 'identify_runner_step', + 'download_grabpl_step', + 'lint_drone_step', +) + +load( + 'scripts/drone/utils/utils.star', + 'pipeline', +) + +def verify_drone(trigger, ver_mode): + steps = [ + identify_runner_step(), + download_grabpl_step(), + lint_drone_step(), + ] + return pipeline( + name='{}-verify-drone'.format(ver_mode), edition="oss", trigger=trigger, services=[], steps=steps, + ) diff --git a/scripts/drone/pipelines/windows.star b/scripts/drone/pipelines/windows.star new file mode 100644 index 00000000000..3b60dee87f8 --- /dev/null +++ b/scripts/drone/pipelines/windows.star @@ -0,0 +1,137 @@ +load( + 'scripts/drone/steps/lib.star', + 'grabpl_version', + 'wix_image', + 'identify_runner_step', +) + +load( + 'scripts/drone/utils/utils.star', + 'pipeline', +) + +load('scripts/drone/vault.star', 'from_secret', 'prerelease_bucket', 'github_token') + +def windows(trigger, edition, ver_mode): + init_cmds = [] + sfx = '' + if edition in ('enterprise', 'enterprise2'): + sfx = '-{}'.format(edition) + else: + init_cmds.extend([ + '$$ProgressPreference = "SilentlyContinue"', + 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format( + grabpl_version), + ]) + steps = [ + { + 'name': 'windows-init', + 'image': wix_image, + 'commands': init_cmds, + }, + ] + if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in ( + 'release', 'release-branch', + ): + bucket_part = '' + bucket = '%PRERELEASE_BUCKET%/artifacts/downloads' + if ver_mode == 'release': + ver_part = '${DRONE_TAG}' + dir = 'release' + else: + dir = 'main' + bucket = 'grafana-downloads' + bucket_part = ' --packages-bucket {}'.format(bucket) + build_no = 'DRONE_BUILD_NUMBER' + ver_part = '--build-id $$env:{}'.format(build_no) + installer_commands = [ + '$$gcpKey = $$env:GCP_KEY', + '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json', + # gcloud fails to read the file unless converted with dos2unix + 'dos2unix gcpkey.json', + 'gcloud auth activate-service-account --key-file=gcpkey.json', + 'rm gcpkey.json', + 'cp C:\\App\\nssm-2.24.zip .', + ] + if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in ( + 'release', + ): + installer_commands.extend([ + '.\\grabpl.exe gen-version {}'.format(ver_part), + '.\\grabpl.exe windows-installer --edition {}{} {}'.format(edition, bucket_part, ver_part), + '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]', + ]) + if ver_mode == 'main': + installer_commands.extend([ + 'gsutil cp $$fname gs://{}/{}/{}/'.format(bucket, edition, dir), + 'gsutil cp "$$fname.sha256" gs://{}/{}/{}/'.format(bucket, edition, dir), + ]) + else: + installer_commands.extend([ + 'gsutil cp $$fname gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir), + 'gsutil cp "$$fname.sha256" gs://{}/{}/{}/{}/'.format(bucket, ver_part, edition, dir), + ]) + steps.append({ + 'name': 'build-windows-installer', + 'image': wix_image, + 'depends_on': [ + 'windows-init', + ], + 'environment': { + 'GCP_KEY': from_secret('gcp_key'), + 'PRERELEASE_BUCKET': from_secret(prerelease_bucket), + 'GITHUB_TOKEN': from_secret('github_token') + }, + 'commands': installer_commands, + }) + + if edition in ('enterprise', 'enterprise2'): + if ver_mode == 'release': + committish = '${DRONE_TAG}' + elif ver_mode == 'release-branch': + committish = '$$env:DRONE_BRANCH' + else: + committish = '$$env:DRONE_COMMIT' + # For enterprise, we have to clone both OSS and enterprise and merge the latter into the former + download_grabpl_step_cmds = [ + '$$ProgressPreference = "SilentlyContinue"', + 'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format( + grabpl_version), + ] + clone_cmds = [ + 'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"', + ] + clone_cmds.extend([ + 'cd grafana-enterprise', + 'git checkout {}'.format(committish), + ]) + steps.insert(0, { + 'name': 'clone', + 'image': wix_image, + 'environment': { + 'GITHUB_TOKEN': from_secret(github_token), + }, + 'commands': download_grabpl_step_cmds + clone_cmds, + }) + steps[1]['depends_on'] = [ + 'clone', + ] + steps[1]['commands'].extend([ + # Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into + 'cp -r grafana-enterprise C:\\App\\grafana-enterprise', + 'rm -r -force grafana-enterprise', + 'cp grabpl.exe C:\\App\\grabpl.exe', + 'rm -force grabpl.exe', + 'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise', + 'cp C:\\App\\grabpl.exe grabpl.exe', + ]) + if 'environment' in steps[1]: + steps[1]['environment'] + {'GITHUB_TOKEN': from_secret(github_token)} + else: + steps[1]['environment'] = {'GITHUB_TOKEN': from_secret(github_token)} + + return pipeline( + name='main-windows', edition=edition, trigger=dict(trigger, repo=['grafana/grafana']), + steps=[identify_runner_step('windows')] + steps, + depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests'], platform='windows', + ) diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index a58217809da..152eef722f3 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -1173,6 +1173,38 @@ def verify_gen_cue_step(edition): ], } +def trigger_test_release(): + return { + 'name': 'trigger-test-release', + 'image': build_image, + 'environment': { + 'GITHUB_TOKEN': from_secret('github_token'), + 'DOWNSTREAM_REPO': from_secret('downstream'), + 'TEST_TAG': 'v0.0.0-test', + }, + 'commands': [ + 'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" --depth=1', + 'cd grafana-enterprise', + 'git fetch origin "refs/tags/*:refs/tags/*"', + 'git tag -d $${TEST_TAG} && git push --delete origin $${TEST_TAG} && git tag $${TEST_TAG} && git push origin $${TEST_TAG}', + 'cd -', + 'git fetch origin "refs/tags/*:refs/tags/*"', + 'git remote add downstream https://$${GITHUB_TOKEN}@github.com/grafana/$${DOWNSTREAM_REPO}.git', + 'git tag -d $${TEST_TAG} && git push --delete downstream --quiet $${TEST_TAG} && git tag $${TEST_TAG} && git push downstream $${TEST_TAG} --quiet', + ], + 'failure': 'ignore', + 'when': { + 'paths': { + 'include': [ + '.drone.yml', + ] + }, + 'repo': [ + 'grafana/grafana', + ] + } + } + def artifacts_page_step(): return { 'name': 'artifacts-page', diff --git a/scripts/drone/version.star b/scripts/drone/version.star index 9a5b39d41de..80f0c6ac48f 100644 --- a/scripts/drone/version.star +++ b/scripts/drone/version.star @@ -1,5 +1,5 @@ load( - 'scripts/drone/pipelines/release.star', + 'scripts/drone/events/release.star', 'release_pipelines', )