From f9fffd3ff1477eda60296b3ae9da43919bdb07be Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 09:48:34 +0000 Subject: [PATCH] Update `make docs` procedure (#77536) * Update `make docs` procedure * Update `make docs` procedure --------- Co-authored-by: grafanabot --- docs/docs.mk | 5 - docs/make-docs | 303 ++++++++++++++++++++++++++++++++++--------------- 2 files changed, 213 insertions(+), 95 deletions(-) diff --git a/docs/docs.mk b/docs/docs.mk index c4ba766fb4f..a08830d907c 100644 --- a/docs/docs.mk +++ b/docs/docs.mk @@ -34,11 +34,6 @@ endif # First project is considered the primary one used for doc-validator. PRIMARY_PROJECT := $(subst /,-,$(firstword $(subst :, ,$(firstword $(PROJECTS))))) -# Name for the container. -ifeq ($(origin DOCS_CONTAINER), undefined) -export DOCS_CONTAINER := $(PRIMARY_PROJECT)-docs -endif - # Host port to publish container port to. ifeq ($(origin DOCS_HOST_PORT), undefined) export DOCS_HOST_PORT := 3002 diff --git a/docs/make-docs b/docs/make-docs index 94683de344c..751e22f4fd4 100755 --- a/docs/make-docs +++ b/docs/make-docs @@ -5,107 +5,146 @@ # Updates should conform to the guidelines in https://keepachangelog.com/en/1.1.0/. # [Semantic versioning](https://semver.org/) is used to help the reader identify the significance of changes. # Changes are relevant to this script and the support docs.mk GNU Make interface. - +# +# ## 5.1.1 (2023-10-30) +# +# ### Added +# +# - Support for Datadog and Oracle data source plugins repositories. +# +# ## 5.1.0 (2023-10-20) +# +# ### Added +# +# - Support for the plugins monorepo. +# +# ## 5.0.0 (2023-10-18) +# +# ### Added +# +# - Improved support for website repository. +# +# Mount more content and provide some feedback to users that the build can take time. +# +# - Ability to enter the `grafana/docs-base` container with a shell using the `ENTER` environment variable. +# +# ### Fixed +# +# - Correct key combination for interrupting the process. +# +# Keyboards use capital letters so this more accurately reflects the exact key combination users are expected to press. +# +# ### Removed +# +# - Imperfect implementation of container name. +# +# Facilitates running `make vale` and `make docs` at once. +# Container names are convenient for recognition in `docker ps` but the current implementation has more downsides than upsides. +# +# - Forced platform specification now that multiple architecture images exist. +# +# Significantly speeds up build times on larger repositories. +# # ## 4.2.2 (2023-10-05) # - Added support for Jira data source and MongoDB data source plugins repositories. - +# # ## 4.2.1 (2023-09-13) # ## Fixed - +# # - Improved consistency of the webserver request loop by polling the Hugo port rather than the proxy port. - +# # ## 4.2.0 (2023-09-01) - +# # ### Added - +# # - Retry the initial webserver request up to ten times to allow for the process to start. # If it is still failing after ten seconds, an error message is logged. - +# # ## 4.1.1 (2023-07-20) - +# # ### Fixed - +# # - Replaced use of `realpath` with POSIX compatible alternative to determine default value for REPOS_PATH. - +# # ## 4.1.0 (2023-06-16) - +# # ### Added - +# # - Mounts of `layouts` and `config` directories for the `website` project. # Ensures that local changes to mounts or shortcodes are reflected in the development server. - +# # ### Fixed - +# # - Version inference for versioned docs pages. # Pages in versioned projects now have the `versioned: true` front matter set to ensure that "version" in $.Page.Scratch is set on builds. - +# # ## 4.0.0 (2023-06-06) - +# # ### Removed - +# # - `doc-validator/%` target. # The behavior of the target was not as described. # Instead, to limit `doc-validator` to only specific files, refer to https://grafana.com/docs/writers-toolkit/writing-guide/tooling-and-workflows/validate-technical-documentation/#run-on-specific-files. - +# # ## 3.0.0 (2023-05-18) - +# # ### Fixed - +# # - Compatibility with the updated Make targets in the `website` repository. # `docs` now runs this script itself, `server-docs` builds the site with the `docs` Hugo environment. - +# # ## 2.0.0 (2023-05-18) - +# # ### Added - +# # - Support for the grafana-cloud/frontend-observability/faro-web-sdk project. # - Use of `doc-validator` v2.0.x which includes breaking changes to command line options. - +# # ### Fixed - +# # - Source grafana-cloud project from website repository. - +# # ### Added - +# # - Support for running the Vale linter with `make vale`. - +# # ## 1.2.1 (2023-05-05) - +# # ### Fixed - +# # - Use `latest` tag of `grafana/vale` image by default instead of hardcoded older version. # - Fix mounting multiple projects broken by the changes in 1.0.1 - +# # ## 1.2.0 (2023-05-05) - +# # ### Added - +# # - Support for running the Vale linter with `make vale`. - +# # ### Fixed - +# # ## 1.1.0 (2023-05-05) - +# # ### Added - +# # - Rewrite error output so it can be followed by text editors. - +# # ### Fixed - +# # - Fix `docs-debug` container process port. - +# # ## 1.0.1 (2023-05-04) - +# # ### Fixed - +# # - Ensure complete section hierarchy so that all projects have a visible menu. - +# # ## 1.0.0 (2023-05-04) - +# # ### Added - +# # - Build multiple projects simultaneously if all projects are checked out locally. # - Run [`doc-validator`](https://github.com/grafana/technical-documentation/tree/main/tools/cmd/doc-validator) over projects. # - Redirect project root to mounted version. @@ -135,7 +174,6 @@ set -ef -readonly DOCS_CONTAINER="${DOCS_CONTAINER:-make-docs}" readonly DOCS_HOST_PORT="${DOCS_HOST_PORT:-3002}" readonly DOCS_IMAGE="${DOCS_IMAGE:-grafana/docs-base:latest}" @@ -217,8 +255,10 @@ SOURCES_grafana_cloud_frontend_observability_faro_web_sdk='faro-web-sdk' SOURCES_helm_charts_mimir_distributed='mimir' SOURCES_helm_charts_tempo_distributed='tempo' SOURCES_opentelemetry='opentelemetry-docs' +SOURCES_plugins_grafana_datadog_datasource='datadog-datasource' SOURCES_plugins_grafana_jira_datasource='jira-datasource' SOURCES_plugins_grafana_mongodb_datasource='mongodb-datasource' +SOURCES_plugins_grafana_oracle_datasource='oracle-datasource' SOURCES_plugins_grafana_splunk_datasource='splunk-datasource' VERSIONS_as_code='UNVERSIONED' @@ -229,6 +269,11 @@ VERSIONS_grafana_cloud_k6='UNVERSIONED' VERSIONS_grafana_cloud_data_configuration_integrations='UNVERSIONED' VERSIONS_grafana_cloud_frontend_observability_faro_web_sdk='UNVERSIONED' VERSIONS_opentelemetry='UNVERSIONED' +VERSIONS_plugins_grafana_datadog_datasource='latest' +VERSIONS_plugins_grafana_jira_datasource='latest' +VERSIONS_plugins_grafana_mongodb_datasource='latest' +VERSIONS_plugins_grafana_oracle_datasource='latest' +VERSIONS_plugins_grafana_splunk_datasource='latest' VERSIONS_technical_documentation='UNVERSIONED' VERSIONS_website='UNVERSIONED' VERSIONS_writers_toolkit='UNVERSIONED' @@ -237,8 +282,13 @@ PATHS_grafana_cloud='content/docs/grafana-cloud' PATHS_helm_charts_mimir_distributed='docs/sources/helm-charts/mimir-distributed' PATHS_helm_charts_tempo_distributed='docs/sources/helm-charts/tempo-distributed' PATHS_mimir='docs/sources/mimir' +PATHS_plugins_grafana_datadog_datasource='docs/sources' +PATHS_plugins_grafana_jira_datasource='docs/sources' +PATHS_plugins_grafana_mongodb_datasource='docs/sources' +PATHS_plugins_grafana_oracle_datasource='docs/sources' +PATHS_plugins_grafana_splunk_datasource='docs/sources' PATHS_tempo='docs/sources/tempo' -PATHS_website='content/docs' +PATHS_website='content' # identifier STR # Replace characters that are not valid in an identifier with underscores. @@ -253,6 +303,77 @@ aget() { eval echo '$'"$(identifier "$1")_$(identifier "$2")" } +# src returns the project source repository name for a project. +src() { + _project="$1" + + case "${_project}" in + plugins/*) + if [ -z "$(aget SOURCES "${_project}")" ]; then + echo plugins-private + else + aget SOURCES "${_project}" + fi + ;; + *) + if [ -z "$(aget SOURCES "${_project}")" ]; then + echo "${_project}" + else + aget SOURCES "${_project}" + fi + ;; + esac + + unset _project +} + +# path returns the relative path within the repository that contain the docs for a project. +path() { + _project="$1" + + case "${_project}" in + plugins/*) + if [ -z "$(aget PATHS "${_project}")" ]; then + echo "${_project}/docs/sources" + else + aget PATHS "${_project}" + fi + ;; + *) + if [ -z "$(aget PATHS "${_project}")" ]; then + echo "docs/sources" + else + aget PATHS "${_project}" + fi + esac + + unset _project +} + +# version returns the version for a project. Unversioned projects return the special value 'UNVERSIONED'. +version() { + _project="$1" + + case "${_project}" in + plugins/*) + if [ -z "$(aget VERSIONS "${_project}")" ]; then + echo "UNVERSIONED" + else + aget VERSIONS "${_project}" + fi + ;; + *) + if [ -z "$(aget VERSIONS "${_project}")" ]; then + echo latest + else + aget VERSIONS "${_project}" + fi + esac + + unset _project +} + + # new_proj populates a new project structure. new_proj() { _project="$1" @@ -263,31 +384,19 @@ new_proj() { # If version is not set, use the script mapping of project to default versions if it exists. # Fallback to 'latest'. if [ -z "${_version}" ]; then - if [ -z "$(aget VERSIONS "${_project}")" ]; then - _version=latest - else - _version="$(aget VERSIONS "${_project}")" - fi + _version="$(version "${_project}")" fi # If repo is not set, use the script mapping of project to repo name if it exists. # Fallback to using the project name. if [ -z "${_repo}" ]; then - if [ -z "$(aget SOURCES "${_project}")" ]; then - _repo="${_project}" - else - _repo="$(aget SOURCES "${_project}")" - fi + _repo="$(src "${_project}")" fi # If path is not set, use the script mapping of project to docs sources path if it exists. # Fallback to using 'docs/sources'. if [ -z "${_path}" ]; then - if [ -z "$(aget PATHS "${_project}")" ]; then - _path="docs/sources" - else - _path="$(aget PATHS "${_project}")" - fi + _path="$(path "${_project}")" fi echo "${_project}:${_version}:${_repo}:${_path}" @@ -336,7 +445,7 @@ $1 POSIX_HERESTRING if [ "${_project}" = 'website' ]; then - echo '/hugo/content/docs' + echo '/hugo/content' unset _project _version return @@ -481,7 +590,7 @@ POSIX_HERESTRING fi done echo - echo 'Press Ctrl+c to stop the server' + echo 'Press Ctrl+C to stop the server' unset i max req url return @@ -490,7 +599,7 @@ POSIX_HERESTRING echo errr 'The build was interrupted or a build error occurred, check the previous logs for possible causes.' - note 'You might need to use Ctrl+c to end the process.' + note 'You might need to use Ctrl+C to end the process.' unset i max req url } @@ -519,10 +628,12 @@ for arg in "$@"; do ${arg} POSIX_HERESTRING if [ "${_project}" = website ]; then + note "Please be patient, building the website can take some time." + _repo="$(repo_path website)" volumes="--volume=${_repo}/config:/hugo/config" - volumes="${volumes} --volume=${_repo}/layouts/partials:/hugo/layouts/partials" - volumes="${volumes} --volume=${_repo}/layouts/shortcodes:/hugo/layouts/shortcodes" + volumes="${volumes} --volume=${_repo}/layouts:/hugo/layouts" + volumes="${volumes} --volume=${_repo}/scripts:/hugo/scripts" fi unset _project _repo done @@ -540,7 +651,7 @@ POSIX_HERESTRING fi fi - debg "DEBG: Mounting '${_src}' at container path '${_dst}'" + debg "Mounting '${_src}' at container path '${_dst}'" if [ -z "${volumes}" ]; then volumes="--volume=${_src}:${_dst}" @@ -569,7 +680,6 @@ case "${image}" in "${PODMAN}" run \ --init \ --interactive \ - --name "${DOCS_CONTAINER}" \ --platform linux/amd64 \ --rm \ --tty \ @@ -586,7 +696,6 @@ case "${image}" in "${PODMAN}" run \ --init \ --interactive \ - --name "${DOCS_CONTAINER}" \ --platform linux/amd64 \ --rm \ --tty \ @@ -601,6 +710,12 @@ case "${image}" in tempfile="$(mktemp -t make-docs.XXX)" cat <"${tempfile}" #!/usr/bin/env bash + +tc() { + set \${*,,} + echo \${*^} +} + for redirect in ${redirects}; do IFS='^' read -r path ver <<<"\${redirect}" echo -e "---\\nredirectURL: \"\${path/\/hugo\/content/}\"\\ntype: redirect\\nversioned: true\\n---\\n" > "\${path/\${ver}/_index.md}" @@ -609,8 +724,12 @@ done for x in "${url_src_dst_vers}"; do IFS='^' read -r _ _ dst _ <<<"\${x}" + title="\${dst%/*}" + title="\$(tc \${title##*/})" while [[ -n "\${dst}" ]]; do - touch "\${dst}/_index.md" + if [[ ! -f "\${dst}/_index.md" ]]; then + echo -e "---title: \${title}\\n---\\n\\n# \${title}\\n\\n{{< section >}}" > "\${dst}/_index.md" + fi dst="\${dst%/*}" done done @@ -630,33 +749,37 @@ ${PODMAN} run \ --env=HUGO_REFLINKSERRORLEVEL=${HUGO_REFLINKSERRORLEVEL} \ --init \ --interactive \ - --name=${DOCS_CONTAINER} \ - --platform=linux/amd64 \ --publish=${DOCS_HOST_PORT}:3002 \ --publish=3003:3003 \ --rm \ --tty \ ${volumes} \ - ${DOCS_IMAGE} \ - /entrypoint + ${DOCS_IMAGE} EOF - await_build http://localhost:3003 & - if [ -n "${DEBUG}" ]; then - ${cmd} + if [ -n "${ENTER}" ]; then + ${cmd} /bin/bash + elif [ -n "${DEBUG}" ]; then + await_build http://localhost:3003 & + + debg "${cmd} /entrypoint" + ${cmd} /entrypoint else - ${cmd} 2>&1| sed \ - -e '/Web Server is available at http:\/\/localhost:3003\/ (bind address 0.0.0.0)/ d' \ - -e '/^hugo server/ d' \ - -e '/fatal: not a git repository (or any parent up to mount point \/)/ d' \ - -e '/Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set)./ d' \ - -e "/Makefile:[0-9]*: warning: overriding recipe for target 'docs'/ d" \ - -e "/docs.mk:[0-9]*: warning: ignoring old recipe for target 'docs'/ d" \ - -e '/\/usr\/bin\/make -j 2 proxy hserver-docs HUGO_PORT=3003/ d' \ - -e '/website-proxy/ d' \ - -e '/rm -rf dist*/ d' \ - -e '/Press Ctrl+C to stop/ d' \ - -e '/make/ d' || echo + await_build http://localhost:3003 & + + ${cmd} /entrypoint 2>&1\ + | sed -u \ + -e '/Web Server is available at http:\/\/localhost:3003\/ (bind address 0.0.0.0)/ d' \ + -e '/^hugo server/ d' \ + -e '/fatal: not a git repository (or any parent up to mount point \/)/ d' \ + -e '/Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set)./ d' \ + -e "/Makefile:[0-9]*: warning: overriding recipe for target 'docs'/ d" \ + -e "/docs.mk:[0-9]*: warning: ignoring old recipe for target 'docs'/ d" \ + -e '/\/usr\/bin\/make -j 2 proxy hserver-docs HUGO_PORT=3003/ d' \ + -e '/website-proxy/ d' \ + -e '/rm -rf dist*/ d' \ + -e '/Press Ctrl+C to stop/ d' \ + -e '/make/ d' fi ;; esac