diff --git a/.github/actions/setup-grafana-bench/action.yml b/.github/actions/setup-grafana-bench/action.yml index 624ad243b7b..b708d862e2e 100644 --- a/.github/actions/setup-grafana-bench/action.yml +++ b/.github/actions/setup-grafana-bench/action.yml @@ -36,9 +36,10 @@ runs: shell: bash env: GH_TOKEN: ${{ steps.generate_token.outputs.token }} + BRANCH: ${{ inputs.branch }} run: | git clone https://x-access-token:${GH_TOKEN}@github.com/grafana/grafana-bench.git ../grafana-bench cd ../grafana-bench - git switch ${{ inputs.branch }} + git switch "$BRANCH" go install . diff --git a/.github/actions/test-coverage-processor/action.yml b/.github/actions/test-coverage-processor/action.yml index 7560031e846..c22fd0ccb19 100644 --- a/.github/actions/test-coverage-processor/action.yml +++ b/.github/actions/test-coverage-processor/action.yml @@ -28,11 +28,13 @@ runs: steps: - name: Process Go coverage output shell: bash + env: + COVERAGE_FILE: ${{ inputs.coverage-file }} run: | # Ensure valid coverage file even if empty - if [ ! -s ${{ inputs.coverage-file }} ]; then + if [ ! -s "$COVERAGE_FILE" ]; then echo "Coverage file is empty, creating a minimal valid file" - echo "mode: set" > ${{ inputs.coverage-file }} + echo "mode: set" > "$COVERAGE_FILE" fi - name: Report coverage to CodeCov diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml index f733b2f244d..942ea69d5e3 100644 --- a/.github/workflows/commands.yml +++ b/.github/workflows/commands.yml @@ -52,7 +52,7 @@ jobs: private_key: ${{ env.GH_APP_PEM }} - name: Checkout Actions - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "grafana/grafana-github-actions" path: ./actions diff --git a/.github/workflows/dashboards-issue-add-label.yml b/.github/workflows/dashboards-issue-add-label.yml index 95abce4355b..c3157a05afb 100644 --- a/.github/workflows/dashboards-issue-add-label.yml +++ b/.github/workflows/dashboards-issue-add-label.yml @@ -38,11 +38,13 @@ jobs: - name: Check if issue is in target project env: GH_TOKEN: ${{ steps.generate_token.outputs.token }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + TARGET_PROJECT: ${{ env.TARGET_PROJECT }} run: | gh api graphql -f query=' query($org: String!, $repo: String!) { repository(name: $repo, owner: $org) { - issue (number: ${{ github.event.issue.number }}) { + issue (number: $ISSUE_NUMBER) { id projectItems(first:20) { nodes { @@ -55,12 +57,14 @@ jobs: } }' -f org=$ORGANIZATION -f repo=$REPO > projects_data.json - echo 'IN_TARGET_PROJ='$(jq '.data.repository.issue.projectItems.nodes[] | select(.project.number==${{ env.TARGET_PROJECT }}) | .project != null' projects_data.json) >> $GITHUB_ENV + echo 'IN_TARGET_PROJ='$(jq '.data.repository.issue.projectItems.nodes[] | select(.project.number=='"$TARGET_PROJECT"') | .project != null' projects_data.json) >> $GITHUB_ENV echo 'ITEM_ID='$(jq '.data.repository.issue.id' projects_data.json) >> $GITHUB_ENV - name: Set up label array if: env.IN_TARGET_PROJ + env: + LABEL_IDS: ${{ env.LABEL_IDS }} run: | - IFS=',' read -ra LABEL_IDs <<< "${{ env.LABEL_IDs }}" + IFS=',' read -ra LABEL_IDs <<< "$LABEL_IDS" for item in "${LABEL_IDs[@]}"; do echo "Item: $item" done @@ -68,6 +72,7 @@ jobs: if: env.IN_TARGET_PROJ env: GH_TOKEN: ${{ steps.generate_token.outputs.token }} + LABEL_IDS: ${{ env.LABEL_IDS }} run: | gh api graphql -f query=' mutation ($labelableId: ID!, $labelIds: [ID!]!) { @@ -76,4 +81,4 @@ jobs: ) { clientMutationId } - }' -f labelableId=$ITEM_ID -f labelIds=${{ env.LABEL_IDs }} + }' -f labelableId=$ITEM_ID -f labelIds=$LABEL_IDS diff --git a/.github/workflows/issue-opened.yml b/.github/workflows/issue-opened.yml index 4de94070072..b478f9647e7 100644 --- a/.github/workflows/issue-opened.yml +++ b/.github/workflows/issue-opened.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Checkout Actions - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "grafana/grafana-github-actions" path: ./actions @@ -83,7 +83,7 @@ jobs: private_key: ${{ env.GH_APP_PEM }} - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Send issue to the auto triager action id: auto_triage @@ -99,7 +99,7 @@ jobs: - name: "Send Slack notification" if: ${{ steps.auto_triage.outputs.triage_labels != '' }} - uses: slackapi/slack-github-action@v1.27.0 + uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0 with: payload: > { diff --git a/.github/workflows/metrics-collector.yml b/.github/workflows/metrics-collector.yml index 2e22a830a88..9238b034433 100644 --- a/.github/workflows/metrics-collector.yml +++ b/.github/workflows/metrics-collector.yml @@ -15,6 +15,9 @@ on: issues: types: [opened, closed] +permissions: + contents: read + jobs: config: runs-on: "ubuntu-latest" @@ -35,7 +38,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Actions - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "grafana/grafana-github-actions" path: ./actions diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index ae2b0898f73..b3bd63fa0d8 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -31,7 +31,7 @@ jobs: if: github.event.pull_request.draft == false steps: - name: Checkout Actions - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "grafana/grafana-github-actions" path: ./actions diff --git a/.github/workflows/pr-commands.yml b/.github/workflows/pr-commands.yml index 51838dc7ae7..f392752ee8e 100644 --- a/.github/workflows/pr-commands.yml +++ b/.github/workflows/pr-commands.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Actions - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "grafana/grafana-github-actions" path: ./actions diff --git a/.github/workflows/pr-patch-check-event.yml b/.github/workflows/pr-patch-check-event.yml index e1389fdac6a..34f8fa1313c 100644 --- a/.github/workflows/pr-patch-check-event.yml +++ b/.github/workflows/pr-patch-check-event.yml @@ -17,6 +17,9 @@ on: # target branch onto the source branch, to verify compatibility before merging. jobs: dispatch-job: + permissions: + contents: read + actions: write env: HEAD_REF: ${{ github.head_ref }} BASE_REF: ${{ github.base_ref }} diff --git a/.github/workflows/run-dashboard-search-e2e.yml b/.github/workflows/run-dashboard-search-e2e.yml index 9eb9930cdd4..5766735a233 100644 --- a/.github/workflows/run-dashboard-search-e2e.yml +++ b/.github/workflows/run-dashboard-search-e2e.yml @@ -107,10 +107,14 @@ jobs: key: ${{ runner.os }}-grafana-${{ hashFiles('go.mod', 'package-lock.json', 'Makefile', 'pkg/storage/**/*.go', 'public/app/features/search/**/*.ts', 'public/app/features/search/**/*.tsx') }} - name: Set the step name id: set_file_name + env: + INI_NAME: ${{ matrix.ini_file }} run: | - FILE_NAME=$(basename "${{ matrix.ini_file }}" .ini) - echo "FILE_NAME=$FILE_NAME" >> $GITHUB_ENV - - name: Run tests for ${{ env.FILE_NAME }} + FILE_NAME=$(basename "$env.INI_NAME" .ini) + echo "FILE_NAME=$FILE_NAME" >> $GITHUB_OUTPUT + - name: Run tests for ${{ steps.set_file_name.outputs.FILE_NAME }} + env: + INI_NAME: ${{ matrix.ini_file }} run: | - cp -rf ${{ matrix.ini_file }} ${{ github.workspace }}/scripts/grafana-server/custom.ini + cp -rf $INI_NAME ${{ github.workspace }}/scripts/grafana-server/custom.ini yarn e2e:dashboards-search || echo "Test failed but marking as success since unified search is behind a feature flag and should not block PRs" diff --git a/.github/workflows/skye-add-to-project.yml b/.github/workflows/skye-add-to-project.yml index 5e8ccc8e557..6788db2d95e 100644 --- a/.github/workflows/skye-add-to-project.yml +++ b/.github/workflows/skye-add-to-project.yml @@ -50,10 +50,12 @@ jobs: # Check if the user is in the list from the secret - name: Check if user is allowed id: check_user + env: + ALLOWED_USERS: ${{ env.ALLOWED_USERS }} + USERNAME: ${{ github.event.sender.login }} run: | # Convert the comma-separated list to an array - IFS=',' read -ra ALLOWED_USERS <<< "${{ env.ALLOWED_USERS }}" - USERNAME="${{ github.event.sender.login }}" + IFS=',' read -ra ALLOWED_USERS <<< "$ALLOWED_USERS" # Check if user is in the allowed list for allowed_user in "${ALLOWED_USERS[@]}"; do diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 746e670dc78..b9ddc051469 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -344,11 +344,19 @@ func processHits(dec *json.Decoder, sr *SearchResponse) error { return err } - if tok == "hits" { + switch tok { + case "hits": if err := streamHitsArray(dec, sr); err != nil { return err } - } else { + case "total": + var total *SearchResponseHitsTotal + err := dec.Decode(&total) + if err != nil { + return err + } + sr.Hits.Total = total + default: // ignore these fields as they are not used in the current implementation err := skipUnknownField(dec) if err != nil { diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index e18fad67f35..c8648f0bdeb 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -44,9 +44,15 @@ func (r *SearchRequest) MarshalJSON() ([]byte, error) { return json.Marshal(root) } +type SearchResponseHitsTotal struct { + Value int `json:"value"` + Relation string `json:"relation"` +} + // SearchResponseHits represents search response hits type SearchResponseHits struct { - Hits []map[string]interface{} + Hits []map[string]interface{} + Total *SearchResponseHitsTotal `json:"total"` } // SearchResponse represents a search response diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 541006fedc2..b38b2c1cc07 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -208,7 +208,12 @@ func processLogsResponse(res *es.SearchResponse, target *Query, configuredFields frames := data.Frames{} frame := data.NewFrame("", fields...) setPreferredVisType(frame, data.VisTypeLogs) - setLogsCustomMeta(frame, searchWords, stringToIntWithDefaultValue(target.Metrics[0].Settings.Get("limit").MustString(), defaultSize)) + + var total int + if res.Hits.Total != nil { + total = res.Hits.Total.Value + } + setLogsCustomMeta(frame, searchWords, stringToIntWithDefaultValue(target.Metrics[0].Settings.Get("limit").MustString(), defaultSize), total) frames = append(frames, frame) queryRes.Frames = frames @@ -1192,7 +1197,7 @@ func setPreferredVisType(frame *data.Frame, visType data.VisType) { frame.Meta.PreferredVisualization = visType } -func setLogsCustomMeta(frame *data.Frame, searchWords map[string]bool, limit int) { +func setLogsCustomMeta(frame *data.Frame, searchWords map[string]bool, limit int, total int) { i := 0 searchWordsList := make([]string, len(searchWords)) for searchWord := range searchWords { @@ -1212,6 +1217,7 @@ func setLogsCustomMeta(frame *data.Frame, searchWords map[string]bool, limit int frame.Meta.Custom = map[string]interface{}{ "searchWords": searchWordsList, "limit": limit, + "total": total, } } diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go index a03c5068ed8..85239537bad 100644 --- a/pkg/tsdb/elasticsearch/response_parser_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -46,6 +46,7 @@ func TestProcessLogsResponse(t *testing.T) { { "aggregations": {}, "hits": { + "total": { "value": 2 }, "hits": [ { "_id": "fdsfs", @@ -107,7 +108,7 @@ func TestProcessLogsResponse(t *testing.T) { logsFrame := frames[0] meta := logsFrame.Meta - require.Equal(t, map[string]any{"searchWords": []string{"hello", "message"}, "limit": 500}, meta.Custom) + require.Equal(t, map[string]any{"searchWords": []string{"hello", "message"}, "limit": 500, "total": 2}, meta.Custom) require.Equal(t, data.VisTypeLogs, string(meta.PreferredVisualization)) logsFieldMap := make(map[string]*data.Field) @@ -431,6 +432,7 @@ func TestProcessLogsResponse(t *testing.T) { require.Equal(t, map[string]any{ "searchWords": []string{"hello", "message"}, "limit": 500, + "total": 109, }, customMeta) }) } @@ -703,7 +705,7 @@ func TestProcessRawDocumentResponse(t *testing.T) { "responses": [ { "hits": { - "total": 100, + "total": { "value": 100 }, "hits": [ { "_id": "1", @@ -3239,7 +3241,7 @@ func TestParseResponse(t *testing.T) { }, { "hits": { - "total": 2, + "total": { "value": 2 }, "hits": [ { "_id": "5", diff --git a/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc b/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc index 105279a888e..7d5a673da01 100644 --- a/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc +++ b/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc @@ -10,7 +10,8 @@ // "searchWords": [ // "hello", // "message" -// ] +// ], +// "total": 81 // }, // "preferredVisualisationType": "logs" // } @@ -45,7 +46,8 @@ "searchWords": [ "hello", "message" - ] + ], + "total": 81 }, "preferredVisualisationType": "logs" }, diff --git a/public/app/features/dashboard-scene/edit-pane/DashboardOutline.tsx b/public/app/features/dashboard-scene/edit-pane/DashboardOutline.tsx index e231c286720..58076081f00 100644 --- a/public/app/features/dashboard-scene/edit-pane/DashboardOutline.tsx +++ b/public/app/features/dashboard-scene/edit-pane/DashboardOutline.tsx @@ -4,7 +4,7 @@ import React, { useEffect, useMemo, useState } from 'react'; import { GrafanaTheme2 } from '@grafana/data'; import { SceneObject } from '@grafana/scenes'; -import { Box, Icon, Stack, Text, useElementSelection, useStyles2 } from '@grafana/ui'; +import { Box, Icon, Text, useElementSelection, useStyles2, useTheme2 } from '@grafana/ui'; import { t, Trans } from 'app/core/internationalization'; import { DashboardGridItem } from '../scene/layout-default/DashboardGridItem'; @@ -24,7 +24,7 @@ export function DashboardOutline({ editPane }: Props) { const dashboard = getDashboardSceneFor(editPane); return ( - + ); @@ -40,6 +40,7 @@ function DashboardOutlineNode({ depth: number; }) { const [isCollapsed, setIsCollapsed] = useState(depth > 0); + const theme = useTheme2(); const { key } = sceneObject.useState(); const styles = useStyles2(getStyles); const { isSelected, onSelect } = useElementSelection(key); @@ -53,7 +54,7 @@ function DashboardOutlineNode({ const elementCollapsed = editableElement.getCollapsedState?.(); const outlineRename = useOutlineRename(editableElement); - const onNameClicked = (evt: React.PointerEvent) => { + const onNodeClicked = (evt: React.PointerEvent) => { // Only select via clicking outline never deselect if (!isSelected) { onSelect?.(evt); @@ -62,7 +63,8 @@ function DashboardOutlineNode({ editableElement.scrollIntoView?.(); }; - const onToggleCollapse = () => { + const onToggleCollapse = (evt: React.MouseEvent) => { + evt.stopPropagation(); setIsCollapsed(!isCollapsed); // Sync expanded state with canvas element @@ -80,16 +82,19 @@ function DashboardOutlineNode({ return ( <> - +
{elementInfo.isContainer && ( - )} - +
{elementInfo.isContainer && !isCollapsed && ( -
+
+
{children.length > 0 ? ( children.map((child) => ( span': { whiteSpace: 'nowrap', overflow: 'hidden', textOverflow: 'ellipsis', }, }), - nodeButtonSelected: css({ - color: theme.colors.text.primary, - outline: `1px dashed ${theme.colors.primary.border} !important`, - outlineOffset: '0px', - '&:hover': { - outline: `1px dashed ${theme.colors.primary.border}`, - }, - }), hiddenIcon: css({ color: theme.colors.text.secondary, marginLeft: theme.spacing(1), }), - nodeButtonClone: css({ + nodeNameClone: css({ color: theme.colors.text.secondary, cursor: 'not-allowed', }), outlineInput: css({ - border: `1px solid ${theme.colors.primary.border}`, + border: `1px solid ${theme.components.input.borderColor}`, height: theme.spacing(3), + borderRadius: theme.shape.radius.default, '&:focus': { outline: 'none', boxShadow: 'none', }, }), + nodeChildren: css({ + display: 'flex', + flexDirection: 'column', + position: 'relative', + }), + nodeChildrenLine: css({ + position: 'absolute', + width: '1px', + height: '100%', + left: '7px', + zIndex: 1, + backgroundColor: theme.colors.border.weak, + }), }; } diff --git a/public/app/features/dashboard-scene/scene/layout-rows/RowItemEditor.tsx b/public/app/features/dashboard-scene/scene/layout-rows/RowItemEditor.tsx index 9f7c0426682..f67256a31c8 100644 --- a/public/app/features/dashboard-scene/scene/layout-rows/RowItemEditor.tsx +++ b/public/app/features/dashboard-scene/scene/layout-rows/RowItemEditor.tsx @@ -24,7 +24,8 @@ export function useEditOptions(model: RowItem, isNewElement: boolean): OptionsPa new OptionsPaneCategoryDescriptor({ title: '', id: 'row-options' }) .addItem( new OptionsPaneItemDescriptor({ - title: t('dashboard.rows-layout.row-options.row.title', 'Title'), + title: '', + skipField: true, render: () => , }) ) @@ -85,6 +86,7 @@ function RowTitleInput({ row, isNewElement }: { row: RowItem; isNewElement: bool return ( { }); }); + it('given one series with total as custom meta property should return correct total', () => { + const series: DataFrame[] = [ + createDataFrame({ + fields: [], + meta: { + custom: { + total: 9999, + }, + }, + }), + ]; + const logsModel = dataFrameToLogsModel(series, 1); + expect(logsModel.meta![0]).toMatchObject({ + label: TOTAL_LABEL, + value: 9999, + kind: LogsMetaKind.Number, + }); + }); + + it('given multiple series with total as custom meta property should return correct total', () => { + const series: DataFrame[] = [ + createDataFrame({ + fields: [], + meta: { + custom: { + total: 4, + }, + }, + }), + createDataFrame({ + fields: [], + meta: { + custom: { + total: 5, + }, + }, + }), + ]; + const logsModel = dataFrameToLogsModel(series, 1); + expect(logsModel.meta![0]).toMatchObject({ + label: TOTAL_LABEL, + value: 9, + kind: LogsMetaKind.Number, + }); + }); + it('should return the expected meta when the line limit is reached', () => { const series: DataFrame[] = getTestDataFrame(); series[0].meta = { diff --git a/public/app/features/logs/logsModel.ts b/public/app/features/logs/logsModel.ts index 7e75684d423..9c1c266cea6 100644 --- a/public/app/features/logs/logsModel.ts +++ b/public/app/features/logs/logsModel.ts @@ -51,6 +51,7 @@ import { createLogRowsMap, getLogLevel, getLogLevelFromKey, sortInAscendingOrder export const LIMIT_LABEL = 'Line limit'; export const COMMON_LABELS = 'Common labels'; +export const TOTAL_LABEL = 'Total lines'; export const LogLevelColor = { [LogLevel.critical]: colors[7], @@ -492,6 +493,15 @@ export function logSeriesToLogsModel( }); } + const totalValue = logSeries.reduce((acc, series) => (acc += series.meta?.custom?.total), 0); + if (totalValue > 0) { + meta.push({ + label: TOTAL_LABEL, + value: totalValue, + kind: LogsMetaKind.Number, + }); + } + let totalBytes = 0; const queriesVisited: { [refId: string]: boolean } = {}; // To add just 1 error message