Transformations: Add 'JSON' field type to ConvertFieldTypeTransformer (#42624)

* Add 'JSON' field type to ConvertFieldTypeTransformer

I've been playing around with #41994 and found that it requires fields
to contain array values, which can't be sent from a backend plugin. This
PR adds the ability for the ConvertFieldTypeTransformer to parse
JSON values and store the result in the transformed field.
The main use case for this right now is so that a field
containing a JSONified array can be transformed into a field
containing an actual array, which can in
turn be used for the table charts in #41994.

Supersedes #42521.

* Add second option to complex field conversion to increase flexibility

This avoids falsely equating 'JSON' with FieldType.other, and instead
allows multiple parsers to be used if the 'Complex' type is selected.

Currently only JSON parsing is implemented, but others could be
supported easily in future.

* Revert "Add second option to complex field conversion to increase flexibility"

This reverts commit 6314ce35eb.

* Improve test for object parsing of complex field transformer

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

docs/sources/panels/transformations/_index.md

@@ -29,6 +29,7 @@ Before you can configure and apply transformations:
 - [Add field from calculation]({{< relref "./types-options.md#add-field-from-calculation" >}})
 - [Concatenate fields]({{< relref "./types-options.md#concatenate-fields" >}})
 - [Config from query results]({{< relref "./config-from-query.md" >}})
+- [Convert field type]({{< relref "./convert-field-type.md" >}})
 - [Filter data by name]({{< relref "./types-options.md#filter-data-by-name" >}})
 - [Filter data by query]({{< relref "./types-options.md#filter-data-by-query" >}})
 - [Filter data by value]({{< relref "./types-options.md#filter-data-by-value" >}})

docs/sources/panels/transformations/types-options.md

@@ -349,24 +349,28 @@ This transformation changes the field type of the specified field.
   - **Time -** attempts to parse the values as time
     - Will show an option to specify a DateFormat for the input field like yyyy-mm-dd or DD MM YYYY hh:mm:ss
   - **Boolean -** will make the values booleans
+  - **JSON -** attempts to parse the values as JSON, potentially resulting in complex objects or arrays
 
-For example the following query could be modified by selecting the time field, as Time, and Input format as YYYY.
+For example the following query could be modified by:
 
-| Time       | Mark  | Value |
-| ---------- | ----- | ----- |
-| 2017-07-01 | above | 25    |
-| 2018-08-02 | below | 22    |
-| 2019-09-02 | below | 29    |
-| 2020-10-04 | above | 22    |
+- selecting the time field as Time, and Input format as YYYY
+- selecting the JSON field as JSON
+
+| Time       | Mark  | Value | JSON     |
+| ---------- | ----- | ----- | -------- |
+| 2017-07-01 | above | 25    | "[0, 1]" |
+| 2018-08-02 | below | 22    | "[2, 3]" |
+| 2019-09-02 | below | 29    | "[4, 5]" |
+| 2020-10-04 | above | 22    | "[6, 7]" |
 
 The result:
 
-| Time                | Mark  | Value |
-| ------------------- | ----- | ----- |
-| 2017-01-01 00:00:00 | above | 25    |
-| 2018-01-01 00:00:00 | below | 22    |
-| 2019-01-01 00:00:00 | below | 29    |
-| 2020-01-01 00:00:00 | above | 22    |
+| Time                | Mark  | Value | JSON   |
+| ------------------- | ----- | ----- | ------ |
+| 2017-01-01 00:00:00 | above | 25    | [0, 1] |
+| 2018-01-01 00:00:00 | below | 22    | [2, 3] |
+| 2019-01-01 00:00:00 | below | 29    | [4, 5] |
+| 2020-01-01 00:00:00 | above | 22    | [6, 7] |
 
 ## Series to rows
 

packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts

@@ -183,6 +183,87 @@ describe('field convert types transformer', () => {
     ]);
   });
 
+  it('will convert field to complex objects', () => {
+    const options = {
+      conversions: [
+        { targetField: 'numbers', destinationType: FieldType.other },
+        { targetField: 'objects', destinationType: FieldType.other },
+        { targetField: 'arrays', destinationType: FieldType.other },
+        { targetField: 'invalids', destinationType: FieldType.other },
+        { targetField: 'mixed', destinationType: FieldType.other },
+      ],
+    };
+
+    const comboTypes = toDataFrame({
+      fields: [
+        {
+          name: 'numbers',
+          type: FieldType.number,
+          values: [-1, 1, null],
+        },
+        {
+          name: 'objects',
+          type: FieldType.string,
+          values: [
+            '{ "neg": -100, "zero": 0, "pos": 1, "null": null, "array": [0, 1, 2], "nested": { "number": 1 } }',
+            '{ "string": "abcd" }',
+            '{}',
+          ],
+        },
+        {
+          name: 'arrays',
+          type: FieldType.string,
+          values: ['[true]', '[99]', '["2021-08-02 00:00:00.000"]'],
+        },
+        {
+          name: 'invalids',
+          type: FieldType.string,
+          values: ['abcd', '{ invalidJson }', '[unclosed array'],
+        },
+        {
+          name: 'mixed',
+          type: FieldType.string,
+          values: [
+            '{ "neg": -100, "zero": 0, "pos": 1, "null": null, "array": [0, 1, 2], "nested": { "number": 1 } }',
+            '["a string", 1234, {"a complex": "object"}]',
+            '["this is invalid JSON]',
+          ],
+        },
+      ],
+    });
+
+    const complex = convertFieldTypes(options, [comboTypes]);
+    expect(
+      complex[0].fields.map((f) => ({
+        type: f.type,
+        values: f.values.toArray(),
+      }))
+    ).toEqual([
+      {
+        type: FieldType.other,
+        values: [-1, 1, null],
+      },
+      {
+        type: FieldType.other,
+        values: [
+          { neg: -100, zero: 0, pos: 1, null: null, array: [0, 1, 2], nested: { number: 1 } },
+          { string: 'abcd' },
+          {},
+        ],
+      },
+      { type: FieldType.other, values: [[true], [99], ['2021-08-02 00:00:00.000']] },
+      { type: FieldType.other, values: [null, null, null] },
+      {
+        type: FieldType.other,
+        values: [
+          { neg: -100, zero: 0, pos: 1, null: null, array: [0, 1, 2], nested: { number: 1 } },
+          ['a string', 1234, { 'a complex': 'object' }],
+          null,
+        ],
+      },
+    ]);
+  });
+
   it('will convert field to strings', () => {
     const options = {
       conversions: [{ targetField: 'numbers', destinationType: FieldType.string }],

packages/grafana-data/src/transformations/transformers/convertFieldType.ts

@@ -99,6 +99,8 @@ export function convertFieldType(field: Field, opts: ConvertFieldTypeOptions): F
       return fieldToStringField(field);
     case FieldType.boolean:
       return fieldToBooleanField(field);
+    case FieldType.other:
+      return fieldToComplexField(field);
     default:
       return field;
   }
@@ -178,6 +180,24 @@ function fieldToStringField(field: Field): Field {
   };
 }
 
+function fieldToComplexField(field: Field): Field {
+  const complexValues = field.values.toArray().slice();
+
+  for (let s = 0; s < complexValues.length; s++) {
+    try {
+      complexValues[s] = JSON.parse(complexValues[s]);
+    } catch {
+      complexValues[s] = null;
+    }
+  }
+
+  return {
+    ...field,
+    type: FieldType.other,
+    values: new ArrayVector(complexValues),
+  };
+}
+
 /**
  * Checks the first value. Assumes any number should be time fieldtype. Otherwise attempts to make the fieldtype time.
  * @param field - field to ensure is a time fieldtype

public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx

@@ -29,6 +29,7 @@ export const ConvertFieldTypeTransformerEditor: React.FC<TransformerUIProps<Conv
     { value: FieldType.string, label: 'String' },
     { value: FieldType.time, label: 'Time' },
     { value: FieldType.boolean, label: 'Boolean' },
+    { value: FieldType.other, label: 'JSON' },
   ];
 
   const onSelectField = useCallback(