Update IAM Folder Reconciler local dev environment to include tools for profiling the operator. (#111641)

Update local dev setup to allow for profiling of the operator

apps/iam/Tiltfile

@@ -3,14 +3,16 @@
 # https://docs.tilt.dev/api.html#api.version_settings
 version_settings(constraint='>=0.22.2')
 
-custom_build(
-    'grafana-iam-operator',
-    command='docker build -t $EXPECTED_REF -f cmd/operator/Dockerfile .',
-    deps=[
-      'cmd/operator',
-      'pkg',
-    ],
-    disable_push=True,
-)
-
 k8s_yaml([filename for filename in listdir('local/yamls') if filename.lower().endswith(('.yaml', '.yml'))])
+
+# Port forward Grafana to localhost:3000
+k8s_resource('grafana', port_forwards=['3000:3000'])
+
+# Port forward Jaeger UI to localhost:16686
+k8s_resource('jaeger-agent', port_forwards=['16686:16686'])
+
+# Port forward Pyroscope UI to localhost:4040
+k8s_resource('pyroscope', port_forwards=['4040:4040'])
+
+# Port forward Alloy UI to localhost:12345
+k8s_resource('alloy', port_forwards=['12345:12345'])

apps/iam/local/yamls/alloy.yaml

@@ -0,0 +1,122 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alloy-config
+  namespace: default
+data:
+  config.alloy: |
+    // Pyroscope configuration to receive profiles
+    pyroscope.write "default" {
+      endpoint {
+        url = "http://pyroscope.default.svc.cluster.local:4040"
+      }
+    }
+
+    // Scrape CPU profiles from the IAM operator
+    pyroscope.scrape "iam_operator" {
+      targets = [
+        {
+          "__address__" = "iam-folder-reconciler.default.svc.cluster.local:6060",
+          "service_name" = "iam-folder-reconciler",
+        },
+      ]
+      forward_to = [pyroscope.write.default.receiver]
+
+      job_name = "iam-operator"
+      scrape_interval = "30s"
+      scrape_timeout = "25s"
+
+      profiling_config {
+        profile.process_cpu {
+          enabled = true
+          path = "/debug/pprof/profile"
+          delta = false
+        }
+
+        profile.godeltaprof_memory {
+          enabled = true
+          path = "/debug/pprof/delta_heap"
+        }
+
+        profile.memory {
+          enabled = true
+          path = "/debug/pprof/heap"
+          delta = false
+        }
+
+        profile.godeltaprof_mutex {
+          enabled = true
+          path = "/debug/pprof/delta_mutex"
+        }
+
+        profile.godeltaprof_block {
+          enabled = true
+          path = "/debug/pprof/delta_block"
+        }
+
+        profile.goroutine {
+          enabled = true
+          path = "/debug/pprof/goroutine"
+          delta = false
+        }
+      }
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: alloy
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: alloy
+  template:
+    metadata:
+      labels:
+        name: alloy
+    spec:
+      containers:
+        - name: alloy
+          image: grafana/alloy:v1.10.0
+          args:
+            - run
+            - /etc/alloy/config.alloy
+            - --storage.path=/var/lib/alloy/data
+            - --server.http.listen-addr=0.0.0.0:12345
+            - --stability.level=experimental
+          ports:
+            - containerPort: 12345
+              name: http
+          volumeMounts:
+            - name: config
+              mountPath: /etc/alloy
+            - name: storage
+              mountPath: /var/lib/alloy/data
+          resources:
+            requests:
+              memory: "128Mi"
+              cpu: "100m"
+            limits:
+              memory: "256Mi"
+              cpu: "200m"
+      volumes:
+        - name: config
+          configMap:
+            name: alloy-config
+        - name: storage
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: alloy
+  namespace: default
+spec:
+  selector:
+    name: alloy
+  ports:
+    - name: http
+      port: 12345
+      targetPort: 12345

apps/iam/local/yamls/grafana.yaml

@@ -0,0 +1,197 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-datasources
+  namespace: default
+data:
+  datasources.yaml: |
+    apiVersion: 1
+    datasources:
+      - name: Jaeger
+        type: jaeger
+        uid: local-jaeger
+        access: proxy
+        url: http://jaeger-agent.jaeger.svc.cluster.local:16686
+        editable: true
+        isDefault: true
+        jsonData:
+          tracesToLogsV2:
+            datasourceUid: 'local-loki'
+            customQuery: false
+            filterByTraceID: true
+            filterBySpanID: true
+          nodeGraph:
+            enabled: true
+          search:
+            hide: false
+          spanBar:
+            type: 'Duration'
+      - name: Pyroscope
+        type: grafana-pyroscope-datasource
+        uid: local-pyroscope
+        access: proxy
+        url: http://pyroscope.default.svc.cluster.local:4040
+        editable: true
+        isDefault: false
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-config
+  namespace: default
+data:
+  grafana.ini: |
+    app_mode = development
+
+    [paths]
+    data = /var/lib/grafana
+    logs = /var/log/grafana
+    plugins = /var/lib/grafana/plugins
+    provisioning = /etc/grafana/provisioning
+
+    [server]
+    http_port = 3000
+
+    [database]
+    type = sqlite3
+    path = grafana.db
+
+    [session]
+    provider = file
+    provider_config = sessions
+
+    [analytics]
+    reporting_enabled = false
+    check_for_updates = false
+
+    [security]
+    admin_user = admin
+    admin_password = admin
+    disable_gravatar = true
+
+    [snapshots]
+    external_enabled = false
+
+    [users]
+    allow_sign_up = false
+    allow_org_create = false
+    auto_assign_org = true
+    auto_assign_org_role = Viewer
+
+    [plugin.grafana-pyroscope-app]
+    app_enabled = true
+
+    [auth.anonymous]
+    enabled = true
+    org_name = Main Org.
+    org_role = Editor
+
+    [log]
+    mode = console
+    level = info
+
+    [explore]
+    enabled = true
+
+    [feature_toggles]
+    enable = traceqlStreaming,correlations,traceToMetrics
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana
+  namespace: default
+  labels:
+    app: grafana
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana
+  template:
+    metadata:
+      labels:
+        app: grafana
+    spec:
+      securityContext:
+        fsGroup: 472
+        supplementalGroups:
+          - 0
+      containers:
+        - name: grafana
+          image: grafana/grafana:latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 3000
+              name: http-grafana
+              protocol: TCP
+          env:
+            - name: GF_PATHS_CONFIG
+              value: /etc/grafana/grafana.ini
+            - name: GF_PATHS_PROVISIONING
+              value: /etc/grafana/provisioning
+          resources:
+            requests:
+              cpu: 250m
+              memory: 750Mi
+            limits:
+              cpu: 500m
+              memory: 1Gi
+          volumeMounts:
+            - mountPath: /var/lib/grafana
+              name: grafana-storage
+            - mountPath: /etc/grafana
+              name: grafana-config
+            - mountPath: /etc/grafana/provisioning/datasources
+              name: grafana-datasources
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /api/health
+              port: 3000
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 2
+          livenessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            httpGet:
+              path: /api/health
+              port: 3000
+            timeoutSeconds: 1
+      volumes:
+        - name: grafana-storage
+          emptyDir: {}
+        - name: grafana-config
+          configMap:
+            name: grafana-config
+        - name: grafana-datasources
+          configMap:
+            name: grafana-datasources
+            items:
+              - key: datasources.yaml
+                path: datasources.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+  namespace: default
+  labels:
+    app: grafana
+spec:
+  type: NodePort
+  ports:
+    - port: 3000
+      protocol: TCP
+      targetPort: http-grafana
+      nodePort: 30000
+      name: grafana-http
+  selector:
+    app: grafana
+---

apps/iam/local/yamls/jaeger.yaml

@@ -0,0 +1,101 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: jaeger
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: jaeger-agent
+  namespace: jaeger
+  labels:
+    app: jaeger-agent
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: jaeger-agent
+  template:
+    metadata:
+      labels:
+        app: jaeger-agent
+    spec:
+      containers:
+        - name: jaeger-agent
+          image: jaegertracing/all-in-one:latest
+          ports:
+            - containerPort: 6831
+              protocol: UDP
+              name: agent-udp
+            - containerPort: 16686
+              name: ui
+            - containerPort: 14268
+              name: collector
+            - containerPort: 4317
+              name: otlp-grpc
+            - containerPort: 4318
+              name: otlp-http
+            - containerPort: 5778
+              name: sampling
+          env:
+            - name: MEMORY_MAX_TRACES
+              value: "100000"
+            - name: SPAN_STORAGE_TYPE
+              value: "badger"
+            - name: COLLECTOR_OTLP_ENABLED
+              value: "true"
+            - name: LOG_LEVEL
+              value: "debug"
+          resources:
+            limits:
+              memory: "1Gi"
+              cpu: "500m"
+            requests:
+              memory: "512Mi"
+              cpu: "250m"
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 16686
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 16686
+            initialDelaySeconds: 30
+            periodSeconds: 30
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: jaeger-agent
+  namespace: jaeger
+  labels:
+    app: jaeger-agent
+spec:
+  selector:
+    app: jaeger-agent
+  ports:
+    - name: agent-udp
+      port: 6831
+      targetPort: 6831
+      protocol: UDP
+    - name: ui
+      port: 16686
+      targetPort: 16686
+    - name: collector
+      port: 14268
+      targetPort: 14268
+    - name: otlp-grpc
+      port: 4317
+      targetPort: 4317
+    - name: otlp-http
+      port: 4318
+      targetPort: 4318
+    - name: sampling
+      port: 5778
+      targetPort: 5778
+  type: ClusterIP
+---

apps/iam/local/yamls/operator.yaml

@@ -4,10 +4,43 @@ metadata:
   name: operator
   namespace: default
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: operator-config
+  namespace: default
+data:
+  operator.ini: |
+    app_mode = development
+    target = operator
+    ensure_default_org_and_user = false
+    skip_migrations = true
+
+
+    [grpc_client_authentication]
+    token_exchange_url = http://host.docker.internal:8080/v1/sign-access-token
+    token_namespace = *
+
+    [operator]
+    folder_app_url = https://host.docker.internal:6446
+    max_concurrent_workers = 20
+    tls_ca_file =
+    tls_insecure = true
+    zanzana_url = zanzana.default.svc.cluster.local:50051
+
+    [tracing.opentelemetry]
+    custom_attributes = namespace:grafana-iam
+    sampler_param = 1
+    sampler_type = remote
+    sampling_server_url = http://jaeger-agent.jaeger.svc.cluster.local.:5778/sampling
+
+    [tracing.opentelemetry.otlp]
+    address = jaeger-agent.jaeger.svc.cluster.local.:4317
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: iam-app-operator
+  name: iam-folder-reconciler
   namespace: default
 spec:
   minReadySeconds: 10
@@ -15,44 +48,63 @@ spec:
   revisionHistoryLimit: 10
   selector:
     matchLabels:
-      name: iam-app-operator
+      name: iam-folder-reconciler
   template:
     metadata:
       labels:
-        name: iam-app-operator
+        name: iam-folder-reconciler
     spec:
-      serviceAccount: operator
       containers:
-        - image: grafana-iam-operator
-          imagePullPolicy: IfNotPresent
-          name: iam-app-operator
-          command: ["/bin/sh"]
-          args:
-            - -c
-            - exec /usr/bin/operator
+        - command:
+            - grafana-server
+            - target
+            - --config=/etc/grafana-config/operator.ini
+            - --homepath=/usr/share/grafana
           env:
             - name: KUBE_FEATURE_WatchListClient
-              value: "false"
-            - name: ZANZANA_ADDR
-              value: zanzana.default.svc.cluster.local:50051
-            - name: FOLDER_APP_URL
-              value: https://host.docker.internal:6446
-            - name: FOLDER_APP_NAMESPACE
-              value: grafana-folder
-            - name: TOKEN_EXCHANGE_URL
-              value: http://host.docker.internal:8080/v1/sign-access-token
-            - name: AUTH_TOKEN
+              value: "true"
+            - name: GF_DEFAULT_TARGET
+              value: operator
+            - name: GF_OPERATOR_NAME
+              value: iam-folder-reconciler
+            - name: GF_DIAGNOSTICS_PROFILING_ENABLED
+              value: "true"
+            - name: GF_DIAGNOSTICS_PROFILING_ADDR
+              value: "0.0.0.0"
+            - name: GF_DIAGNOSTICS_PROFILING_PORT
+              value: "6060"
+            - name: GF_GRPC_CLIENT_AUTHENTICATION_TOKEN
               valueFrom:
                 secretKeyRef:
-                  name: iam-app-operator-secrets
-                  key: auth-token
-            - name: ZANZANA_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: iam-app-operator-secrets
-                  key: zanzana-token
-            - name: FOLDER_RECONCILER_NAMESPACE
-              valueFrom:
-                secretKeyRef:
-                  name: iam-app-operator-secrets
-                  key: folder-reconciler-namespace
+                  name: iam-operator-secrets
+                  key: grpc_auth_token
+          image: grafana/grafana-dev:12.3.0-17863745596
+          imagePullPolicy: IfNotPresent
+          name: iam-folder-reconciler
+          volumeMounts:
+            - name: operator-config
+              mountPath: /etc/grafana-config
+      serviceAccount: operator
+      volumes:
+        - configMap:
+            name: operator-config
+          name: operator-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: iam-folder-reconciler
+  namespace: default
+  labels:
+    name: iam-folder-reconciler
+spec:
+  ports:
+    - name: iam-folder-reconciler-http-metrics
+      port: 8080
+      targetPort: 8080
+    - name: iam-folder-reconciler-pprof
+      port: 6060
+      targetPort: 6060
+  selector:
+    name: iam-folder-reconciler
+---

apps/iam/local/yamls/pyroscope.yaml

@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pyroscope
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: pyroscope
+  template:
+    metadata:
+      labels:
+        name: pyroscope
+    spec:
+      containers:
+        - name: pyroscope
+          image: grafana/pyroscope:1.14.0
+          ports:
+            - containerPort: 4040
+              name: http
+          env:
+            - name: PYROSCOPE_LOG_LEVEL
+              value: "info"
+          resources:
+            requests:
+              memory: "128Mi"
+              cpu: "100m"
+            limits:
+              memory: "512Mi"
+              cpu: "500m"
+      volumes: []
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pyroscope
+  namespace: default
+spec:
+  selector:
+    name: pyroscope
+  ports:
+    - name: http
+      port: 4040
+      targetPort: 4040

apps/iam/local/yamls/zanzana.yaml

@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
         - name: postgres
-          image: postgres:15.7
+          image: postgres:15
           env:
             - name: POSTGRES_USER
               value: grafana
@@ -32,13 +32,6 @@ spec:
           volumeMounts:
             - name: postgres-storage
               mountPath: /var/lib/postgresql/data
-          resources:
-            requests:
-              cpu: 100m
-              memory: 256Mi
-            limits:
-              cpu: 500m
-              memory: 512Mi
           readinessProbe:
             exec:
               command:
@@ -165,7 +158,7 @@ spec:
           env:
             - name: GF_PATHS_CONFIG
               value: /etc/grafana-config/grafana.ini
-          image: grafana/grafana-dev:12.2.0-257970
+          image: grafana/grafana-dev:12.3.0-17863745596
           imagePullPolicy: IfNotPresent
           name: zanzana
           ports: