root
/
grafana
mirror of https://github.com/grafana/grafana.git
1
0
Fork 0
Code Issues Actions 236 Packages Projects Releases Wiki Activity

codegen: Refactor core jennies for reusability, add version-picking metajennies (#58995) 

* Add each-major jenny, refactor TS jenny for it

* Introduce LatestJenny, refactor GoTypesJenny for it
This commit is contained in:
sam boyer 2022-11-22 09:00:29 -05:00 committed by GitHub
parent 4eed56193f
commit 42baad837a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 206 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
kinds/gen.go
View File

@ -21,8 +21,6 @@ import (
"github.com/grafana/grafana/pkg/kindsys"
)
const sep = string(filepath.Separator)
func main() {
if len(os.Args) > 1 {
fmt.Fprintf(os.Stderr, "plugin thema code generator does not currently accept any arguments\n, got %q", os.Args)
@ -37,16 +35,11 @@ func main() {
// All the jennies that comprise the core kinds generator pipeline
coreKindsGen.Append(
codegen.GoTypesJenny(kindsys.GoCoreKindParentPath, nil),
codegen.LatestJenny(kindsys.GoCoreKindParentPath, codegen.GoTypesJenny{}),
codegen.CoreStructuredKindJenny(kindsys.GoCoreKindParentPath, nil),
codegen.RawKindJenny(kindsys.GoCoreKindParentPath, nil),
codegen.BaseCoreRegistryJenny(filepath.Join("pkg", "registry", "corekind"), kindsys.GoCoreKindParentPath),
codegen.TSTypesJenny(kindsys.TSCoreKindParentPath, &codegen.TSTypesGeneratorConfig{
GenDirName: func(decl *codegen.DeclForGen) string {
// FIXME this hardcodes always generating to experimental dir. OK for now, but need generator fanout
return filepath.Join(decl.Meta.Common().MachineName, "x")
},
}),
codegen.LatestMajorsOrXJenny(kindsys.TSCoreKindParentPath, codegen.TSTypesJenny{}),
codegen.TSVeneerIndexJenny(filepath.Join("packages", "grafana-schema", "src")),
)

1
packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts
View File

@ -4,6 +4,7 @@
// kinds/gen.go
// Using jennies:
// TSTypesJenny
// LatestMajorsOrXJenny
//
// Run 'make gen-cue' from repository root to regenerate.

1
packages/grafana-schema/src/raw/playlist/x/playlist_types.gen.ts
View File

@ -4,6 +4,7 @@
// kinds/gen.go
// Using jennies:
// TSTypesJenny
// LatestMajorsOrXJenny
//
// Run 'make gen-cue' from repository root to regenerate.

48
pkg/codegen/generators.go
View File

@ -57,7 +57,6 @@ func StructuredForGen(k kindsys.Structured) *DeclForGen {
type DeclForGen struct {
*kindsys.SomeDecl
lin thema.Lineage
sch thema.Lineage
}
// Lineage returns the [thema.Lineage] for the underlying [kindsys.SomeDecl].
@ -65,10 +64,15 @@ func (decl *DeclForGen) Lineage() thema.Lineage {
return decl.lin
}
// Schema returns the [thema.Schema] that a jenny should operate against, for those
// jennies that target a single schema.
func (decl *DeclForGen) Schema() thema.Lineage {
return decl.sch
// ForLatestSchema returns a [SchemaForGen] for the latest schema in this
// DeclForGen's lineage.
func (decl *DeclForGen) ForLatestSchema() SchemaForGen {
comm := decl.Meta.Common()
return SchemaForGen{
Name: comm.Name,
Schema: decl.Lineage().Latest(),
IsGroup: comm.LineageIsGroup,
}
}
// SlashHeaderMapper produces a FileMapper that injects a comment header onto
@ -82,22 +86,28 @@ func SlashHeaderMapper(maingen string) codejen.FileMapper {
case ".json", ".yml", ".yaml":
return f, nil
default:
b := new(bytes.Buffer)
fmt.Fprintf(b, headerTmpl, filepath.ToSlash(maingen), f.FromString())
fmt.Fprint(b, string(f.Data))
f.Data = b.Bytes()
buf := new(bytes.Buffer)
if err := tmpls.Lookup("gen_header.tmpl").Execute(buf, tvars_gen_header{
MainGenerator: maingen,
Using: f.From,
}); err != nil {
return codejen.File{}, fmt.Errorf("failed executing gen header template: %w", err)
}
fmt.Fprint(buf, string(f.Data))
f.Data = buf.Bytes()
}
return f, nil
}
}
var headerTmpl = `// THIS FILE IS GENERATED. EDITING IS FUTILE.
//
// Generated by:
// %s
// Using jennies:
// %s
//
// Run 'make gen-cue' from repository root to regenerate.
`
// SchemaForGen is an intermediate values type for jennies that holds both a thema.Schema,
// and values relevant to generating the schema that should properly, eventually, be in
// thema itself.
type SchemaForGen struct {
// The PascalCase name of the schematized type.
Name string
// The schema to be rendered for the type itself.
Schema thema.Schema
// Whether the schema is grouped. See https://github.com/grafana/thema/issues/62
IsGroup bool
}

74
pkg/codegen/jenny_eachmajor.go Normal file
View File

@ -0,0 +1,74 @@
package codegen
import (
"fmt"
"path/filepath"
"github.com/grafana/codejen"
"github.com/grafana/grafana/pkg/kindsys"
)
// LatestMajorsOrXJenny returns a jenny that repeats the input for the latest in each major version,
func LatestMajorsOrXJenny(parentdir string, inner codejen.OneToOne[SchemaForGen]) OneToMany {
if inner == nil {
panic("inner jenny must not be nil")
}
return &lmox{
parentdir: parentdir,
inner: inner,
}
}
type lmox struct {
parentdir string
inner codejen.OneToOne[SchemaForGen]
}
func (j *lmox) JennyName() string {
return "LatestMajorsOrXJenny"
}
func (j *lmox) Generate(decl *DeclForGen) (codejen.Files, error) {
if decl.IsRaw() {
return nil, nil
}
comm := decl.Meta.Common()
sfg := SchemaForGen{
Name: comm.Name,
IsGroup: comm.LineageIsGroup,
}
do := func(sfg SchemaForGen, infix string) (codejen.Files, error) {
f, err := j.inner.Generate(sfg)
if err != nil {
return nil, fmt.Errorf("%s jenny failed on %s schema for %s: %w", j.inner.JennyName(), sfg.Schema.Version(), decl.Meta.Common().Name, err)
}
if f == nil || !f.Exists() {
return nil, nil
}
f.RelativePath = filepath.Join(j.parentdir, comm.MachineName, infix, f.RelativePath)
f.From = append(f.From, j)
return codejen.Files{*f}, nil
}
if comm.Maturity.Less(kindsys.MaturityStable) {
sfg.Schema = decl.Lineage().Latest()
return do(sfg, "x")
}
var fl codejen.Files
for sch := decl.Lineage().First(); sch != nil; sch.Successor() {
sfg.Schema = sch.LatestInMajor()
files, err := do(sfg, fmt.Sprintf("v%v", sch.Version()[0]))
if err != nil {
return nil, err
}
fl = append(fl, files...)
}
if fl.Validate() != nil {
return nil, fl.Validate()
}
return fl, nil
}

89
pkg/codegen/jenny_gotypes.go
View File

@ -1,98 +1,29 @@
package codegen
import (
"fmt"
"path/filepath"
"github.com/grafana/codejen"
"github.com/grafana/thema"
"github.com/grafana/thema/encoding/gocode"
"golang.org/x/tools/go/ast/astutil"
)
// GoTypesJenny creates a [OneToOne] that produces Go types for the latest
// Thema schema in a structured kind's lineage.
//
// At minimum, a gokindsdir must be provided. This should be the path to the parent
// directory of the directory in which the types should be generated, relative
// to the project root. For example, if the types for a kind named "foo"
// should live at pkg/kind/foo/foo_gen.go, relpath should be "pkg/kind".
//
// This generator is a no-op for raw kinds.
func GoTypesJenny(gokindsdir string, cfg *GoTypesGeneratorConfig) OneToOne {
if cfg == nil {
cfg = new(GoTypesGeneratorConfig)
}
if cfg.GenDirName == nil {
cfg.GenDirName = func(decl *DeclForGen) string {
return decl.Meta.Common().MachineName
}
}
// GoTypesJenny creates a [OneToOne] that produces Go types for the provided
// [thema.Schema].
type GoTypesJenny struct{}
return &genGoTypes{
gokindsdir: gokindsdir,
cfg: cfg,
}
}
// GoTypesGeneratorConfig holds configuration options for [GoTypesJenny].
type GoTypesGeneratorConfig struct {
// Apply is an optional AST manipulation func that, if provided, will be run
// against the generated Go file prior to running it through goimports.
Apply astutil.ApplyFunc
// GenDirName returns the name of the parent directory in which the type file
// should be generated. If nil, the DeclForGen.Lineage().Name() will be used.
GenDirName func(*DeclForGen) string
// Version of the schema to generate. If nil, latest is generated.
Version *thema.SyntacticVersion
}
type genGoTypes struct {
gokindsdir string
cfg *GoTypesGeneratorConfig
}
func (gen *genGoTypes) JennyName() string {
func (j GoTypesJenny) JennyName() string {
return "GoTypesJenny"
}
func (gen *genGoTypes) Generate(decl *DeclForGen) (*codejen.File, error) {
if decl.IsRaw() {
return nil, nil
}
var sch thema.Schema
var err error
lin := decl.Lineage()
if gen.cfg.Version == nil {
sch = lin.Latest()
} else {
sch, err = lin.Schema(*gen.cfg.Version)
if err != nil {
return nil, fmt.Errorf("error in configured version for %s generator: %w", *gen.cfg.Version, err)
}
}
// always drop prefixes.
var appf []astutil.ApplyFunc
if gen.cfg.Apply != nil {
appf = append(appf, gen.cfg.Apply)
}
appf = append(appf, PrefixDropper(decl.Meta.Common().Name))
pdir := gen.cfg.GenDirName(decl)
fpath := filepath.Join(gen.gokindsdir, pdir, lin.Name()+"_types_gen.go")
func (j GoTypesJenny) Generate(sfg SchemaForGen) (*codejen.File, error) {
// TODO allow using name instead of machine name in thema generator
b, err := gocode.GenerateTypesOpenAPI(sch, &gocode.TypeConfigOpenAPI{
PackageName: filepath.Base(pdir),
ApplyFuncs: appf,
b, err := gocode.GenerateTypesOpenAPI(sfg.Schema, &gocode.TypeConfigOpenAPI{
// TODO will need to account for sanitizing e.g. dashes here at some point
PackageName: sfg.Schema.Lineage().Name(),
ApplyFuncs: []astutil.ApplyFunc{PrefixDropper(sfg.Name)},
})
if err != nil {
return nil, err
}
return codejen.NewFile(fpath, b, gen), nil
return codejen.NewFile(sfg.Schema.Lineage().Name()+"_types_gen.go", b, j), nil
}

79
pkg/codegen/jenny_tstypes.go
View File

@ -1,85 +1,32 @@
package codegen
import (
"fmt"
"path/filepath"
"github.com/grafana/codejen"
"github.com/grafana/thema"
"github.com/grafana/thema/encoding/typescript"
)
// TSTypesJenny creates a [OneToOne] that produces TypeScript types and
// defaults for the latest Thema schema in a structured kind's lineage.
// TSTypesJenny is a [OneToOne] that produces TypeScript types and
// defaults for a Thema schema.
//
// At minimum, a tskindsdir must be provided. This should be the path to the parent
// directory of the directory in which the types should be generated, relative
// to the project root. For example, if the types for a kind named "foo"
// should live at packages/grafana-schema/src/raw/foo, relpath should be "pkg/kind".
//
// This generator is a no-op for raw kinds.
func TSTypesJenny(tskindsdir string, cfg *TSTypesGeneratorConfig) OneToOne {
if cfg == nil {
cfg = new(TSTypesGeneratorConfig)
}
if cfg.GenDirName == nil {
cfg.GenDirName = func(decl *DeclForGen) string {
return decl.Meta.Common().MachineName
}
}
// Thema's generic TS jenny will be able to replace this one once
// https://github.com/grafana/thema/issues/89 is complete.
type TSTypesJenny struct{}
return &genTSTypes{
tskindsdir: tskindsdir,
cfg: cfg,
}
}
var _ codejen.OneToOne[SchemaForGen] = &TSTypesJenny{}
// TSTypesGeneratorConfig holds configuration options for [TSTypesJenny].
type TSTypesGeneratorConfig struct {
// GenDirName returns the name of the parent directory in which the type file
// should be generated. If nil, the DeclForGen.Lineage().Name() will be used.
GenDirName func(*DeclForGen) string
// Version of the schema to generate. If nil, latest is generated.
Version *thema.SyntacticVersion
}
type genTSTypes struct {
tskindsdir string
cfg *TSTypesGeneratorConfig
}
func (gen *genTSTypes) JennyName() string {
func (j TSTypesJenny) JennyName() string {
return "TSTypesJenny"
}
func (gen *genTSTypes) Generate(decl *DeclForGen) (*codejen.File, error) {
if decl.IsRaw() {
return nil, nil
}
var sch thema.Schema
var err error
lin := decl.Lineage()
if gen.cfg.Version == nil {
sch = lin.Latest()
} else {
sch, err = lin.Schema(*gen.cfg.Version)
if err != nil {
return nil, fmt.Errorf("error in configured version for %s generator: %w", *gen.cfg.Version, err)
}
}
func (j TSTypesJenny) Generate(sfg SchemaForGen) (*codejen.File, error) {
// TODO allow using name instead of machine name in thema generator
f, err := typescript.GenerateTypes(sch, &typescript.TypeConfig{
RootName: decl.Meta.Common().Name,
Group: decl.Meta.Common().LineageIsGroup,
f, err := typescript.GenerateTypes(sfg.Schema, &typescript.TypeConfig{
RootName: sfg.Name,
Group: sfg.IsGroup,
})
if err != nil {
return nil, err
}
return codejen.NewFile(
filepath.Join(gen.tskindsdir, gen.cfg.GenDirName(decl), lin.Name()+"_types.gen.ts"),
[]byte(f.String()),
gen), nil
return codejen.NewFile(sfg.Schema.Lineage().Name()+"_types.gen.ts", []byte(f.String()), j), nil
}

56
pkg/codegen/latest_jenny.go Normal file
View File

@ -0,0 +1,56 @@
package codegen
import (
"fmt"
"path/filepath"
"github.com/grafana/codejen"
)
// LatestJenny returns a jenny that runs another jenny for only the latest
// schema in a DeclForGen, and prefixes the resulting file with the provided
// parentdir (e.g. "pkg/kinds/") and with a directory based on the kind's
// machine name (e.g. "dashboard/").
func LatestJenny(parentdir string, inner codejen.OneToOne[SchemaForGen]) OneToOne {
if inner == nil {
panic("inner jenny must not be nil")
}
return &latestj{
parentdir: parentdir,
inner: inner,
}
}
type latestj struct {
parentdir string
inner codejen.OneToOne[SchemaForGen]
}
func (j *latestj) JennyName() string {
return "LatestJenny"
}
func (j *latestj) Generate(decl *DeclForGen) (*codejen.File, error) {
if decl.IsRaw() {
return nil, nil
}
comm := decl.Meta.Common()
sfg := SchemaForGen{
Name: comm.Name,
Schema: decl.Lineage().Latest(),
IsGroup: comm.LineageIsGroup,
}
f, err := j.inner.Generate(sfg)
if err != nil {
return nil, fmt.Errorf("%s jenny failed on %s schema for %s: %w", j.inner.JennyName(), sfg.Schema.Version(), decl.Meta.Common().Name, err)
}
if f == nil || !f.Exists() {
return nil, nil
}
f.RelativePath = filepath.Join(j.parentdir, comm.MachineName, f.RelativePath)
f.From = append(f.From, j)
return f, nil
}

7
pkg/codegen/tmpl.go
View File

@ -5,6 +5,8 @@ import (
"embed"
"text/template"
"time"
"github.com/grafana/codejen"
)
// All the parsed templates in the tmpl subdirectory
@ -29,6 +31,11 @@ type (
LineageCUEPath string
GenLicense bool
}
tvars_gen_header struct {
MainGenerator string
Using []codejen.NamedJenny
From string
}
tvars_kind_registry struct {
// Header tvars_autogen_header
NumRaw, NumStructured int

11
pkg/codegen/tmpl/gen_header.tmpl Normal file
View File

@ -0,0 +1,11 @@
// THIS FILE IS GENERATED. EDITING IS FUTILE.
//
// Generated by:
// {{ .MainGenerator }}
// Using jennies:
{{- range .Using }}
// {{ .JennyName }}
{{- end }}
//
// Run 'make gen-cue' from repository root to regenerate.

1
pkg/kinds/dashboard/dashboard_types_gen.go
View File

@ -4,6 +4,7 @@
// kinds/gen.go
// Using jennies:
// GoTypesJenny
// LatestJenny
//
// Run 'make gen-cue' from repository root to regenerate.

1
pkg/kinds/playlist/playlist_types_gen.go
View File

@ -4,6 +4,7 @@
// kinds/gen.go
// Using jennies:
// GoTypesJenny
// LatestJenny
//
// Run 'make gen-cue' from repository root to regenerate.