Skip to content

Commit 0061aac

Browse files
committed
feat: add creator plugins
Problem: we did not have a way to define a creator as a plugin. Solution: add a plugin interface to create. I originally was going to create separate plugin types, but I like the idea that one plugin family can decide to define both easily. When this is refactored to have a more "register" design (to make it flexible to changing the set available) it will be nice to provide Create/Extract from the same interface and keep the number of interfaces / functions for them minimal. I was going to add hwloc now, but there seems to be a bug so we will need to move forward prototyping with the current proxy for nodes, which is just using the go runtime package. We obviously need to improve upon this. Signed-off-by: vsoch <[email protected]>
1 parent 11118b1 commit 0061aac

34 files changed

+864
-622
lines changed

.gitignore

+4-1
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,7 @@
1717
# Dependency directories (remove the comment below to include it)
1818
# vendor/
1919
bin
20-
vendor
20+
vendor
21+
cache
22+
lib
23+
*.json

Makefile.hwloc

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# This makefile will be used when we can add hwloc - there is currently a bug.
2+
HERE ?= $(shell pwd)
3+
LOCALBIN ?= $(shell pwd)/bin
4+
5+
# Install hwloc here for use to compile, etc.
6+
LOCALLIB ?= $(shell pwd)/lib
7+
HWLOC_INCLUDE ?= $(LOCALLIB)/include/hwloc.h
8+
BUILDENVVAR=CGO_CFLAGS="-I$(LOCALLIB)/include" CGO_LDFLAGS="-L$(LOCALLIB)/lib -lhwloc"
9+
10+
.PHONY: all
11+
12+
all: build
13+
14+
.PHONY: $(LOCALBIN)
15+
$(LOCALBIN):
16+
mkdir -p $(LOCALBIN)
17+
18+
.PHONY: $(LOCALLIB)
19+
$(LOCALLIB):
20+
mkdir -p $(LOCALLIB)
21+
22+
$(HWLOC_INCLUDE):
23+
git clone --depth 1 https://github.com/open-mpi/hwloc /tmp/hwloc || true && \
24+
cd /tmp/hwloc && ./autogen.sh && \
25+
./configure --enable-static --disable-shared LDFLAGS="-static" --prefix=$(LOCALLIB)/ && \
26+
make LDFLAGS=-all-static && make install
27+
28+
build: $(LOCALBIN) $(HWLOC_INCLUDE)
29+
GO111MODULE="on" $(BUILDENVVAR) go build -ldflags '-w' -o $(LOCALBIN)/compspec cmd/compspec/compspec.go
30+
31+
build-arm: $(LOCALBIN) $(HWLOC_INCLUDE)
32+
GO111MODULE="on" $(BUILDENVVAR) GOARCH=arm64 go build -ldflags '-w' -o $(LOCALBIN)/compspec-arm cmd/compspec/compspec.go
33+
34+
build-ppc: $(LOCALBIN) $(HWLOC_INCLUDE)
35+
GO111MODULE="on" $(BUILDENVVAR) GOARCH=ppc64le go build -ldflags '-w' -o $(LOCALBIN)/compspec-ppc cmd/compspec/compspec.go

README.md

+2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ This is a prototype compatibility checking tool. Right now our aim is to use in
1111

1212
- I'm starting with just Linux. I know there are those "other" platforms, but if it doesn't run on HPC or Kubernetes easily I'm not super interested (ahem, Mac and Windows)!
1313
- not all extractors work in containers (e.g., kernel needs to be on the host)
14+
- The node feature discovery source doesn't provide mapping of socket -> cores, nor does it give details about logical vs. physical CPU.
15+
- We will likely want to add hwloc go bindings, but there is a bug currently.
1416

1517
Note that for development we are using nfd-source that does not require kubernetes:
1618

cmd/compspec/create/artifact.go

+14-110
Original file line numberDiff line numberDiff line change
@@ -1,127 +1,31 @@
11
package create
22

33
import (
4-
"fmt"
5-
"os"
4+
"strings"
65

7-
"github.com/compspec/compspec-go/pkg/types"
8-
ep "github.com/compspec/compspec-go/plugins/extractors"
9-
10-
p "github.com/compspec/compspec-go/plugins"
6+
"github.com/compspec/compspec-go/plugins/creators/artifact"
117
)
128

139
// Artifact will create a compatibility artifact based on a request in YAML
1410
// TODO likely want to refactor this into a proper create plugin
1511
func Artifact(specname string, fields []string, saveto string, allowFail bool) error {
1612

17-
// Cut out early if a spec not provided
18-
if specname == "" {
19-
return fmt.Errorf("a spec input -i/--input is required")
20-
}
21-
request, err := loadRequest(specname)
22-
if err != nil {
23-
return err
24-
}
25-
26-
// Right now we only know about extractors, when we define subfields
27-
// we can further filter here.
28-
extractors := request.GetExtractors()
29-
plugins, err := ep.GetPlugins(extractors)
30-
if err != nil {
31-
return err
32-
}
33-
34-
// Finally, add custom fields and extract metadata
35-
result, err := plugins.Extract(allowFail)
36-
if err != nil {
37-
return err
13+
// This is janky, oh well
14+
allowFailFlag := "false"
15+
if allowFail {
16+
allowFailFlag = "true"
3817
}
3918

40-
// Update with custom fields (either new or overwrite)
41-
result.AddCustomFields(fields)
42-
43-
// The compspec returned is the populated Compatibility request!
44-
compspec, err := PopulateExtractors(&result, request)
19+
// assemble options for node creator
20+
creator, err := artifact.NewPlugin()
4521
if err != nil {
4622
return err
4723
}
48-
49-
output, err := compspec.ToJson()
50-
if err != nil {
51-
return err
24+
options := map[string]string{
25+
"specname": specname,
26+
"fields": strings.Join(fields, "||"),
27+
"saveto": saveto,
28+
"allowFail": allowFailFlag,
5229
}
53-
if saveto == "" {
54-
fmt.Println(string(output))
55-
} else {
56-
err = os.WriteFile(saveto, output, 0644)
57-
if err != nil {
58-
return err
59-
}
60-
}
61-
return nil
62-
}
63-
64-
// LoadExtractors loads a compatibility result into a compatibility request
65-
// After this we can save the populated thing into an artifact (json DUMP)
66-
func PopulateExtractors(result *ep.Result, request *types.CompatibilityRequest) (*types.CompatibilityRequest, error) {
67-
68-
// Every metadata attribute must be known under a schema
69-
schemas := request.Metadata.Schemas
70-
if len(schemas) == 0 {
71-
return nil, fmt.Errorf("the request must have one or more schemas")
72-
}
73-
for i, compat := range request.Compatibilities {
74-
75-
// The compatibility section name is a schema, and must be defined
76-
url, ok := schemas[compat.Name]
77-
if !ok {
78-
return nil, fmt.Errorf("%s is missing a schema", compat.Name)
79-
}
80-
if url == "" {
81-
return nil, fmt.Errorf("%s has an empty schema", compat.Name)
82-
}
83-
84-
for key, extractorKey := range compat.Attributes {
85-
86-
// Get the extractor, section, and subfield from the extractor lookup key
87-
f, err := p.ParseField(extractorKey)
88-
if err != nil {
89-
fmt.Printf("warning: cannot parse %s: %s, setting to empty\n", key, extractorKey)
90-
compat.Attributes[key] = ""
91-
continue
92-
}
93-
94-
// If we get here, we can parse it and look it up in our result metadata
95-
extractor, ok := result.Results[f.Extractor]
96-
if !ok {
97-
fmt.Printf("warning: extractor %s is unknown, setting to empty\n", f.Extractor)
98-
compat.Attributes[key] = ""
99-
continue
100-
}
101-
102-
// Now get the section
103-
section, ok := extractor.Sections[f.Section]
104-
if !ok {
105-
fmt.Printf("warning: section %s.%s is unknown, setting to empty\n", f.Extractor, f.Section)
106-
compat.Attributes[key] = ""
107-
continue
108-
}
109-
110-
// Now get the value!
111-
value, ok := section[f.Field]
112-
if !ok {
113-
fmt.Printf("warning: field %s.%s.%s is unknown, setting to empty\n", f.Extractor, f.Section, f.Field)
114-
compat.Attributes[key] = ""
115-
continue
116-
}
117-
118-
// If we get here - we found it! Hooray!
119-
compat.Attributes[key] = value
120-
}
121-
122-
// Update the compatibiity
123-
request.Compatibilities[i] = compat
124-
}
125-
126-
return request, nil
30+
return creator.Create(options)
12731
}

cmd/compspec/create/create.go

-23
This file was deleted.

0 commit comments

Comments
 (0)