diff --git a/cmd/dump.go b/cmd/dump.go index 8c0b274..16d9ae2 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -32,7 +32,7 @@ func runDump(ctx context.Context, w io.Writer, name string) error { return err } - endpointID, err := getPodEndpointID(ctx, dynamicClient, rootOptions.namespace, name) + endpointID, _, err := getPodEndpointID(ctx, dynamicClient, rootOptions.namespace, name) if err != nil { return err } diff --git a/cmd/helper.go b/cmd/helper.go index 7f0c40f..8c30a22 100644 --- a/cmd/helper.go +++ b/cmd/helper.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strconv" "github.com/cilium/cilium/pkg/client" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,13 +15,17 @@ import ( "k8s.io/client-go/rest" ) +const ( + directionEgress = "EGRESS" + directionIngress = "INGRESS" +) + func createClients(ctx context.Context, name string) (*kubernetes.Clientset, *dynamic.DynamicClient, *client.Client, error) { config, err := rest.InClusterConfig() if err != nil { return nil, nil, nil, err } - // Create Kubernetes Clients clientset, err := kubernetes.NewForConfig(config) if err != nil { return nil, nil, nil, err @@ -31,12 +36,7 @@ func createClients(ctx context.Context, name string) (*kubernetes.Clientset, *dy return nil, nil, nil, err } - // Create Cilium Client - endpoint, err := getProxyEndpoint(ctx, clientset, rootOptions.namespace, name) - if err != nil { - return nil, nil, nil, err - } - ciliumClient, err := client.NewClient(endpoint) + ciliumClient, err := createCiliumClient(ctx, clientset, rootOptions.namespace, name) if err != nil { return nil, nil, nil, err } @@ -44,6 +44,18 @@ func createClients(ctx context.Context, name string) (*kubernetes.Clientset, *dy return clientset, dynamicClient, ciliumClient, err } +func createCiliumClient(ctx context.Context, clientset *kubernetes.Clientset, namespace, name string) (*client.Client, error) { + endpoint, err := getProxyEndpoint(ctx, clientset, namespace, name) + if err != nil { + return nil, err + } + client, err := client.NewClient(endpoint) + if err != nil { + return nil, err + } + return client, nil +} + func getProxyEndpoint(ctx context.Context, c *kubernetes.Clientset, namespace, name string) (string, error) { targetPod, err := c.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { @@ -67,7 +79,7 @@ func getProxyEndpoint(ctx context.Context, c *kubernetes.Clientset, namespace, n return fmt.Sprintf("http://%s:%d", podIP, rootOptions.proxyPort), nil } -func getPodEndpointID(ctx context.Context, d *dynamic.DynamicClient, namespace, name string) (int64, error) { +func getPodEndpointID(ctx context.Context, d *dynamic.DynamicClient, namespace, name string) (int64, int64, error) { gvr := schema.GroupVersionResource{ Group: "cilium.io", Version: "v2", @@ -76,16 +88,46 @@ func getPodEndpointID(ctx context.Context, d *dynamic.DynamicClient, namespace, ep, err := d.Resource(gvr).Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { - return 0, err + return 0, 0, err } endpointID, found, err := unstructured.NestedInt64(ep.Object, "status", "id") if err != nil { - return 0, err + return 0, 0, err + } + if !found { + return 0, 0, errors.New("CiliumEndpoint does not have .status.id") + } + + endpointIdentity, found, err := unstructured.NestedInt64(ep.Object, "status", "identity", "id") + if err != nil { + return 0, 0, err } if !found { - return 0, errors.New("endpoint resource is broken") + return 0, 0, errors.New("CiliumEndpoint does not have .status.identity.id") } - return endpointID, nil + return endpointID, endpointIdentity, nil +} + +func listCiliumIDs(ctx context.Context, d *dynamic.DynamicClient) (*unstructured.UnstructuredList, error) { + gvr := schema.GroupVersionResource{ + Group: "cilium.io", + Version: "v2", + Resource: "ciliumidentities", + } + return d.Resource(gvr).List(ctx, metav1.ListOptions{}) +} + +func findCiliumID(dict *unstructured.UnstructuredList, id int64) *unstructured.Unstructured { + if dict == nil { + return nil + } + name := strconv.FormatInt(id, 10) + for _, item := range dict.Items { + if item.GetName() == name { + return &item + } + } + return nil } diff --git a/cmd/l3.go b/cmd/l3.go new file mode 100644 index 0000000..96615f0 --- /dev/null +++ b/cmd/l3.go @@ -0,0 +1,13 @@ +package cmd + +import "github.com/spf13/cobra" + +var l3Cmd = &cobra.Command{ + Use: "l3", + Short: "inspect l3 rules", + Long: `inspect l3 rules`, +} + +func init() { + rootCmd.AddCommand(l3Cmd) +} diff --git a/cmd/l3_inspect.go b/cmd/l3_inspect.go new file mode 100644 index 0000000..6a3e4bf --- /dev/null +++ b/cmd/l3_inspect.go @@ -0,0 +1,221 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "slices" + "sort" + "strconv" + "strings" + "text/tabwriter" + + "github.com/cilium/cilium/api/v1/client/endpoint" + "github.com/spf13/cobra" + "golang.org/x/exp/maps" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func init() { + l3Cmd.AddCommand(l3InspectCmd) +} + +var l3InspectCmd = &cobra.Command{ + Use: "inspect", + Short: "", + Long: ``, + + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runL3Inspect(context.Background(), cmd.OutOrStdout(), args[0]) + }, +} + +type l3InspectEntry struct { + Direction string `json:"direction"` + Allowed bool `json:"allowed"` + Namespace string `json:"namespace"` + Identity int64 `json:"identity"` + Labels map[string]string `json:"labels"` +} + +func buildL3InspectEntry(dict *unstructured.UnstructuredList, id int64, direction string, allowed bool) (l3InspectEntry, error) { + val := l3InspectEntry{ + Direction: direction, + Allowed: allowed, + Identity: id, + } + + // https://docs.cilium.io/en/latest/gettingstarted/terminology/#special-identities + switch id { + case 0: + val.Labels = map[string]string{"!! reserved": "unknown"} + return val, nil + case 1: + val.Labels = map[string]string{"!! reserved": "host"} + return val, nil + case 2: + val.Labels = map[string]string{"!! reserved": "world"} + return val, nil + case 3: + val.Labels = map[string]string{"!! reserved": "unmanaged"} + return val, nil + case 4: + val.Labels = map[string]string{"!! reserved": "health"} + return val, nil + case 5: + val.Labels = map[string]string{"!! reserved": "init"} + return val, nil + case 6: + val.Labels = map[string]string{"!! reserved": "remote-node"} + return val, nil + case 7: + val.Labels = map[string]string{"!! reserved": "kube-apiserver"} + return val, nil + case 8: + val.Labels = map[string]string{"!! reserved": "ingress"} + return val, nil + } + + obj := findCiliumID(dict, id) + if obj == nil { + return l3InspectEntry{}, fmt.Errorf("CiliumID is not found for ID: %d", id) + } + + labels, found, err := unstructured.NestedStringMap(obj.Object, "security-labels") + if !found { + return l3InspectEntry{}, fmt.Errorf("security label is missing for CiliumID: %d", id) + } + if err != nil { + return l3InspectEntry{}, err + } + val.Labels = labels + + ns, ok := labels["k8s:io.kubernetes.pod.namespace"] + if !ok { + return l3InspectEntry{}, fmt.Errorf("namespace label is missing for CiliumID: %d", id) + } + val.Namespace = ns + return val, nil +} + +func compareL3InspectEntry(x, y *l3InspectEntry) bool { + if x.Direction != y.Direction { + return strings.Compare(x.Direction, y.Direction) < 0 + } + if x.Allowed != y.Allowed { + return x.Allowed + } + if x.Namespace != y.Namespace { + return strings.Compare(x.Namespace, y.Namespace) < 0 + } + if x.Identity != y.Identity { + return x.Identity < y.Identity + } + // Labels should differ between identities + return false +} + +func runL3Inspect(ctx context.Context, w io.Writer, name string) error { + _, dynamicClient, client, err := createClients(ctx, name) + if err != nil { + return err + } + + ciliumIDs, err := listCiliumIDs(ctx, dynamicClient) + if err != nil { + return err + } + + endpointID, _, err := getPodEndpointID(ctx, dynamicClient, rootOptions.namespace, name) + if err != nil { + return err + } + + params := endpoint.GetEndpointIDParams{ + Context: ctx, + ID: strconv.FormatInt(endpointID, 10), + } + response, err := client.Endpoint.GetEndpointID(¶ms) + if err != nil { + return err + } + + policyList := make([]l3InspectEntry, 0) + + allowedEgress := response.Payload.Status.Policy.Realized.AllowedEgressIdentities + for _, id := range allowedEgress { + entry, err := buildL3InspectEntry(ciliumIDs, id, directionEgress, true) + if err != nil { + return err + } + policyList = append(policyList, entry) + } + + deniedEgress := response.Payload.Status.Policy.Realized.DeniedEgressIdentities + for _, id := range deniedEgress { + entry, err := buildL3InspectEntry(ciliumIDs, id, directionEgress, false) + if err != nil { + return err + } + policyList = append(policyList, entry) + } + + allowedIngress := response.Payload.Status.Policy.Realized.AllowedIngressIdentities + for _, id := range allowedIngress { + entry, err := buildL3InspectEntry(ciliumIDs, id, directionIngress, true) + if err != nil { + return err + } + policyList = append(policyList, entry) + } + + deniedIngress := response.Payload.Status.Policy.Realized.DeniedIngressIdentities + for _, id := range deniedIngress { + entry, err := buildL3InspectEntry(ciliumIDs, id, directionIngress, false) + if err != nil { + return err + } + policyList = append(policyList, entry) + } + + sort.Slice(policyList, func(i, j int) bool { return compareL3InspectEntry(&policyList[i], &policyList[j]) }) + + switch rootOptions.output { + case OutputJson: + text, err := json.MarshalIndent(policyList, "", " ") + if err != nil { + return err + } + _, err = w.Write(text) + return err + case OutputSimple: + tw := tabwriter.NewWriter(w, 0, 1, 1, ' ', 0) + _, err := tw.Write([]byte("DIRECTION\tALLOWED\tNAMESPACE\tIDENTITY\tLABELS\n")) + if err != nil { + return err + } + for _, p := range policyList { + keys := maps.Keys(p.Labels) + slices.Sort(keys) + for i, k := range keys { + switch i { + case 0: + _, err := tw.Write([]byte(fmt.Sprintf("%v\t%v\t%v\t%v\t%v=%v\n", p.Direction, p.Allowed, p.Namespace, p.Identity, k, p.Labels[k]))) + if err != nil { + return err + } + default: + _, err := tw.Write([]byte(fmt.Sprintf("\t\t\t\t%v=%v\n", k, p.Labels[k]))) + if err != nil { + return err + } + } + } + } + return tw.Flush() + default: + return fmt.Errorf("unknown format: %s", rootOptions.output) + } +} diff --git a/cmd/l3_reach.go b/cmd/l3_reach.go new file mode 100644 index 0000000..f9b1a7e --- /dev/null +++ b/cmd/l3_reach.go @@ -0,0 +1,152 @@ +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "slices" + "strconv" + "strings" + "text/tabwriter" + + "github.com/cilium/cilium/api/v1/client/endpoint" + "github.com/spf13/cobra" +) + +var l3ReachOptions struct { + to string +} + +func init() { + l3ReachCmd.Flags().StringVar(&l3ReachOptions.to, "to", "", "target pod namespace/name") + l3Cmd.AddCommand(l3ReachCmd) +} + +const ( + typeExplicit = "EXPLICIT" + typeImplicit = "IMPLICIT" +) + +type l3ReachEntry struct { + Direction string `json:"direction"` + Allowed bool `json:"allowed"` + Type string `json:"type"` +} + +var l3ReachCmd = &cobra.Command{ + Use: "reach", + Short: "", + Long: ``, + + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + to := l3ReachOptions.to + if to == "" { + return errors.New("please specify --to parameter") + } + toList := strings.Split(to, "/") + if len(toList) != 2 { + return errors.New("--to must be NAMESPACE/NAME") + } + return runL3Reach(context.Background(), cmd.OutOrStdout(), args[0], toList[0], toList[1]) + }, +} + +func runL3Reach(ctx context.Context, w io.Writer, name, toNamespace, toName string) error { + clientset, dynamicClient, senderClient, err := createClients(ctx, name) + if err != nil { + return err + } + receiverClient, err := createCiliumClient(ctx, clientset, toNamespace, toName) + if err != nil { + return err + } + + senderEndpointID, senderIdentity, err := getPodEndpointID(ctx, dynamicClient, rootOptions.namespace, name) + if err != nil { + return err + } + + receiverEndpointID, receiverIdentity, err := getPodEndpointID(ctx, dynamicClient, toNamespace, toName) + if err != nil { + return err + } + + // Check egress rule to see if sender can send packets to receiver + params := endpoint.GetEndpointIDParams{ + Context: ctx, + ID: strconv.FormatInt(senderEndpointID, 10), + } + senderResponse, err := senderClient.Endpoint.GetEndpointID(¶ms) + if err != nil { + return err + } + + allowedEgressIdentities := senderResponse.Payload.Status.Policy.Realized.AllowedEgressIdentities + deniedEgressIdentities := senderResponse.Payload.Status.Policy.Realized.DeniedEgressIdentities + egressOK := slices.Contains(allowedEgressIdentities, receiverIdentity) + egressDeny := slices.Contains(deniedEgressIdentities, receiverIdentity) + egressType := typeExplicit + if !egressOK && !egressDeny { + egressType = typeImplicit + } + + // Check ingress rule to see if receiver can receive packets from sender + params = endpoint.GetEndpointIDParams{ + Context: ctx, + ID: strconv.FormatInt(receiverEndpointID, 10), + } + receiverResponse, err := receiverClient.Endpoint.GetEndpointID(¶ms) + if err != nil { + return err + } + + allowedIngressIdentities := receiverResponse.Payload.Status.Policy.Realized.AllowedIngressIdentities + deniedIngressIdentities := receiverResponse.Payload.Status.Policy.Realized.DeniedIngressIdentities + ingressOK := slices.Contains(allowedIngressIdentities, senderIdentity) + ingressDeny := slices.Contains(deniedIngressIdentities, senderIdentity) + ingressType := typeExplicit + if !ingressOK && !ingressDeny { + ingressType = typeImplicit + } + + policyList := []l3ReachEntry{ + { + Direction: directionEgress, + Allowed: egressOK, + Type: egressType, + }, + { + Direction: directionIngress, + Allowed: ingressOK, + Type: ingressType, + }, + } + + switch rootOptions.output { + case OutputJson: + text, err := json.MarshalIndent(policyList, "", " ") + if err != nil { + return err + } + _, err = w.Write(text) + return err + case OutputSimple: + tw := tabwriter.NewWriter(w, 0, 1, 1, ' ', 0) + _, err := tw.Write([]byte("DIRECTION\tALLOWED\tTYPE\n")) + if err != nil { + return err + } + for _, p := range policyList { + _, err := tw.Write([]byte(fmt.Sprintf("%v\t%v\t%v\n", p.Direction, p.Allowed, p.Type))) + if err != nil { + return err + } + } + return tw.Flush() + default: + return fmt.Errorf("unknown format: %s", rootOptions.output) + } +} diff --git a/cmd/list.go b/cmd/list.go index 03a5861..a9cbf70 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -5,12 +5,14 @@ import ( "encoding/json" "fmt" "io" + "sort" "strconv" "strings" "text/tabwriter" "github.com/cilium/cilium/api/v1/client/endpoint" "github.com/spf13/cobra" + "golang.org/x/exp/maps" ) func init() { @@ -28,11 +30,6 @@ var listCmd = &cobra.Command{ }, } -const ( - directionEgress = "EGRESS" - directionIngress = "INGRESS" -) - type derivedFromEntry struct { Direction string `json:"direction"` Kind string `json:"kind"` @@ -57,13 +54,29 @@ func parseDerivedFromEntry(input []string, direction string) derivedFromEntry { return val } +func compareDerivedFromEntry(x, y *derivedFromEntry) bool { + if x.Direction != y.Direction { + return strings.Compare(x.Direction, y.Direction) < 0 + } + if x.Kind != y.Kind { + return strings.Compare(x.Kind, y.Kind) < 0 + } + if x.Namespace != y.Namespace { + return strings.Compare(x.Namespace, y.Namespace) < 0 + } + if x.Name != y.Name { + return strings.Compare(x.Name, y.Name) < 0 + } + return false +} + func runList(ctx context.Context, w io.Writer, name string) error { _, dynamicClient, client, err := createClients(ctx, name) if err != nil { return err } - endpointID, err := getPodEndpointID(ctx, dynamicClient, rootOptions.namespace, name) + endpointID, _, err := getPodEndpointID(ctx, dynamicClient, rootOptions.namespace, name) if err != nil { return err } @@ -77,22 +90,27 @@ func runList(ctx context.Context, w io.Writer, name string) error { return err } - policyList := make([]derivedFromEntry, 0) + policySet := make(map[derivedFromEntry]struct{}) ingressRules := response.Payload.Status.Policy.Realized.L4.Ingress for _, rule := range ingressRules { for _, r := range rule.DerivedFromRules { - policyList = append(policyList, parseDerivedFromEntry(r, directionIngress)) + entry := parseDerivedFromEntry(r, directionIngress) + policySet[entry] = struct{}{} } } egressRules := response.Payload.Status.Policy.Realized.L4.Egress for _, rule := range egressRules { for _, r := range rule.DerivedFromRules { - policyList = append(policyList, parseDerivedFromEntry(r, directionEgress)) + entry := parseDerivedFromEntry(r, directionEgress) + policySet[entry] = struct{}{} } } + policyList := maps.Keys(policySet) + sort.Slice(policyList, func(i, j int) bool { return compareDerivedFromEntry(&policyList[i], &policyList[j]) }) + switch rootOptions.output { case OutputJson: text, err := json.MarshalIndent(policyList, "", " ") diff --git a/e2e/Makefile b/e2e/Makefile index a6264b9..1118e5b 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -51,8 +51,17 @@ install-test-pod: $(MAKE) --no-print-directory run-test-pod-l3-ingress-explicit-deny $(MAKE) --no-print-directory run-test-pod-l3-egress-implicit-deny $(MAKE) --no-print-directory run-test-pod-l3-egress-explicit-deny + + $(MAKE) --no-print-directory run-test-pod-l4-ingress-explicit-allow + $(MAKE) --no-print-directory run-test-pod-l4-ingress-explicit-allow-tcp + $(MAKE) --no-print-directory run-test-pod-l4-ingress-explicit-deny + $(MAKE) --no-print-directory run-test-pod-l4-ingress-explicit-deny-udp + $(MAKE) --no-print-directory run-test-pod-l4-egress-implicit-deny + $(MAKE) --no-print-directory run-test-pod-l4-egress-explicit-deny $(MAKE) --no-print-directory wait-for-workloads + $(KUBECTL) apply -f testdata/policy/l3.yaml + $(KUBECTL) apply -f testdata/policy/l4.yaml .PHONY: install-cilium-policy install-cilium-policy: diff --git a/e2e/list_test.go b/e2e/list_test.go index 2f64dee..63900c2 100644 --- a/e2e/list_test.go +++ b/e2e/list_test.go @@ -35,6 +35,12 @@ func testList() { "kind": "CiliumNetworkPolicy", "namespace": "default", "name": "l3-egress" + }, + { + "direction": "EGRESS", + "kind": "CiliumNetworkPolicy", + "namespace": "default", + "name": "l4-egress" }]`, }, { diff --git a/e2e/testdata/policy/l3.yaml b/e2e/testdata/policy/l3.yaml index ccc3433..935b4f9 100644 --- a/e2e/testdata/policy/l3.yaml +++ b/e2e/testdata/policy/l3.yaml @@ -10,6 +10,9 @@ spec: - toEndpoints: - matchLabels: k8s:test: l3-ingress-explicit-allow + - toEndpoints: + - matchLabels: + k8s:test: l3-ingress-no-rule - toEndpoints: - matchLabels: k8s:test: l3-ingress-implicit-deny diff --git a/e2e/testdata/policy/l4.yaml b/e2e/testdata/policy/l4.yaml new file mode 100644 index 0000000..233bc97 --- /dev/null +++ b/e2e/testdata/policy/l4.yaml @@ -0,0 +1,35 @@ +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: l4-egress +spec: + endpointSelector: + matchLabels: + k8s:test: self + egress: + - toEndpoints: + - matchLabels: + k8s:test: l4-ingress-explicit-allow + toPorts: + - ports: + - port: "53" + - toEndpoints: + - matchLabels: + k8s:test: l4-ingress-explicit-allow-tcp + toPorts: + - ports: + - port: "8080" + protocol: TCP + - toEndpoints: + - matchLabels: + k8s:test: l4-ingress-explicit-deny + toPorts: + - ports: + - port: "53" + - toEndpoints: + - matchLabels: + k8s:test: l4-ingress-explicit-deny-udp + toPorts: + - ports: + - port: "161" # SNMP (UDP) + protocol: UDP diff --git a/e2e/testdata/ubuntu.yaml b/e2e/testdata/ubuntu.yaml index 2319064..24c5392 100644 --- a/e2e/testdata/ubuntu.yaml +++ b/e2e/testdata/ubuntu.yaml @@ -21,6 +21,7 @@ rules: - cilium.io resources: - ciliumendpoints + - ciliumidentities verbs: - get - list