Skip to content

Commit

Permalink
additional changes
Browse files Browse the repository at this point in the history 
Signed-off-by: Prateek <[email protected]>
  • Loading branch information
@Prateeknandle
Prateeknandle committed Jan 14, 2025
1 parent e7f1d52 commit 2bd7f2a
Show file tree
Hide file tree
Showing 5 changed files with 136 additions and 44 deletions.
18 changes: 18 additions & 0 deletions KubeArmor/common/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,24 @@ var ContainerRuntimeSocketMap = map[string][]string{
},
}

// NRISocketMap Structure
var NRISocketMap = map[string][]string{
"nri": {
"/var/run/nri/nri.sock",
"/run/nri/nri.sock",
},
}

// GetNRISocket Function
func GetNRISocket(ContainerRuntime string) string {
for _, candidate := range NRISocketMap["nri"] {
if _, err := os.Stat(candidate); err == nil {
return candidate
}
}
return ""
}

// GetCRISocket Function
func GetCRISocket(ContainerRuntime string) string {
for _, k := range ContainerRuntimeSocketKeys {
Expand Down
32 changes: 32 additions & 0 deletions KubeArmor/core/kubeArmor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,22 @@ func KubeArmor() {
// Un-orchestrated workloads
if !dm.K8sEnabled && cfg.GlobalCfg.Policy {

// Check if nri socket set, if not then auto detect
if cfg.GlobalCfg.NRISocket == "" {
if kl.GetNRISocket("") != "" {
cfg.GlobalCfg.NRISocket = kl.GetNRISocket("")
} else {
dm.Logger.Warnf("Error while looking for NRI socket file")
enableContainerPolicy = false
}
} else {
// NRI socket supplied by user, check for existence
_, err := os.Stat(cfg.GlobalCfg.NRISocket)
if err != nil {
dm.Logger.Warnf("Error while looking for NRI socket file %s", err.Error())
}
}

// Check if cri socket set, if not then auto detect
if cfg.GlobalCfg.CRISocket == "" {
if kl.GetCRISocket("") == "" {
Expand Down Expand Up @@ -614,6 +630,22 @@ func KubeArmor() {
}

if dm.K8sEnabled && cfg.GlobalCfg.Policy {
// Check if nri socket set, if not then auto detect
if cfg.GlobalCfg.NRISocket == "" {
if kl.GetNRISocket("") != "" {
cfg.GlobalCfg.NRISocket = kl.GetNRISocket("")
} else {
dm.Logger.Warnf("Error while looking for NRI socket file")
enableContainerPolicy = false
}
} else {
// NRI socket supplied by user, check for existence
_, err := os.Stat(cfg.GlobalCfg.NRISocket)
if err != nil {
dm.Logger.Warnf("Error while looking for NRI socket file %s", err.Error())
}
}

if cfg.GlobalCfg.NRISocket != "" {
// monitor NRI events
go dm.MonitorNRIEvents()
Expand Down
99 changes: 57 additions & 42 deletions KubeArmor/core/nriHandler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

"github.com/containerd/nri/pkg/api"
"github.com/containerd/nri/pkg/stub"
"github.com/kubearmor/KubeArmor/KubeArmor/common"
kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
kg "github.com/kubearmor/KubeArmor/KubeArmor/log"
Expand Down Expand Up @@ -45,18 +46,20 @@ type NRIHandler struct {
// active containers
containers map[string]tp.Container

dm *KubeArmorDaemon

containersByNamespaces map[namespaceKey]string

handleDeletedContainer func(tp.Container)
handleNewContainer func(tp.Container)
}

// NewNRIHandler creates a new NRIHandler with the given event callbacks.
func NewNRIHandler(
func (dm *KubeArmorDaemon) NewNRIHandler(
handleDeletedContainer func(tp.Container),
handleNewContainer func(tp.Container),
) *NRIHandler {
nri := &NRIHandler{}
nri := &NRIHandler{dm: dm}

opts := []stub.Option{
stub.WithSocketPath(cfg.GlobalCfg.NRISocket),
Expand Down Expand Up @@ -101,7 +104,7 @@ func (nh *NRIHandler) Synchronize(
nriContainers []*api.Container,
) ([]*api.ContainerUpdate, error) {
for _, nriContainer := range nriContainers {
container := nriToKubeArmorContainer(nriContainer)
container := nh.nriToKubeArmorContainer(nriContainer)
container = nh.mergeContainer(container, false)

// Overlapping namespace IDs between containers should be impossible
Expand Down Expand Up @@ -135,7 +138,7 @@ func (nh *NRIHandler) StartContainer(
_ *api.PodSandbox,
nriContainer *api.Container,
) error {
container := nriToKubeArmorContainer(nriContainer)
container := nh.nriToKubeArmorContainer(nriContainer)
container = nh.mergeContainer(container, false)

namespaceKey := namespaceKeyFromContainer(container)
Expand Down Expand Up @@ -175,7 +178,7 @@ func (nh *NRIHandler) StopContainer(
_ *api.PodSandbox,
nriContainer *api.Container,
) ([]*api.ContainerUpdate, error) {
container := nriToKubeArmorContainer(nriContainer)
container := nh.nriToKubeArmorContainer(nriContainer)
container = nh.mergeContainer(container, true)

// Only handle the container deleted event if it wasn't already 'deleted' by
Expand All @@ -201,7 +204,7 @@ func (nh *NRIHandler) RemoveContainer(
_ *api.PodSandbox,
nriContainer *api.Container,
) ([]*api.ContainerUpdate, error) {
container := nriToKubeArmorContainer(nriContainer)
container := nh.nriToKubeArmorContainer(nriContainer)
container = nh.mergeContainer(container, true)

// Only handle the container deleted event if it wasn't already 'deleted' by
Expand Down Expand Up @@ -237,7 +240,7 @@ func (nh *NRIHandler) mergeContainer(container tp.Container, removing bool) tp.C
}

// nriToKubeArmorContainer converts an NRI container to a KubeArmor container.
func nriToKubeArmorContainer(nriContainer *api.Container) tp.Container {
func (nh *NRIHandler) nriToKubeArmorContainer(nriContainer *api.Container) tp.Container {
container := tp.Container{}

container.ContainerID = nriContainer.Id
Expand All @@ -264,16 +267,20 @@ func nriToKubeArmorContainer(nriContainer *api.Container) tp.Container {
podNamespace = namespace
}

pod, err := K8s.K8sClient.CoreV1().Pods(podNamespace).Get(context.TODO(), podName, metav1.GetOptions{})
if err != nil {
kg.Warnf("failed to fetch Pod: %w\n", err)
}
if nh.dm.K8sEnabled {
pod, err := K8s.K8sClient.CoreV1().Pods(podNamespace).Get(context.TODO(), podName, metav1.GetOptions{})
if err != nil {
kg.Warnf("failed to fetch Pod: %w\n", err)
}

if appArmorProfile, ok := pod.Annotations["container.apparmor.security.beta.kubernetes.io/"+nriContainer.Name]; ok {
profile := strings.Split(appArmorProfile, "/")
if len(profile) > 1 {
container.AppArmorProfile = profile[1]
if appArmorProfile, ok := pod.Annotations["container.apparmor.security.beta.kubernetes.io/"+nriContainer.Name]; ok {
profile := strings.Split(appArmorProfile, "/")
if len(profile) > 1 {
container.AppArmorProfile = profile[1]
}
}
} else {
container.AppArmorProfile = "kubearmor_" + container.ContainerName
}

// Read PID and mount namespaces from container root PID
Expand Down Expand Up @@ -301,32 +308,6 @@ func (dm *KubeArmorDaemon) MonitorNRIEvents() {
dm.WgDaemon.Add(1)
defer dm.WgDaemon.Done()

handleDeletedContainer := func(container tp.Container) {
dm.ContainersLock.Lock()
_, ok := dm.Containers[container.ContainerID]
if !ok {
dm.ContainersLock.Unlock()
return
}
if !dm.K8sEnabled {
dm.EndPointsLock.Lock()
dm.MatchandRemoveContainerFromEndpoint(container.ContainerID)
dm.EndPointsLock.Unlock()
}
delete(dm.Containers, container.ContainerID)
dm.ContainersLock.Unlock()

// TODO: Can't update AppArmor profiles since we can't get them from NRI

if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
// update NsMap
dm.SystemMonitor.DeleteContainerIDFromNsMap(container.ContainerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.UnregisterContainer(container.ContainerID)
}

dm.Logger.Printf("Detected a container (removed/%.12s/pidns=%d/mntns=%d)", container.ContainerID, container.PidNS, container.MntNS)
}

handleNewContainer := func(container tp.Container) {
endpoint := tp.EndPoint{}

Expand Down Expand Up @@ -382,6 +363,11 @@ func (dm *KubeArmorDaemon) MonitorNRIEvents() {
}

if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
// for throttling
dm.SystemMonitor.Logger.ContainerNsKey[container.ContainerID] = common.OuterKey{
MntNs: container.MntNS,
PidNs: container.PidNS,
}
// update NsMap
dm.SystemMonitor.AddContainerIDToNsMap(container.ContainerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.RegisterContainer(container.ContainerID, container.PidNS, container.MntNS)
Expand All @@ -406,7 +392,36 @@ func (dm *KubeArmorDaemon) MonitorNRIEvents() {
dm.Logger.Printf("Detected a container (added/%.12s/pidns=%d/mntns=%d)", container.ContainerID, container.PidNS, container.MntNS)
}

NRI = NewNRIHandler(handleDeletedContainer, handleNewContainer)
handleDeletedContainer := func(container tp.Container) {
dm.ContainersLock.Lock()
_, ok := dm.Containers[container.ContainerID]
if !ok {
dm.ContainersLock.Unlock()
return
}
if !dm.K8sEnabled {
dm.EndPointsLock.Lock()
dm.MatchandRemoveContainerFromEndpoint(container.ContainerID)
dm.EndPointsLock.Unlock()
}
delete(dm.Containers, container.ContainerID)
dm.ContainersLock.Unlock()

// TODO: Can't update AppArmor profiles since we can't get them from NRI

if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
outkey := dm.SystemMonitor.Logger.ContainerNsKey[container.ContainerID]
dm.Logger.DeleteAlertMapKey(outkey)
delete(dm.SystemMonitor.Logger.ContainerNsKey, container.ContainerID)
// update NsMap
dm.SystemMonitor.DeleteContainerIDFromNsMap(container.ContainerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.UnregisterContainer(container.ContainerID)
}

dm.Logger.Printf("Detected a container (removed/%.12s/pidns=%d/mntns=%d)", container.ContainerID, container.PidNS, container.MntNS)
}

NRI = dm.NewNRIHandler(handleDeletedContainer, handleNewContainer)

// check if NRI exists
if NRI == nil {
Expand Down
9 changes: 9 additions & 0 deletions pkg/KubeArmorOperator/common/defaults.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,8 @@ var ContainerRuntimeSocketMap = map[string][]string{
"/run/docker.sock",
},
"containerd": {
"/var/run/nri/nri.sock",
"/run/nri/nri.sock",
"/var/snap/microk8s/common/run/containerd.sock",
"/run/k0s/containerd.sock",
"/run/k3s/containerd/containerd.sock",
Expand All @@ -163,9 +165,15 @@ var ContainerRuntimeSocketMap = map[string][]string{
"/run/dockershim.sock",
},
"cri-o": {
"/var/run/nri/nri.sock",
"/run/nri/nri.sock",
"/var/run/crio/crio.sock",
"/run/crio/crio.sock",
},
"nri": {
"/var/run/nri/nri.sock",
"/run/nri/nri.sock",
},
}

var HostPathDirectory = corev1.HostPathDirectory
Expand Down Expand Up @@ -200,6 +208,7 @@ var RuntimeSocketLocation = map[string]string{
"docker": "/var/run/docker.sock",
"containerd": "/var/run/containerd/containerd.sock",
"cri-o": "/var/run/crio/crio.sock",
"nri": "/var/run/nri/nri.sock",
}

func ShortSHA(s string) string {
Expand Down
22 changes: 20 additions & 2 deletions pkg/KubeArmorOperator/runtime/runtime.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,37 @@ package runtime

import (
"os"
"path/filepath"
"strings"

"github.com/kubearmor/KubeArmor/KubeArmor/log"
"github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/common"
"go.uber.org/zap"
)

func DetectNRI(pathPrefix, runtime string) (string, string) {
for _, path := range common.ContainerRuntimeSocketMap[runtime] {
if _, err := os.Stat(filepath.Clean(pathPrefix + path)); err == nil || os.IsPermission(err) {
if strings.Contains(path, "nri") {
return "nri", path
}
return runtime, path
} else {
log.Warnf("%s", err)
}
}
return "NA", "NA"
}

func DetectRuntimeViaMap(pathPrefix string, k8sRuntime string, log zap.SugaredLogger) (string, string) {
log.Infof("Checking for %s socket\n", k8sRuntime)
if k8sRuntime != "" {
for _, path := range common.ContainerRuntimeSocketMap[k8sRuntime] {
if _, err := os.Stat(pathPrefix + path); err == nil || os.IsPermission(err) {
if k8sRuntime == "docker" && strings.Contains(path, "containerd") {
return "containerd", path
if (k8sRuntime == "docker" && strings.Contains(path, "containerd")) || k8sRuntime == "containerd" {
return DetectNRI(pathPrefix, "containerd")
} else if k8sRuntime == "cri-o" && strings.Contains(path, "nri") {
return "nri", path
}
return k8sRuntime, path
} else {
Expand Down

0 comments on commit 2bd7f2a

Please sign in to comment.