From a31c4bc07c78b9e4a61961cbba9641793655fa04 Mon Sep 17 00:00:00 2001
From: Li Feng <lifeng2221dd1@zoho.com.cn>
Date: Thu, 6 May 2021 16:27:53 +0800
Subject: [PATCH] isgx: add support isgx for sgx device plugin

steps to build sgx device plugin binary:
1. cd cmd/sgx_plugin
2. go env -w GOPROXY=https://goproxy.cn
3. GO111MODULE=on go build

build sgx device plugin container image:

1. make intel-sgx-plugin

Signed-off-by: Li Feng <lifeng2221dd1@zoho.com.cn>
---
 .../docker/intel-sgx-initcontainer.Dockerfile |  1 +
 build/docker/intel-sgx-plugin.Dockerfile      |  2 +-
 cmd/sgx_plugin/sgx_plugin.go                  | 82 +++++++++++++------
 pkg/deviceplugin/server.go                    |  2 +
 4 files changed, 60 insertions(+), 27 deletions(-)

diff --git a/build/docker/intel-sgx-initcontainer.Dockerfile b/build/docker/intel-sgx-initcontainer.Dockerfile
index 8a50b21..f70559c 100644
--- a/build/docker/intel-sgx-initcontainer.Dockerfile
+++ b/build/docker/intel-sgx-initcontainer.Dockerfile
@@ -30,6 +30,7 @@ RUN mkdir /install_root \
 
 # Build NFD Feature Detector Hook
 RUN cd $DIR/cmd/sgx_epchook && \
+    go env -w GOPROXY=https://goproxy.cn && \
     GO111MODULE=${GO111MODULE} go install && \
     chmod a+x /go/bin/sgx_epchook && \
     cd $DIR && \
diff --git a/build/docker/intel-sgx-plugin.Dockerfile b/build/docker/intel-sgx-plugin.Dockerfile
index 8450d8f..5462771 100644
--- a/build/docker/intel-sgx-plugin.Dockerfile
+++ b/build/docker/intel-sgx-plugin.Dockerfile
@@ -27,7 +27,7 @@ RUN mkdir /install_root \
     --no-boot-update \
     && rm -rf /install_root/var/lib/swupd/*
 
-RUN cd cmd/sgx_plugin; GO111MODULE=${GO111MODULE} go install; cd -
+RUN cd cmd/sgx_plugin; go env -w GOPROXY=https://goproxy.cn; GO111MODULE=${GO111MODULE} go install; cd -
 RUN chmod a+x /go/bin/sgx_plugin \
     && install -D /go/bin/sgx_plugin /install_root/usr/local/bin/intel_sgx_device_plugin \
     && install -D ${DIR}/LICENSE /install_root/usr/local/share/package-licenses/intel-device-plugins-for-kubernetes/LICENSE \
diff --git a/cmd/sgx_plugin/sgx_plugin.go b/cmd/sgx_plugin/sgx_plugin.go
index 99d550f..9c92153 100644
--- a/cmd/sgx_plugin/sgx_plugin.go
+++ b/cmd/sgx_plugin/sgx_plugin.go
@@ -23,6 +23,7 @@ import (
 	"strconv"
 
 	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
+	"github.com/klauspost/cpuid/v2"
 	"k8s.io/klog/v2"
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 )
@@ -32,6 +33,7 @@ const (
 	namespace                   = "sgx.intel.com"
 	deviceTypeEnclave           = "enclave"
 	deviceTypeProvision         = "provision"
+	deviceTypeHuaweiEPC         = "huawei_sgx_epc_MiB"
 	devicePath                  = "/dev"
 	podsPerCoreEnvVariable      = "PODS_PER_CORE"
 	defaultPodCount        uint = 110
@@ -68,35 +70,63 @@ func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error {
 func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
 	devTree := dpapi.NewDeviceTree()
 
-	// Assume that both /dev/sgx_enclave and /dev/sgx_provision must be present.
-	sgxEnclavePath := path.Join(dp.devfsDir, "sgx_enclave")
-	sgxProvisionPath := path.Join(dp.devfsDir, "sgx_provision")
-	if _, err := os.Stat(sgxEnclavePath); err != nil {
-		klog.Error("No SGX enclave file available: ", err)
-		return devTree, nil
-	}
-	if _, err := os.Stat(sgxProvisionPath); err != nil {
-		klog.Error("No SGX provision file available: ", err)
-		return devTree, nil
-	}
+	iSgxDevicePath := path.Join(dp.devfsDir, "isgx")
+	if _, err := os.Stat(iSgxDevicePath); err != nil {
 
-	deprecatedMounts := []pluginapi.Mount{
-		{
-			HostPath:      "/dev/sgx",
-			ContainerPath: "/dev/sgx",
-		},
-	}
+		// Assume that both /dev/sgx_enclave and /dev/sgx_provision must be present.
+		sgxEnclavePath := path.Join(dp.devfsDir, "sgx_enclave")
+		sgxProvisionPath := path.Join(dp.devfsDir, "sgx_provision")
+		if _, err := os.Stat(sgxEnclavePath); err != nil {
+			klog.Error("No SGX enclave file available: ", err)
+			return devTree, nil
+		}
+		if _, err := os.Stat(sgxProvisionPath); err != nil {
+			klog.Error("No SGX provision file available: ", err)
+			return devTree, nil
+		}
 
-	for i := uint(0); i < dp.nEnclave; i++ {
-		devID := fmt.Sprintf("%s-%d", "sgx-enclave", i)
-		nodes := []pluginapi.DeviceSpec{{HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"}}
-		devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil))
-	}
-	for i := uint(0); i < dp.nProvision; i++ {
-		devID := fmt.Sprintf("%s-%d", "sgx-provision", i)
-		nodes := []pluginapi.DeviceSpec{{HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"}}
-		devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil))
+		deprecatedMounts := []pluginapi.Mount{
+			{
+				HostPath:      "/dev/sgx",
+				ContainerPath: "/dev/sgx",
+			},
+		}
+
+		for i := uint(0); i < dp.nEnclave; i++ {
+			devID := fmt.Sprintf("%s-%d", "sgx-enclave", i)
+			nodes := []pluginapi.DeviceSpec{{HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"}}
+			devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil))
+		}
+		for i := uint(0); i < dp.nProvision; i++ {
+			devID := fmt.Sprintf("%s-%d", "sgx-provision", i)
+			nodes := []pluginapi.DeviceSpec{{HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"}}
+			devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil))
+		}
+	} else {
+		// get the EPC size
+		var epcSize uint64
+		if cpuid.CPU.SGX.Available {
+			for _, s := range cpuid.CPU.SGX.EPCSections {
+				epcSize += s.EPCSize
+			}
+		}
+		klog.Infof("epc capacity: %d bytes", epcSize)
+
+		deprecatedMounts := []pluginapi.Mount{
+			{
+				HostPath:      "/dev/isgx",
+				ContainerPath: "/dev/isgx",
+			},
+		}
+
+		sizeMB := epcSize / 1024 / 1024
+		for i := uint64(0); i < sizeMB; i++ {
+			devID := fmt.Sprintf("%s-%d", "huawei_sgx_epc_MiB", i)
+			nodes := []pluginapi.DeviceSpec{{HostPath: iSgxDevicePath, ContainerPath: iSgxDevicePath, Permissions: "rw"}}
+			devTree.AddDevice(deviceTypeHuaweiEPC, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil))
+		}
 	}
+
 	return devTree, nil
 }
 
diff --git a/pkg/deviceplugin/server.go b/pkg/deviceplugin/server.go
index 10f4892..ea00312 100644
--- a/pkg/deviceplugin/server.go
+++ b/pkg/deviceplugin/server.go
@@ -110,10 +110,12 @@ func (srv *server) sendDevices(stream pluginapi.DevicePlugin_ListAndWatchServer)
 func (srv *server) ListAndWatch(empty *pluginapi.Empty, stream pluginapi.DevicePlugin_ListAndWatchServer) error {
 	klog.V(4).Info("Started ListAndWatch for", srv.devType)
 
+	// 主动发送，实现list
 	if err := srv.sendDevices(stream); err != nil {
 		return err
 	}
 
+	// 通过updatasCh 等待变更，主动发送变更到kubelet，实现watch
 	for srv.devices = range srv.updatesCh {
 		if err := srv.sendDevices(stream); err != nil {
 			return err
-- 
2.25.1