From 888d41666fdfdc10a9b59bd8ad34ebfdabe0ce11 Mon Sep 17 00:00:00 2001 From: Li Feng Date: Fri, 28 May 2021 14:28:53 +0800 Subject: [PATCH] arm: support arm itrustee device plugin Signed-off-by: Li Feng --- ...d-support-arm-itrustee-device-plugin.patch | 354 ++++++++++++++++++ ...n-image.sh => make-secgear-plugin-image.sh | 9 +- series.conf | 1 + 3 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 0003-itrustee-add-support-arm-itrustee-device-plugin.patch rename make-sgx-plugin-image.sh => make-secgear-plugin-image.sh (90%) diff --git a/0003-itrustee-add-support-arm-itrustee-device-plugin.patch b/0003-itrustee-add-support-arm-itrustee-device-plugin.patch new file mode 100644 index 0000000..53f1a6b --- /dev/null +++ b/0003-itrustee-add-support-arm-itrustee-device-plugin.patch @@ -0,0 +1,354 @@ +From f44becf8c83443255ca9e9019b2a4be430ce14d2 Mon Sep 17 00:00:00 2001 +From: Li Feng +Date: Fri, 28 May 2021 10:03:43 +0800 +Subject: [PATCH] itrustee: add support arm itrustee device plugin + +steps to build sgx device plugin binary: +1. cd cmd/arm_itrustee +2. go env -w GOPROXY=https://goproxy.cn +3. GO111MODULE=on go build + +steps to build arm itrustee plugin image: +1. sudo make arm-itrustee-plugin + +Signed-off-by: LiFeng +Signed-off-by: Li Feng +--- + build/docker/arm-itrustee-plugin.Dockerfile | 24 +++ + cmd/arm_itrustee/README.md | 1 + + cmd/arm_itrustee/arm_itrustee.go | 123 ++++++++++++++++ + cmd/arm_itrustee/arm_itrustee_test.go | 153 ++++++++++++++++++++ + 4 files changed, 301 insertions(+) + create mode 100644 build/docker/arm-itrustee-plugin.Dockerfile + create mode 100644 cmd/arm_itrustee/README.md + create mode 100644 cmd/arm_itrustee/arm_itrustee.go + create mode 100644 cmd/arm_itrustee/arm_itrustee_test.go + +diff --git a/build/docker/arm-itrustee-plugin.Dockerfile b/build/docker/arm-itrustee-plugin.Dockerfile +new file mode 100644 +index 0000000..574569c +--- /dev/null ++++ b/build/docker/arm-itrustee-plugin.Dockerfile +@@ -0,0 +1,24 @@ ++# GOLANG_IMAGE can be used to make the build ++# ++# This is used on release branches before tagging a stable version. ++# The master branch defaults to using the latest golang and centos. ++ARG GOLANG_IMAGE=golang:1.15 ++ ++FROM ${GOLANG_IMAGE} as builder ++ ++ARG DIR=/intel-device-plugins-for-kubernetes ++ARG GO111MODULE=on ++WORKDIR $DIR ++COPY . . ++ ++RUN mkdir /install_root ++ ++RUN cd cmd/arm_itrustee; go env -w GOPROXY=https://goproxy.cn; GO111MODULE=${GO111MODULE} go install; cd - ++RUN chmod a+x /go/bin/arm_itrustee \ ++ && install -D /go/bin/arm_itrustee /install_root/usr/local/bin/arm_itrustee_device_plugin \ ++ && install -D ${DIR}/LICENSE /install_root/usr/local/share/package-licenses/intel-device-plugins-for-kubernetes/LICENSE \ ++ && scripts/copy-modules-licenses.sh ./cmd/arm_itrustee /install_root/usr/local/share/ ++ ++FROM ubuntu:20.04 as final ++COPY --from=builder /install_root / ++ENTRYPOINT ["/usr/local/bin/arm_itrustee_device_plugin"] +diff --git a/cmd/arm_itrustee/README.md b/cmd/arm_itrustee/README.md +new file mode 100644 +index 0000000..35d57d3 +--- /dev/null ++++ b/cmd/arm_itrustee/README.md +@@ -0,0 +1 @@ ++# Device Plugin for arm iTrustee +diff --git a/cmd/arm_itrustee/arm_itrustee.go b/cmd/arm_itrustee/arm_itrustee.go +new file mode 100644 +index 0000000..2e60799 +--- /dev/null ++++ b/cmd/arm_itrustee/arm_itrustee.go +@@ -0,0 +1,123 @@ ++// Copyright 2021 Huawei Technologies Co., Ltd. All Rights Reserved. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++package main ++ ++import ( ++ "flag" ++ "fmt" ++ "os" ++ "path" ++ "runtime" ++ "strconv" ++ ++ dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" ++ "k8s.io/klog/v2" ++ pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" ++) ++ ++const ( ++ // Device plugin settings. ++ namespace = "arm.itrustee" ++ deviceTypeEnclave = "enclave" ++ devicePath = "/dev" ++ podsPerCoreEnvVariable = "PODS_PER_CORE" ++ defaultPodCount uint = 100 ++) ++ ++type devicePlugin struct { ++ devfsDir string ++ nEnclave uint ++ scanDone chan bool ++} ++ ++func newDevicePlugin(devfsDir string, nEnclave uint) *devicePlugin { ++ return &devicePlugin{ ++ devfsDir: devfsDir, ++ nEnclave: nEnclave, ++ scanDone: make(chan bool, 1), ++ } ++} ++ ++func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error { ++ devTree, err := dp.scan() ++ if err != nil { ++ return err ++ } ++ notifier.Notify(devTree) ++ ++ // Wait forever to prevent manager run loop from exiting. ++ <-dp.scanDone ++ return nil ++} ++ ++func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) { ++ devTree := dpapi.NewDeviceTree() ++ ++ iTrusteeDevicePath := path.Join(dp.devfsDir, "tc_ns_client") ++ if _, err := os.Stat(iTrusteeDevicePath); err != nil { ++ klog.Error("No arm iTrustee enclave file available: ", err) ++ return devTree, nil ++ } ++ ++ deprecatedMounts := []pluginapi.Mount{ ++ { ++ HostPath: "/dev/tc_ns_client", ++ ContainerPath: "/dev/tc_ns_client", ++ }, ++ } ++ ++ for i := uint(0); i < dp.nEnclave; i++ { ++ devID := fmt.Sprintf("%s-%d", "arm-itrustee", i) ++ nodes := []pluginapi.DeviceSpec{{HostPath: iTrusteeDevicePath, ContainerPath: iTrusteeDevicePath, Permissions: "rw"}} ++ devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil)) ++ } ++ ++ return devTree, nil ++} ++ ++func getDefaultPodCount(nCPUs uint) uint { ++ // By default we provide as many enclave resources as there can be pods ++ // running on the node. The problem is that this value is configurable ++ // either via "--pods-per-core" or "--max-pods" kubelet options. We get the ++ // limit by multiplying the number of cores in the system with env variable ++ // "PODS_PER_CORE". ++ ++ envPodsPerCore := os.Getenv(podsPerCoreEnvVariable) ++ if envPodsPerCore != "" { ++ tmp, err := strconv.ParseUint(envPodsPerCore, 10, 32) ++ if err != nil { ++ klog.Errorf("Error: failed to parse %s value as uint, using default value.", podsPerCoreEnvVariable) ++ } else { ++ return uint(tmp) * nCPUs ++ } ++ } ++ ++ return defaultPodCount ++} ++ ++func main() { ++ var enclaveLimit uint ++ ++ podCount := getDefaultPodCount(uint(runtime.NumCPU())) ++ ++ flag.UintVar(&enclaveLimit, "enclave-limit", podCount, "Number of \"iTrustee enclave\" resources") ++ flag.Parse() ++ ++ klog.V(4).Infof("iTrustee device plugin started with %d \"%s/enclave\" resources.", enclaveLimit, namespace) ++ ++ plugin := newDevicePlugin(devicePath, enclaveLimit) ++ manager := dpapi.NewManager(namespace, plugin) ++ manager.Run() ++} +diff --git a/cmd/arm_itrustee/arm_itrustee_test.go b/cmd/arm_itrustee/arm_itrustee_test.go +new file mode 100644 +index 0000000..e7c4643 +--- /dev/null ++++ b/cmd/arm_itrustee/arm_itrustee_test.go +@@ -0,0 +1,153 @@ ++// Copyright 2021 Huawei Technologies Co., Ltd. All Rights Reserved. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++package main ++ ++import ( ++ "flag" ++ "io/ioutil" ++ "os" ++ "path" ++ "testing" ++ ++ dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" ++) ++ ++func init() { ++ _ = flag.Set("v", "4") // Enable debug output ++} ++ ++// mockNotifier implements Notifier interface. ++type mockNotifier struct { ++ scanDone chan bool ++ enclaveDevCount int ++} ++ ++// Notify stops plugin Scan. ++func (n *mockNotifier) Notify(newDeviceTree dpapi.DeviceTree) { ++ n.enclaveDevCount = len(newDeviceTree[deviceTypeEnclave]) ++ n.scanDone <- true ++} ++ ++func TestPodCount(t *testing.T) { ++ tcases := []struct { ++ name string ++ envValue string ++ nCPUs uint ++ expectedPodCount uint ++ }{ ++ { ++ name: "Default pod count", ++ envValue: "", ++ expectedPodCount: defaultPodCount, ++ }, ++ { ++ name: "Broken ENV", ++ envValue: "foobar", ++ nCPUs: 5, ++ expectedPodCount: defaultPodCount, ++ }, ++ { ++ name: "Valid ENV", ++ envValue: "2200", ++ nCPUs: 5, ++ expectedPodCount: 2200 * 5, ++ }, ++ } ++ for _, tc := range tcases { ++ t.Run(tc.name, func(t *testing.T) { ++ if err := os.Unsetenv(podsPerCoreEnvVariable); err != nil { ++ t.Fatalf("Unable to unset environment: %+v", err) ++ } ++ if tc.envValue != "" { ++ if err := os.Setenv(podsPerCoreEnvVariable, tc.envValue); err != nil { ++ t.Fatalf("Unable to set environment: %+v", err) ++ } ++ } ++ ++ count := getDefaultPodCount(tc.nCPUs) ++ ++ if tc.expectedPodCount != count { ++ t.Errorf("Wrong value for expected pod count") ++ } ++ }) ++ } ++} ++ ++func TestScan(t *testing.T) { ++ tcases := []struct { ++ name string ++ enclaveDevice string ++ requestedEnclaveDevs uint ++ expectedEnclaveDevs int ++ }{ ++ { ++ name: "no device installed", ++ }, ++ { ++ name: "only enclave file", ++ enclaveDevice: "tc_ns_client", ++ requestedEnclaveDevs: 1, ++ expectedEnclaveDevs: 1, ++ }, ++ { ++ name: "one device", ++ enclaveDevice: "tc_ns_client", ++ requestedEnclaveDevs: 1, ++ expectedEnclaveDevs: 1, ++ }, ++ { ++ name: "one device", ++ enclaveDevice: "tc_ns_client", ++ requestedEnclaveDevs: 10, ++ expectedEnclaveDevs: 10, ++ }, ++ } ++ ++ for _, tc := range tcases { ++ t.Run(tc.name, func(t *testing.T) { ++ root, err := ioutil.TempDir("", "test_itrustee_device_plugin_") ++ if err != nil { ++ t.Fatalf("can't create temporary directory: %+v", err) ++ } ++ defer func() { _ = os.RemoveAll(root) }() ++ ++ devfs := path.Join(root, "dev") ++ err = os.MkdirAll(devfs, 0750) ++ if err != nil { ++ t.Fatalf("Failed to create fake device directory: %+v", err) ++ } ++ if tc.enclaveDevice != "" { ++ err = ioutil.WriteFile(path.Join(devfs, tc.enclaveDevice), []byte{}, 0600) ++ if err != nil { ++ t.Fatalf("Failed to create fake enclave file: %+v", err) ++ } ++ } ++ ++ plugin := newDevicePlugin(devfs, tc.requestedEnclaveDevs) ++ ++ notifier := &mockNotifier{ ++ scanDone: plugin.scanDone, ++ } ++ ++ err = plugin.Scan(notifier) ++ if err != nil { ++ t.Errorf("Unexpected error: %v", err) ++ } ++ if tc.expectedEnclaveDevs != notifier.enclaveDevCount { ++ t.Errorf("Wrong number of discovered enclave devices") ++ } ++ }) ++ } ++} +-- +2.25.1 + diff --git a/make-sgx-plugin-image.sh b/make-secgear-plugin-image.sh similarity index 90% rename from make-sgx-plugin-image.sh rename to make-secgear-plugin-image.sh index 323af42..fd4fb17 100755 --- a/make-sgx-plugin-image.sh +++ b/make-secgear-plugin-image.sh @@ -33,6 +33,13 @@ do patch -p1 -F1 -s < $cwd/$line done -make intel-sgx-plugin + +arch=`uname -m` + +if [ "$arch" = "aarch64" ]; then + make arm-itrustee-plugin +else + make intel-sgx-plugin +fi cd $cwd diff --git a/series.conf b/series.conf index 14778c8..d946ba9 100644 --- a/series.conf +++ b/series.conf @@ -1,2 +1,3 @@ 0001-isgx-add-support-isgx-for-sgx-device-plugin.patch 0002-isgx-add-support-isgx-for-sgx-device-plugin-operator.patch +0003-itrustee-add-support-arm-itrustee-device-plugin.patch -- Gitee