diff --git a/drivers/acpi/acpi_memhotplug.c b/drivers/acpi/acpi_memhotplug.c index d0c1a71007d0a3054608bec8fddc8e86bdffb78b..456647c6ef8198f6f6a35795df7cd7b723a4dd28 100644 --- a/drivers/acpi/acpi_memhotplug.c +++ b/drivers/acpi/acpi_memhotplug.c @@ -57,6 +57,11 @@ struct acpi_memory_device { int mgid; }; +#if IS_ENABLED(CONFIG_HISI_HBMDEV) +struct acpi_device *hotplug_mdev[MAX_NUMNODES]; +EXPORT_SYMBOL_GPL(hotplug_mdev); +#endif + static acpi_status acpi_memory_get_resource(struct acpi_resource *resource, void *context) { @@ -167,6 +172,27 @@ static void acpi_unbind_memory_blocks(struct acpi_memory_info *info) acpi_unbind_memblk); } +#if IS_ENABLED(CONFIG_HISI_HBMDEV) +static void hotplug_mdev_set(struct acpi_memory_device *mem_device) +{ + acpi_handle handle = mem_device->device->handle; + int nid = acpi_get_node(handle); + + hotplug_mdev[nid] = mem_device->device; +} + +static void hotplug_mdev_clear(struct acpi_memory_device *mem_device) +{ + acpi_handle handle = mem_device->device->handle; + int nid = acpi_get_node(handle); + + hotplug_mdev[nid] = NULL; +} +#else +static void hotplug_mdev_set(struct acpi_memory_device *mem_device) {} +static void hotplug_mdev_clear(struct acpi_memory_device *mem_device) {} +#endif + static int acpi_memory_enable_device(struct acpi_memory_device *mem_device) { acpi_handle handle = mem_device->device->handle; @@ -235,6 +261,8 @@ static int acpi_memory_enable_device(struct acpi_memory_device *mem_device) * Add num_enable even if add_memory() returns -EEXIST, so the * device is bound to this driver. */ + + hotplug_mdev_set(mem_device); num_enabled++; } if (!num_enabled) { @@ -256,6 +284,7 @@ static void acpi_memory_remove_memory(struct acpi_memory_device *mem_device) { struct acpi_memory_info *info, *n; + hotplug_mdev_clear(mem_device); list_for_each_entry_safe(info, n, &mem_device->res_list, list) { if (!info->enabled) continue; diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h index 866c7c4ed2331710736606664e19098b49a4098e..253afdcf3aa994f5f58486bff6d445381216cbde 100644 --- a/drivers/acpi/internal.h +++ b/drivers/acpi/internal.h @@ -77,7 +77,6 @@ static inline void acpi_lpss_init(void) {} void acpi_apd_init(void); -acpi_status acpi_hotplug_schedule(struct acpi_device *adev, u32 src); bool acpi_queue_hotplug_work(struct work_struct *work); void acpi_device_hotplug(struct acpi_device *adev, u32 src); bool acpi_scan_is_offline(struct acpi_device *adev, bool uevent); diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c index f725813d0cce6a4c5e0418334a9d4bd812dc8ef8..ae4f9818b6b380a578d7ecba4d2820688b2e971f 100644 --- a/drivers/acpi/osl.c +++ b/drivers/acpi/osl.c @@ -1190,6 +1190,7 @@ acpi_status acpi_hotplug_schedule(struct acpi_device *adev, u32 src) } return AE_OK; } +EXPORT_SYMBOL_GPL(acpi_hotplug_schedule); bool acpi_queue_hotplug_work(struct work_struct *work) { diff --git a/drivers/base/container.c b/drivers/base/container.c index 1ba42d2d353223e683be57bad5154df07abda10e..055dfa8daff1278212eab1d0f6a7a58085decf17 100644 --- a/drivers/base/container.c +++ b/drivers/base/container.c @@ -30,6 +30,9 @@ struct bus_type container_subsys = { .online = trivial_online, .offline = container_offline, }; +#if IS_ENABLED(CONFIG_HISI_HBMDEV) +EXPORT_SYMBOL_GPL(container_subsys); +#endif void __init container_dev_init(void) { diff --git a/drivers/soc/hisilicon/Kconfig b/drivers/soc/hisilicon/Kconfig index 0ab688af308fed625ec8c04723a87911b8da7373..0a07d0e266a556a4c9b27e505a549d91af21a1c8 100644 --- a/drivers/soc/hisilicon/Kconfig +++ b/drivers/soc/hisilicon/Kconfig @@ -3,6 +3,42 @@ menu "Hisilicon SoC drivers" depends on ARCH_HISI || COMPILE_TEST +config HISI_HBMDEV + tristate "add extra support for hbm memory device" + depends on ACPI_HOTPLUG_MEMORY + select ACPI_CONTAINER + help + This driver add two extra supports for memory devices. The driver + provides methods for userpace to control the power of memory devices + in a container. Besides, it provides extra locality information + between cpus and memory devices for userspace, which can take + advantage of this functionality to select the closet memory device + to a certain cpu. + + To compile this driver as a module, choose M here: + the module will be called hisi_hbmdev. + +config HISI_HBMCACHE + tristate "HBM cache memory device" + depends on ACPI + help + This driver provids methods to control the power of hbm cache device + in hisi soc. Use hbm as a cache can take advantage of hbm's high + bandwidth in normal memory access. + + To compile the driver as a module, choose M here: + the module will be called hisi_hbmcache. + +config HISI_HBMDEV_ACLS + bool "Add support for HISI ACLS repair" + depends on HISI_HBMDEV + help + Add ACLS support for hbm device, which can be used to query and + repair hardware error in HBM devices. This feature need to work with + hardware firmwares. + + If not sure say no. + config KUNPENG_HCCS tristate "HCCS driver on Kunpeng SoC" depends on ACPI diff --git a/drivers/soc/hisilicon/Makefile b/drivers/soc/hisilicon/Makefile index 226e747e70d67511e1954010e143d39df6b73f55..cbc01ad5b8fd3cd9557e4e67cf10c846e5106c8d 100644 --- a/drivers/soc/hisilicon/Makefile +++ b/drivers/soc/hisilicon/Makefile @@ -1,2 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_KUNPENG_HCCS) += kunpeng_hccs.o + +obj-$(CONFIG_HISI_HBMDEV) += hisi_hbmdev.o +obj-$(CONFIG_HISI_HBMCACHE) += hisi_hbmcache.o diff --git a/drivers/soc/hisilicon/hisi_hbmcache.c b/drivers/soc/hisilicon/hisi_hbmcache.c new file mode 100644 index 0000000000000000000000000000000000000000..34121320742ea6f151380470c8df0f88bfd8f129 --- /dev/null +++ b/drivers/soc/hisilicon/hisi_hbmcache.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2023. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include "hisi_internal.h" + +#define MODULE_NAME "hbm_cache" + +static struct kobject *cache_kobj; + +static ssize_t state_store(struct device *d, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct acpi_device *adev = ACPI_COMPANION(d); + const int type = online_type_from_str(buf); + int ret = -EINVAL; + + switch (type) { + case STATE_ONLINE: + ret = acpi_device_set_power(adev, ACPI_STATE_D0); + break; + case STATE_OFFLINE: + ret = acpi_device_set_power(adev, ACPI_STATE_D3); + break; + default: + break; + } + + if (ret) + return ret; + + return count; +} + +static ssize_t state_show(struct device *d, struct device_attribute *attr, + char *buf) +{ + struct acpi_device *adev = ACPI_COMPANION(d); + unsigned long long sta = 0; + acpi_status status; + const char *output; + + status = acpi_evaluate_integer(adev->handle, "_STA", NULL, &sta); + if (ACPI_FAILURE(status)) + return -EINVAL; + + output = (sta & 0x01) ? online_type_to_str[STATE_ONLINE] : + online_type_to_str[STATE_OFFLINE]; + + return sysfs_emit(buf, "%s\n", output); +} +static DEVICE_ATTR_RW(state); + +static ssize_t socket_id_show(struct device *d, struct device_attribute *attr, + char *buf) +{ + int socket_id; + + if (device_property_read_u32(d, "socket_id", &socket_id)) + return -EINVAL; + + return sysfs_emit(buf, "%d\n", socket_id); +} +static DEVICE_ATTR_RO(socket_id); + +static struct attribute *attrs[] = { + &dev_attr_state.attr, + &dev_attr_socket_id.attr, + NULL, +}; + +static struct attribute_group attr_group = { + .attrs = attrs, +}; + +static int cache_probe(struct platform_device *pdev) +{ + int ret; + + ret = sysfs_create_group(&pdev->dev.kobj, &attr_group); + if (ret) + return ret; + + ret = sysfs_create_link(cache_kobj, + &pdev->dev.kobj, + kobject_name(&pdev->dev.kobj)); + if (ret) { + sysfs_remove_group(&pdev->dev.kobj, &attr_group); + return ret; + } + + return 0; +} + +static int cache_remove(struct platform_device *pdev) +{ + sysfs_remove_group(&pdev->dev.kobj, &attr_group); + sysfs_remove_link(&pdev->dev.kobj, + kobject_name(&pdev->dev.kobj)); + return 0; +} + +static const struct acpi_device_id cache_acpi_ids[] = { + {"HISI04A1"}, + {}, +}; + +static struct platform_driver hbm_cache_driver = { + .probe = cache_probe, + .remove = cache_remove, + .driver = { + .name = MODULE_NAME, + .acpi_match_table = ACPI_PTR(cache_acpi_ids), + }, +}; + +static int __init hbm_cache_module_init(void) +{ + int ret; + + cache_kobj = kobject_create_and_add("hbm_cache", kernel_kobj); + if (!cache_kobj) + return -ENOMEM; + + ret = platform_driver_register(&hbm_cache_driver); + if (ret) { + kobject_put(cache_kobj); + return ret; + } + return 0; +} +module_init(hbm_cache_module_init); + +static void __exit hbm_cache_module_exit(void) +{ + kobject_put(cache_kobj); + platform_driver_unregister(&hbm_cache_driver); +} +module_exit(hbm_cache_module_exit); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/hisilicon/hisi_hbmdev.c b/drivers/soc/hisilicon/hisi_hbmdev.c new file mode 100644 index 0000000000000000000000000000000000000000..e60fe1c14c0ff94e71140055a7f1b4849d8af5b0 --- /dev/null +++ b/drivers/soc/hisilicon/hisi_hbmdev.c @@ -0,0 +1,435 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2023. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hisi_internal.h" + +#define ACPI_MEMORY_DEVICE_HID "PNP0C80" +#define ACPI_GENERIC_CONTAINER_DEVICE_HID "PNP0A06" + +struct cdev_node { + struct device *dev; + struct list_head clist; +}; + +struct memory_dev { + struct kobject *memdev_kobj; + struct kobject *topo_kobj; +#ifdef CONFIG_HISI_HBMDEV_ACLS + struct kobject *acls_kobj; +#endif + struct cdev_node cdev_list; + nodemask_t cluster_cpumask[MAX_NUMNODES]; +}; + +static struct memory_dev *mdev; + +static ssize_t memory_locality_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + int i, count = 0; + + for (i = 0; i < MAX_NUMNODES; i++) { + if (hotplug_mdev[i] != NULL && !nodes_empty(mdev->cluster_cpumask[i])) { + count += sysfs_emit_at(buf, count, "%d %*pbl\n", i, + nodemask_pr_args(&mdev->cluster_cpumask[i])); + } + } + + return count; +} + +static struct kobj_attribute memory_locality_attribute = + __ATTR(memory_locality, 0444, memory_locality_show, NULL); + +static void memory_topo_init(void) +{ + int ret, nid, cluster_id, cpu; + struct acpi_device *adev; + nodemask_t mask; + + for (nid = 0; nid < MAX_NUMNODES; nid++) { + if (!hotplug_mdev[nid]) + continue; + + adev = hotplug_mdev[nid]; + ret = fwnode_property_read_u32(acpi_fwnode_handle(adev), + "cluster-id", &cluster_id); + if (ret < 0) { + pr_debug("Failed to read cluster id\n"); + return; + } + + nodes_clear(mask); + for_each_possible_cpu(cpu) { + if (topology_cluster_id(cpu) == cluster_id) + node_set(cpu, mask); + } + mdev->cluster_cpumask[nid] = mask; + } + + mdev->topo_kobj = kobject_create_and_add("memory_topo", mdev->memdev_kobj); + if (!mdev->topo_kobj) + return; + + ret = sysfs_create_file(mdev->topo_kobj, &memory_locality_attribute.attr); + if (ret) + kobject_put(mdev->topo_kobj); +} + +#ifdef CONFIG_HISI_HBMDEV_ACLS +static struct acpi_device *paddr_to_acpi_device(u64 paddr) +{ + unsigned long pfn; + int nid; + + pfn = __phys_to_pfn(paddr); + if (!pfn_valid(pfn)) + return NULL; + + nid = pfn_to_nid(pfn); + if (nid < 0 && nid >= MAX_NUMNODES) + return NULL; + + return hotplug_mdev[nid]; +} + +static ssize_t acls_query_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct acpi_object_list arg_list; + struct acpi_device *adev; + union acpi_object obj; + acpi_status status; + u64 paddr, res; + + if (kstrtoull(buf, 16, &paddr)) + return -EINVAL; + + adev = paddr_to_acpi_device(paddr); + if (!adev) + return -EINVAL; + + obj.type = ACPI_TYPE_INTEGER; + obj.integer.value = paddr; + arg_list.count = 1; + arg_list.pointer = &obj; + + status = acpi_evaluate_integer(adev->handle, "AQRY", &arg_list, &res); + if (ACPI_FAILURE(status)) + return -ENODEV; + + /* AQRY will return a positive error code to represent error status */ + if (IS_ERR_VALUE(-res)) + return -res; + else if (res) + return -ENODEV; + + return count; +} + +static struct kobj_attribute acls_query_store_attribute = + __ATTR(acls_query, 0200, NULL, acls_query_store); + +static ssize_t acls_repair_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct acpi_object_list arg_list; + struct acpi_device *adev; + union acpi_object obj; + acpi_status status; + u64 paddr, res; + + if (kstrtoull(buf, 16, &paddr)) + return -EINVAL; + + adev = paddr_to_acpi_device(paddr); + if (!adev) + return -EINVAL; + + obj.type = ACPI_TYPE_INTEGER; + obj.integer.value = paddr; + arg_list.count = 1; + arg_list.pointer = &obj; + + status = acpi_evaluate_integer(adev->handle, "AREP", &arg_list, &res); + if (ACPI_FAILURE(status)) + return -ENODEV; + + /* AREP will return a positive error code to represent error status */ + if (IS_ERR_VALUE(-res)) + return -res; + else if (res) + return -ENODEV; + + return count; +} +static struct kobj_attribute acls_repair_store_attribute = + __ATTR(acls_repair, 0200, NULL, acls_repair_store); + +static struct attribute *acls_attrs[] = { + &acls_query_store_attribute.attr, + &acls_repair_store_attribute.attr, + NULL, +}; + +static struct attribute_group acls_attr_group = { + .attrs = acls_attrs, +}; + +static void acls_init(void) +{ + int ret = -ENOMEM; + + mdev->acls_kobj = kobject_create_and_add("acls", mdev->memdev_kobj); + if (!mdev->acls_kobj) + goto out; + + ret = sysfs_create_group(mdev->acls_kobj, &acls_attr_group); + if (ret) + kobject_put(mdev->acls_kobj); + +out: + if (ret) + pr_err("ACLS hot repair is not enabled\n"); +} + +static void acls_remove(void) +{ + kobject_put(mdev->acls_kobj); +} +#else +static void acls_init(void) {} +static void acls_remove(void) {} +#endif + +static int get_pxm(struct acpi_device *acpi_device, void *arg) +{ + acpi_handle handle = acpi_device->handle; + nodemask_t *mask = arg; + unsigned long long sta; + acpi_status status; + int nid; + + status = acpi_evaluate_integer(handle, "_STA", NULL, &sta); + if (ACPI_SUCCESS(status) && (sta & ACPI_STA_DEVICE_ENABLED)) { + nid = acpi_get_node(handle); + if (nid >= 0) + node_set(nid, *mask); + } + + return 0; +} + +static ssize_t pxms_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct acpi_device *adev = ACPI_COMPANION(dev); + nodemask_t mask; + + nodes_clear(mask); + acpi_dev_for_each_child(adev, get_pxm, &mask); + + return sysfs_emit(buf, "%*pbl\n", + nodemask_pr_args(&mask)); +} +static DEVICE_ATTR_RO(pxms); + +static int memdev_power_on(struct acpi_device *adev) +{ + acpi_handle handle = adev->handle; + acpi_status status; + + status = acpi_evaluate_object(handle, "_ON", NULL, NULL); + if (ACPI_FAILURE(status)) { + acpi_handle_warn(handle, "Power on failed (0x%x)\n", status); + return -ENODEV; + } + + return 0; +} + +static int eject_device(struct acpi_device *acpi_device, void *not_used) +{ + acpi_object_type unused; + acpi_status status; + + status = acpi_get_type(acpi_device->handle, &unused); + if (ACPI_FAILURE(status) || !acpi_device->flags.ejectable) + return -ENODEV; + + get_device(&acpi_device->dev); + status = acpi_hotplug_schedule(acpi_device, ACPI_OST_EC_OSPM_EJECT); + if (ACPI_SUCCESS(status)) + return 0; + + put_device(&acpi_device->dev); + acpi_evaluate_ost(acpi_device->handle, ACPI_OST_EC_OSPM_EJECT, + ACPI_OST_SC_NON_SPECIFIC_FAILURE, NULL); + + return status == AE_NO_MEMORY ? -ENOMEM : -EAGAIN; +} + +static int memdev_power_off(struct acpi_device *adev) +{ + return acpi_dev_for_each_child(adev, eject_device, NULL); +} + +static ssize_t state_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct acpi_device *adev = ACPI_COMPANION(dev); + const int type = online_type_from_str(buf); + int ret = -EINVAL; + + switch (type) { + case STATE_ONLINE: + ret = memdev_power_on(adev); + break; + case STATE_OFFLINE: + ret = memdev_power_off(adev); + break; + default: + break; + } + + if (ret) + return ret; + + return count; +} +static DEVICE_ATTR_WO(state); + +static int hbmdev_find(struct acpi_device *adev, void *arg) +{ + const char *hid = acpi_device_hid(adev); + bool *found = arg; + + if (!strcmp(hid, ACPI_MEMORY_DEVICE_HID)) { + *found = true; + return -1; + } + + return 0; +} + +static bool has_hbmdev(struct device *dev) +{ + struct acpi_device *adev = ACPI_COMPANION(dev); + const char *hid = acpi_device_hid(adev); + bool found = false; + + if (strcmp(hid, ACPI_GENERIC_CONTAINER_DEVICE_HID)) + return found; + + acpi_dev_for_each_child(adev, hbmdev_find, &found); + + return found; +} + +static int container_add(struct device *dev, void *data) +{ + struct cdev_node *cnode; + + if (!has_hbmdev(dev)) + return 0; + + cnode = kmalloc(sizeof(struct cdev_node), GFP_KERNEL); + if (!cnode) + return -ENOMEM; + + cnode->dev = dev; + list_add_tail(&cnode->clist, &mdev->cdev_list.clist); + + return 0; +} + +static void container_remove(void) +{ + struct cdev_node *cnode, *tmp; + + list_for_each_entry_safe(cnode, tmp, &mdev->cdev_list.clist, clist) { + device_remove_file(cnode->dev, &dev_attr_state); + device_remove_file(cnode->dev, &dev_attr_pxms); + list_del(&cnode->clist); + kfree(cnode); + } +} + +static int container_init(void) +{ + struct cdev_node *cnode; + + INIT_LIST_HEAD(&mdev->cdev_list.clist); + + if (bus_for_each_dev(&container_subsys, NULL, NULL, container_add)) { + container_remove(); + return -ENOMEM; + } + + if (list_empty(&mdev->cdev_list.clist)) + return -ENODEV; + + list_for_each_entry(cnode, &mdev->cdev_list.clist, clist) { + device_create_file(cnode->dev, &dev_attr_state); + device_create_file(cnode->dev, &dev_attr_pxms); + } + + return 0; +} + + +static int __init mdev_init(void) +{ + int ret; + + mdev = kzalloc(sizeof(struct memory_dev), GFP_KERNEL); + if (!mdev) + return -ENOMEM; + + ret = container_init(); + if (ret) { + kfree(mdev); + return ret; + } + + mdev->memdev_kobj = kobject_create_and_add("hbm_memory", kernel_kobj); + if (!mdev->memdev_kobj) { + container_remove(); + kfree(mdev); + return -ENOMEM; + } + + memory_topo_init(); + acls_init(); + return ret; +} +module_init(mdev_init); + +static void __exit mdev_exit(void) +{ + container_remove(); + kobject_put(mdev->memdev_kobj); + kobject_put(mdev->topo_kobj); + acls_remove(); + kfree(mdev); +} +module_exit(mdev_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Zhang Zekun "); diff --git a/drivers/soc/hisilicon/hisi_internal.h b/drivers/soc/hisilicon/hisi_internal.h new file mode 100644 index 0000000000000000000000000000000000000000..5345174f6b84f16400562c80ae87676e33820c39 --- /dev/null +++ b/drivers/soc/hisilicon/hisi_internal.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Huawei Technologies Co., Ltd. 2023. All rights reserved. + */ + +#ifndef _HISI_INTERNAL_H +#define _HISI_INTERNAL_H + +enum { + STATE_ONLINE, + STATE_OFFLINE, +}; + +static const char *const online_type_to_str[] = { + [STATE_ONLINE] = "online", + [STATE_OFFLINE] = "offline", +}; + +static inline int online_type_from_str(const char *str) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(online_type_to_str); i++) { + if (sysfs_streq(str, online_type_to_str[i])) + return i; + } + + return -EINVAL; +} + +#endif diff --git a/include/linux/acpi.h b/include/linux/acpi.h index afd94c9b8b8afd100c54fb07b5f1e8dcce08e92c..1c2bb14975af737dbb4f29fc7bfd47dcd507eedb 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1543,6 +1543,7 @@ static inline void acpi_init_ffh(void) { } #ifdef CONFIG_ACPI extern void acpi_device_notify(struct device *dev); extern void acpi_device_notify_remove(struct device *dev); +extern acpi_status acpi_hotplug_schedule(struct acpi_device *adev, u32 src); #else static inline void acpi_device_notify(struct device *dev) { } static inline void acpi_device_notify_remove(struct device *dev) { } diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 7d207658349416b0304722dfedf642f506d4c356..a7164c67dcd851de1facd9b741fc1e028c8c1a84 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -67,6 +67,10 @@ static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat) #ifdef CONFIG_MEMORY_HOTPLUG struct page *pfn_to_online_page(unsigned long pfn); +#if IS_ENABLED(CONFIG_HISI_HBMDEV) +extern struct acpi_device *hotplug_mdev[MAX_NUMNODES]; +#endif + /* Types for control the zone type of onlined and offlined memory */ enum { /* Offline the memory. */